def df(): '''disk_usage''' df = [] for part in psutil.disk_partitions(all=False): usage = psutil.disk_usage(part.mountpoint) percent = str(int(usage.percent)) + '%' disk = [part.device, bytes2human(usage.total), bytes2human(usage.used), bytes2human(usage.free), percent, part.mountpoint] df.append(disk) return df
async def disk_check(self, data, exists=True): try: disk = psutil.disk_usage(data["disk"]) except Exception: exists = False self.logger.error(f"Disk {data['disk']} does not exist") if exists: percent_used = round(disk.percent) total = bytes2human(disk.total) free = bytes2human(disk.free) if (100 - percent_used) < data["threshold"]: status = "failing" else: status = "passing" more_info = f"Total: {total}B, Free: {free}B" else: status = "failing" more_info = f"Disk {data['disk']} does not exist" payload = { "status": status, "more_info": more_info, } self.logger.debug(payload) resp = requests.patch( f"{self.astor.server}/api/v1/{data['id']}/checkrunner/", json.dumps(payload), headers=self.headers, timeout=15, ) if ( status == "failing" and data["assigned_task"] and data["assigned_task"]["enabled"] ): from taskrunner import TaskRunner task = TaskRunner( task_pk=data["assigned_task"]["id"], log_level=self.log_level, log_to=self.log_to, ) await task.run_while_in_event_loop() return status
def df(): '''disk_usage''' df = [] for part in psutil.disk_partitions(all=False): usage = psutil.disk_usage(part.mountpoint) percent = str(int(usage.percent)) + '%' disk = [ part.device, bytes2human(usage.total), bytes2human(usage.used), bytes2human(usage.free), percent, part.mountpoint ] df.append(disk) return df
def get_rusage_dict(children=False): # TODO: # number of child processes? # page size # difference between a page out and a major fault? # NOTE: values commented with an * are untracked on Linux if not resource: return {} who = resource.RUSAGE_SELF if children: who = resource.RUSAGE_CHILDREN rr = resource.getrusage(who) max_rss_human = bytes2human(rr.ru_maxrss * 1024, ndigits=1) ret = {'cpu_times': {'user_time': rr.ru_utime, 'sys_time': rr.ru_stime}, 'memory': {'max_rss_human': max_rss_human, 'max_rss': rr.ru_maxrss, 'shared_rss': rr.ru_ixrss, # * 'unshared_rss': rr.ru_idrss, # * 'stack_rss': rr.ru_isrss}, # * 'page_faults': {'minor_faults': rr.ru_minflt, 'major_faults': rr.ru_majflt, 'page_outs': rr.ru_nswap}, # * 'blocking_io': {'input_ops': rr.ru_inblock, 'output_ops': rr.ru_oublock}, 'messages': {'sent': rr.ru_msgsnd, # * 'received': rr.ru_msgrcv}, # * 'signals': {'received': rr.ru_nsignals}, # * 'ctx_switches': {'voluntary': rr.ru_nvcsw, 'involuntary': rr.ru_nivcsw}} return ret
def df(): '''disk''' name_list = [ 'disk_name', 'total_space', 'used_space', 'free_space', 'perent' ] df = [] for part in psutil.disk_partitions(all=False): usage = psutil.disk_usage(part.mountpoint) percent = str(int(usage.percent)) + '%' disk = [ part.device, bytes2human(usage.total), bytes2human(usage.used), bytes2human(usage.free), percent ] df.append(dict(zip(name_list, disk))) return df
def global_stats(verbose=True): global STATS total = {key: 0 for key in STATS[0] if not 'human' in key} for stat in STATS: for key in total: total[key] += stats[key] for key in total: total[key] /= len(STATS) total['size_round_human'] = bytes2human(total['size_round']) + 'Byte' total['bps_node_human'] = bytes2human( total['byte_per_sec'] / HN_COUNT * 8) + 'bps' total['monthly_node_traffic_human'] = bytes2human( total['byte_per_sec'] / HN_COUNT * 3600 * 24 * 30.5) + 'B' total['time_to_sync_pc_human'] = str( math.ceil(total['time_to_sync'] / SLOT_TIME_SEC * 100)) + '%' if verbose: print(json.dumps(total)) return total
def get_disks(self): disks = defaultdict(dict) try: for part in psutil.disk_partitions(all=False): if os.name == "nt": if "cdrom" in part.opts or part.fstype == "": continue usage = psutil.disk_usage(part.mountpoint) device = part.device.split("\\", 1)[0] disks[device]["device"] = device disks[device]["total"] = bytes2human(usage.total) disks[device]["used"] = bytes2human(usage.used) disks[device]["free"] = bytes2human(usage.free) disks[device]["percent"] = int(usage.percent) disks[device]["fstype"] = part.fstype except Exception: disks = {"error": "error getting disk info"} return disks
def get_stats(): global g global DOWN_COUNT g = Graph() g.add_vertices(len(NODES)) for link in LINKS: from_juror = NODES[link[0]]['juror'] link_color = PALETTE[0] if from_juror else "#aaaaaa" g.add_edge(link[0], link[1], link_color=link_color) # print(g.degree_distribution()) giant = max(g.components().sizes()) running = HN_COUNT - DOWN_COUNT stats = {} stats['down_count'] = DOWN_COUNT stats['size_round'] = int(len(LINKS) * NETWORK_BYTES_PER_ROUND_PER_LINK) stats['size_round_human'] = bytes2human(stats['size_round']) + 'Byte' stats['byte_per_sec'] = int(stats['size_round'] / ROUND_TIME_SEC) stats['size_round_per_node'] = int(stats['size_round'] / HN_COUNT) stats['byte_per_sec_per_node'] = int(stats['byte_per_sec'] / HN_COUNT) stats['bps_per_node'] = int(stats['byte_per_sec'] / HN_COUNT * 8) stats['bps_node_human'] = bytes2human( stats['byte_per_sec'] / HN_COUNT * 8) + 'bps' stats['monthly_node_traffic_human'] = bytes2human( stats['byte_per_sec'] / HN_COUNT * 3600 * 24 * 30.5) + 'B' path_length_hist = g.path_length_hist(directed=False) stats['mean_path'] = path_length_hist.mean stats['time_to_sync'] = math.ceil(stats['mean_path']) * WAIT stats['time_to_sync_pc_human'] = str( math.ceil(stats['time_to_sync'] / SLOT_TIME_SEC * 100)) + '%' stats['disconnected'] = running - giant stats['avg_degree'] = mean([node['in'] + node['out'] for node in NODES]) stats['avg_degree_non_juror'] = mean( [node['in'] + node['out'] for node in NODES if not node['juror']]) # These one are important, that's what determine the forging state possibility stats['avg_degree_juror'] = mean( [node['in'] + node['out'] for node in NODES if node['juror']]) stats['min_degree_juror'] = min( [node['in'] + node['out'] for node in NODES if node['juror']]) stats['max_degree_juror'] = max( [node['in'] + node['out'] for node in NODES if node['juror']]) return stats
def get_rusage_dict(children=False): # TODO: # number of child processes? # page size # difference between a page out and a major fault? # NOTE: values commented with an * are untracked on Linux if not resource: return {} who = resource.RUSAGE_SELF if children: who = resource.RUSAGE_CHILDREN rr = resource.getrusage(who) if sys.platform == 'darwin': rss_bytes = rr.ru_maxrss # darwin breaks posix else: rss_bytes = rr.ru_maxrss * 1024 max_rss_human = bytes2human(rss_bytes, ndigits=1) ret = { 'cpu_times': { 'user_time': rr.ru_utime, 'sys_time': rr.ru_stime }, 'memory': { 'max_rss_human': max_rss_human, 'max_rss': rr.ru_maxrss, 'shared_rss': rr.ru_ixrss, # * 'unshared_rss': rr.ru_idrss, # * 'stack_rss': rr.ru_isrss }, # * 'page_faults': { 'minor_faults': rr.ru_minflt, 'major_faults': rr.ru_majflt, 'page_outs': rr.ru_nswap }, # * 'blocking_io': { 'input_ops': rr.ru_inblock, 'output_ops': rr.ru_oublock }, 'messages': { 'sent': rr.ru_msgsnd, # * 'received': rr.ru_msgrcv }, # * 'signals': { 'received': rr.ru_nsignals }, # * 'ctx_switches': { 'voluntary': rr.ru_nvcsw, 'involuntary': rr.ru_nivcsw } } ret['rlimit'] = get_rlimit_dict() return ret
def disp_online_info(online_info): log('Online status: ' + online_info['online']) if online_info['online'] == 'Online': log('Online user: '******'username']) log('Time of login: '******'login_timestamp'])) log('Online time: ' + sec2human(online_info['online_time'])) log('Monthly online time: ' + sec2human(online_info['monthly_online_time'])) log('Online IP: ' + online_info['ip']) log('IPv4 uasge: ' + bytes2human(online_info['bytes'])) log('Account balance: ' + str(online_info['balance']))
def main(): parser = argparse.ArgumentParser(prog="pyuvssim", description="Compares two YUV I420/IYUV raw video files using the SSIM metric") parser.add_argument('base_video', metavar='video1.yuv') parser.add_argument('comparison_video', metavar='video2.yuv') parser.add_argument('-W', '--width', type=int, action='store', default=1920, nargs='?', help='video width in pixels') parser.add_argument('-H', '--height', type=int, action='store', default=1080, nargs='?', help='video width in pixels') args = parser.parse_args() vid1 = args.base_video vid2 = args.comparison_video width = args.width height = args.height frame_size = width * height frame_weight = (frame_size * 3) / 2 video_size = min(os.stat(vid1)[6], os.stat(vid2)[6]) nb_frames = video_size / frame_weight print("Videos information:") print("width: {} px".format(width)) print("height: {} px".format(height)) print("frame size: {} px^2".format(frame_size)) print("frame weight: {} ({})".format(utils.bytes2human(frame_weight), frame_weight)) print("video size: {} ({})".format(utils.bytes2human(video_size), video_size)) print("number of frames: {}\n".format(nb_frames)) f1 = open(vid1, 'rb') f2 = open(vid2, 'rb') print("Pic #, SSIM value") for n in range(nb_frames): frame_offset = (n * frame_weight) im1 = Image.new("RGB", (width, height)) im2 = Image.new("RGB", (width, height)) pix1 = im1.load() pix2 = im2.load() # I420/IYUV: NxN Y plane, then (N/2)x(N/2) U and V planes for y in range(height): for x in range(width): pos_y = frame_offset + (y * width + x) pos_u = frame_offset + (y/2 * width/2 + x/2 + frame_size) pos_v = frame_offset + (y/2 * width/2 + x/2 + frame_size + frame_size/4) f1.seek(pos_y, 0) y1 = ord(f1.read(1)) f1.seek(pos_u, 0) u1 = ord(f1.read(1)) f1.seek(pos_v, 0) v1 = ord(f1.read(1)) f2.seek(pos_y, 0) y2 = ord(f2.read(1)) f2.seek(pos_u, 0) u2 = ord(f2.read(1)) f2.seek(pos_v, 0) v2 = ord(f2.read(1)) pix1[x, y] = utils.yuv2rgb(y1, u1, v1) pix2[x, y] = utils.yuv2rgb(y2, u2, v2) print("{}, {}".format(n, ssim.compute_ssim(im1, im2))) f1.close() f2.close()
def convert_worker(target_format, message, url, config, bot): """Generic process spawned every time user sends a link or a file""" input_filename = "".join([config["temp_path"], utils.random_string()]) output_filename = "".join( [config["temp_path"], utils.random_string(), ".", target_format]) # Tell user that we are working status_message = bot.reply_to(message, text.starting, parse_mode="HTML") def update_status_message(new_text): bot.edit_message_text(chat_id=status_message.chat.id, message_id=status_message.message_id, text=new_text, parse_mode="HTML") # Try to download URL try: r = requests.get(url, stream=True) except: update_status_message(text.error.downloading) return # Check file size if int(r.headers.get("Content-Length", "0")) >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) return # Download the file update_status_message(text.downloading) chunk_size = 4096 raw_input_size = 0 try: with open(input_filename, "wb") as f: for chunk in r.iter_content(chunk_size=chunk_size): f.write(chunk) raw_input_size += chunk_size # Download files without Content-Length, but apply standard limit to them if raw_input_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) utils.rm(input_filename) return except: update_status_message(text.error.downloading) bot.reply_to(message, f"HTTP {r.status_code}") return # Start ffmpeg ffmpeg_process = None if target_format == "mp4": ffmpeg_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-threads", str(config["ffmpeg_threads"]), "-i", input_filename, "-map", "V:0?", # select video stream "-map", "0:a?", # ignore audio if doesn't exist "-c:v", "libx264", # specify video encoder "-max_muxing_queue_size", "9999", # https://trac.ffmpeg.org/ticket/6375 "-movflags", "+faststart", # optimize for streaming "-preset", "veryslow", # https://trac.ffmpeg.org/wiki/Encode/H.264#a2.Chooseapresetandtune "-timelimit", "900", # prevent DoS (exit after 15 min) "-vf", "pad=ceil(iw/2)*2:ceil(ih/2)*2", # https://stackoverflow.com/questions/20847674/ffmpeg-libx264-height-not-divisible-by-2#20848224 output_filename, ]) elif target_format == "png": ffmpeg_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-threads", str(config["ffmpeg_threads"]), "-thread_type", "slice", "-i", input_filename, "-timelimit", "60", # prevent DoS (exit after 15 min) output_filename, ]) # Update progress while ffmpeg is alive old_progress = "" while ffmpeg_process.poll() == None: try: raw_output_size = utils.filesize(output_filename) except FileNotFoundError: raw_output_size = 0 if raw_output_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) ffmpeg_process.kill() utils.rm(output_filename) input_size = utils.bytes2human(raw_input_size) output_size = utils.bytes2human(raw_output_size) progress = f"{output_size} / {input_size}" # Update progress only if it changed if progress != old_progress: update_status_message(text.converting.format(progress)) old_progress = progress time.sleep(2) # Exit in case of error with ffmpeg if ffmpeg_process.returncode != 0: update_status_message(text.error.converting) # Clean up and close pipe explicitly utils.rm(output_filename) return # Check output file size output_size = utils.filesize(output_filename) if output_size >= MAXIMUM_FILESIZE_ALLOWED: update_status_message(text.error.huge_file) # Clean up and close pipe explicitly utils.rm(output_filename) return # Default params for sending operation data = { "chat_id": message.chat.id, "reply_to_message_id": message.message_id } if target_format == "mp4": data.update({"supports_streaming": True}) # 1. Get video duration in seconds video_duration = subprocess.run( [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", output_filename, ], stdout=subprocess.PIPE, ).stdout.decode("utf-8").strip() video_duration = round(float(video_duration)) data.update({"duration": video_duration}) # 2. Get video height and width video_props = subprocess.run( [ "ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height", "-of", "csv=s=x:p=0", output_filename, ], stdout=subprocess.PIPE, ).stdout.decode("utf-8").strip() video_width, video_height = video_props.split("x") data.update({"width": video_width, "height": video_height}) # 3. Take one frame from the middle of the video update_status_message(text.generating_thumbnail) thumbnail = "".join( [config["temp_path"], utils.random_string(), ".jpg"]) generate_thumbnail_process = subprocess.Popen([ "ffmpeg", "-v", "error", "-i", output_filename, "-vcodec", "mjpeg", "-vframes", "1", "-an", "-f", "rawvideo", "-ss", str(int(video_duration / 2)), # keep the limit of 90px height/width (Telegram API) while preserving the aspect ratio "-vf", "scale='if(gt(iw,ih),90,trunc(oh*a/2)*2)':'if(gt(iw,ih),trunc(ow/a/2)*2,90)'", thumbnail, ]) # While process is alive (i.e. is working) while generate_thumbnail_process.poll() == None: time.sleep(1) # Exit in case of error with ffmpeg if generate_thumbnail_process.returncode != 0: update_status_message(text.error.generating_thumbnail) return update_status_message(text.uploading) requests.post( "https://api.telegram.org/bot{}/sendVideo".format( config["telegram_token"]), data=data, files=[ ("video", (utils.random_string() + ".mp4", open(output_filename, "rb"), "video/mp4")), ("thumb", (utils.random_string() + ".jpg", open(thumbnail, "rb"), "image/jpeg")), ], ) utils.rm(input_filename) utils.rm(output_filename) utils.rm(thumbnail) elif target_format == "png": # Upload to Telegram update_status_message(text.uploading) requests.post( "https://api.telegram.org/bot{}/sendPhoto".format( config["telegram_token"]), data=data, files=[("photo", (utils.random_string() + ".png", open(output_filename, "rb"), "image/png"))], ) requests.post( "https://api.telegram.org/bot{}/sendDocument".format( config["telegram_token"]), data=data, files=[("document", (utils.random_string() + ".png", open(output_filename, "rb"), "image/png"))], ) utils.rm(input_filename) utils.rm(output_filename) bot.delete_message(message.chat.id, status_message.message_id)
sizes[len(e)].append(k) sorted_sizes = list(sizes.keys()) sorted_sizes.sort(reverse=True) total_ids = 0 total_images = 0 for k in sorted_sizes: print('{:4} ids with {:4} images'.format(len(sizes[k]), k)) total_ids += len(sizes[k]) total_images += len(sizes[k]) * k print('Total ids: {}'.format(total_ids)) print('Total images: {}'.format(total_images)) print(utils.bytes2human(expected_size), 'will be needed to download all the images') a = input('> ') ids_to_download = sizes[a] download_imgs = True with FTP(server) as ftp: print('Login into {}'.format(server)) ftp.login() base_dir = ftp.pwd() for idx, source_dir in enumerate(source_dirs): ftp.cwd(base_dir) ftp.cwd(source_dir)
def main(): parser = argparse.ArgumentParser( prog="pyuvssim", description= "Compares two YUV I420/IYUV raw video files using the SSIM metric") parser.add_argument('base_video', metavar='video1.yuv') parser.add_argument('comparison_video', metavar='video2.yuv') parser.add_argument('-W', '--width', type=int, action='store', default=1920, nargs='?', help='video width in pixels') parser.add_argument('-H', '--height', type=int, action='store', default=1080, nargs='?', help='video width in pixels') args = parser.parse_args() vid1 = args.base_video vid2 = args.comparison_video width = args.width height = args.height frame_size = width * height frame_weight = (frame_size * 3) / 2 video_size = min(os.stat(vid1)[6], os.stat(vid2)[6]) nb_frames = video_size / frame_weight print("Videos information:") print("width: {} px".format(width)) print("height: {} px".format(height)) print("frame size: {} px^2".format(frame_size)) print("frame weight: {} ({})".format(utils.bytes2human(frame_weight), frame_weight)) print("video size: {} ({})".format(utils.bytes2human(video_size), video_size)) print("number of frames: {}\n".format(nb_frames)) f1 = open(vid1, 'rb') f2 = open(vid2, 'rb') print("Pic #, SSIM value") for n in range(nb_frames): frame_offset = (n * frame_weight) im1 = Image.new("RGB", (width, height)) im2 = Image.new("RGB", (width, height)) pix1 = im1.load() pix2 = im2.load() # I420/IYUV: NxN Y plane, then (N/2)x(N/2) U and V planes for y in range(height): for x in range(width): pos_y = frame_offset + (y * width + x) pos_u = frame_offset + (y / 2 * width / 2 + x / 2 + frame_size) pos_v = frame_offset + (y / 2 * width / 2 + x / 2 + frame_size + frame_size / 4) f1.seek(pos_y, 0) y1 = ord(f1.read(1)) f1.seek(pos_u, 0) u1 = ord(f1.read(1)) f1.seek(pos_v, 0) v1 = ord(f1.read(1)) f2.seek(pos_y, 0) y2 = ord(f2.read(1)) f2.seek(pos_u, 0) u2 = ord(f2.read(1)) f2.seek(pos_v, 0) v2 = ord(f2.read(1)) pix1[x, y] = utils.yuv2rgb(y1, u1, v1) pix2[x, y] = utils.yuv2rgb(y2, u2, v2) print("{}, {}".format(n, ssim.compute_ssim(im1, im2))) f1.close() f2.close()