def make_app_container(args): # We add some empty values here so it's easier to modify this template later app_container = { 'name': 'app', 'image': args.image, 'imagePullPolicy': args.image_pull_policy, 'resources': { 'limits': {}, 'requests': {}, }, 'volumeMounts': [], 'env': [], 'envFrom': [], } # Resource limits if args.cpu_limit: app_container['resources']['limits']['cpu'] = args.cpu_limit if args.cpu_request: app_container['resources']['requests']['cpu'] = args.cpu_request if args.memory_limit != 'none': app_container['resources']['limits']['memory'] = \ humanfriendly.parse_size(args.memory_limit, binary=True) if args.memory_request != 'none': app_container['resources']['requests']['memory'] = \ humanfriendly.parse_size(args.memory_request, binary=True) return app_container
def check_raw_config(raw_config): C = int(humanfriendly.parse_size(raw_config['C'], binary=True)) L = int(humanfriendly.parse_size(raw_config['L'], binary=True)) K = int(raw_config['K']) N = int(raw_config['N']) return C == L * K * N
def container_ram_swap() -> Tuple[int, int]: """Return ram and swap available to a container. See https://docs.docker.com/config/containers/resource_constraints/#--memory-swap-details for mere details. """ ram = settings.DOCKER_PARAMETERS.get("mem_limit", -1) if ram == "-1": ram = -1 if ram != -1: ram = humanfriendly.parse_size(ram) swap = settings.DOCKER_PARAMETERS.get("memswap_limit", 0) if swap == "-1": swap = -1 if swap != -1: swap = humanfriendly.parse_size(swap) if ram == -1 and swap == -1: return -1, -1 if ram == swap: return ram, 0 if ram >= 0 and swap == 0: return ram, ram return ram, swap - ram
def update_project_quota(apiversion, project): headers = { 'X-Auth-Token': request.headers['X-Auth-Token'], 'X-Subject-Token': request.headers['X-Auth-Token'] } ks_url = config['swift']['keystone_url'] + '/auth/tokens' r = requests.get(ks_url, headers=headers) if not r.status_code == 200: abort(r.status_code) ks_token = r.json() user = ks_token['token']['user']['name'] if user not in config['admin']: abort(403) data = request.get_json() quota = db_quota.find_one({'id': project}) if quota: db_quota.update({'id': project}, { '$set': { 'quota': humanfriendly.parse_size(data['quota'], binary=True) } }) else: db_quota.insert({ 'id': project, 'quota': humanfriendly.parse_size(data['quota'], binary=True) }) __set_quotas(project) return jsonify({'project': project, 'quota': data['quota']})
def calc_score(metrics, baseline): if not isinstance(baseline, numbers.Number): baseline = humanfriendly.parse_size(baseline) return mean([ m if isinstance(m, numbers.Number) else humanfriendly.parse_size(m) for m in metrics ]) / baseline
def info(self) -> SwarmStats: info = self.cli.info() pl_status = SwarmStats() pl_status.container_count = info["Containers"] pl_status.image_count = info["Images"] pl_status.memory_total = info["MemTotal"] pl_status.cores_total = info["NCPU"] # DriverStatus is a list... idx = 1 assert 'Strategy' in info["DriverStatus"][idx][0] pl_status.placement_strategy = info["DriverStatus"][idx][1] idx = 2 assert 'Filters' in info["DriverStatus"][idx][0] pl_status.active_filters = [ x.strip() for x in info["DriverStatus"][idx][1].split(", ") ] idx = 3 assert 'Nodes' in info["DriverStatus"][idx][0] node_count = int(info["DriverStatus"][idx][1]) idx = 4 for node in range(node_count): idx2 = 0 ns = SwarmNodeStats(info["DriverStatus"][idx + node][0]) ns.docker_endpoint = info["DriverStatus"][idx + node][1] idx2 += 1 if 'Status' in info["DriverStatus"][idx + node + idx2][0]: # new docker version ns.status = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 ns.container_count = int(info["DriverStatus"][idx + node + idx2][1]) idx2 += 1 ns.cores_reserved = int( info["DriverStatus"][idx + node + idx2][1].split(' / ')[0]) ns.cores_total = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[1]) idx2 += 1 ns.memory_reserved = info["DriverStatus"][idx + node + idx2][1].split(' / ')[0] ns.memory_total = info["DriverStatus"][idx + node + idx2][1].split(' / ')[1] idx2 += 1 ns.labels = info["DriverStatus"][idx + node + idx2][1:] idx2 += 1 ns.error = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 ns.last_update = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 ns.server_version = info["DriverStatus"][idx + node + idx2][1] ns.memory_reserved = humanfriendly.parse_size(ns.memory_reserved) ns.memory_total = humanfriendly.parse_size(ns.memory_total) pl_status.nodes.append(ns) idx += idx2 pl_status.timestamp = time.time() return pl_status
def get_stats(self): self.logger.info("Getting stats") now = datetime.now(timezone.utc).astimezone().isoformat() influx_payload = [] with Session() as s: try: cookies = {} for key, morsel in self.cookies.items(): cookies[key] = morsel.value response = s.get(self.base_url + "/userdetails.php", cookies=cookies) soup = BeautifulSoup(response.content, "lxml") usertable = soup.find('div', class_='p sh').find('table') for tablerow in usertable.find_all('tr'): key = tablerow.find("th").text value = tablerow.find("td").text if key == "Opplastet": upload = tablerow.find("td").text upload = upload[upload.find("[") + 1:upload.find("]")].replace(',', '') upload = humanfriendly.parse_size(upload) elif key == "Nedlastet": download = tablerow.find("td").text download = download[download.find("[") + 1:download.find("]")].replace( ',', '') download = humanfriendly.parse_size(download) elif key == "Poeng": bonus_points = tablerow.find("td").text bonus_points = bonus_points.split('.')[0] bonus_points = int(bonus_points) elif key == "Ratio": ratio = tablerow.find("td").find("span").text ratio = float(ratio) influx_payload.append( { "measurement": "stats", "tags": { "tracker": self.name, }, "time": now, "fields": { "ratio": ratio, "download": download, "upload": upload, "bonus_points": bonus_points }, }, ) self.dbmanager.write_points(influx_payload) self.logger.info(f"Updated {self.name}") except: self.logger.error(f"Failed to update {self.name}")
def get_dir_size(self): """Will go one level deeper and sum the sizes of all objects. If the objects are directories, you'll have to do the iteration yourself!""" total = humanfriendly.parse_size(self.size) for f in self.list_dir(): total += humanfriendly.parse_size(f.size) print(total) t = humanfriendly.parse_size(str(total)) return humanfriendly.format_size(t)
def memory_expand( tree, subnode, memory_start = 0xbeef, prop_name = 'memory', verbose = 0 ): # /* # * 1:1 map, it should match the memory regions # * specified under access below. # * # * It is in the form: # * memory = <address size address size ...> # */ try: mem = [subnode.props( prop_name )[0][0]] mem_list = [] for m in mem: try: start = str(m['start']) except: start = str(int(memory_start)) try: size = str(m['size']) except: size = str(int(0xbeef)) if 'flags' in m.keys(): flags = str(m['flags']) flag_node = LopperNode(-1, subnode.abs_path+'/flags') tree.add(flag_node) flag_reference_name = flag_node.abs_path+"/"+str(flags) flag_reference_node = LopperNode(-1, flag_reference_name ) tree.add(flag_reference_node) flags_cells = LopperProp(name='#flags-cells',value=0x1) flags_prop = LopperProp(name='flags',value=0x0) flags_names = LopperProp('flags-names',value = str(flags)) subnode + flags_cells subnode + flags_prop subnode + flags_names #print( "memory expand: start/size as read: %s/%s" % (start,size)) start = humanfriendly.parse_size( start, True ) size = humanfriendly.parse_size( size, True ) #print( "memory expand: start/size as converted: %s/%s" % (start,size)) mem_list.append(int(start)) mem_list.append(int(size)) except Exception as e: mem_list = [0xdead, 0xffff ] if verbose: # dump the memory as hex print( '[DBG] memory: [{}]'.format(', '.join(hex(x) for x in mem_list)) ) property_set( prop_name, mem_list, subnode )
def save_mp3s(id_list, name='unknown', start_enum=1, size_threshold='100MB', overwrite=False): # First result has enum value of 1, instead of 0 enum = start_enum print("Query: {}".format(name)) pathlib.Path('../_data/mp3s/{}'.format(name)).mkdir(parents=True, exist_ok=True) for file_id in id_list: file_path = "../_data/mp3s/{0}/{2:04}_{0}_{1}.mp3".format( name, file_id, enum) print("======== Result no: {} | File id: {} ========".format( enum, file_id)) print("Saving to {}".format(file_path)) if overwrite: mode = "wb+" else: mode = "xb" try: with open(file_path, mode) as handle: res = requests.get( 'https://freesound.org/apiv2/sounds/{}/'.format(file_id), params={ 'token': token, }) json_res = json.loads(res.content.decode('utf8')) filesize = json_res['filesize'] print("File size is {}".format(format_size(filesize))) if filesize > parse_size(size_threshold): raise Exception("size_threshold") url = json_res['previews']['preview-hq-mp3'] response = requests.get(url, stream=True) for data in tqdm(response.iter_content()): handle.write(data) except FileExistsError as e: print("File already exists, skipping.") except Exception as e: if str(e) != "size_threshold": raise else: print("File size greater than {}, skipping.".format( format_size(parse_size(size_threshold)))) os.remove(file_path) enum += 1 return
def __init__(self, max_size: str, usual_object_size: str, margin: str, index: Index, storage: Storage): self.max_size = humanfriendly.parse_size(max_size, binary=True) self.usual_object_size = humanfriendly.parse_size(usual_object_size, binary=True) self.margin = humanfriendly.parse_size(margin, binary=True) self._index = index self._storage = storage
def parse_log_line(self, line) -> None: groups = self.regex.match(line).groups() self.date_time = datetime.datetime.strptime(groups[0], '%Y-%m-%d %H:%M:%S.%f') self.bytes_copied = humanfriendly.parse_size(groups[1].replace(',', '.')) self.bytes_to_copy = humanfriendly.parse_size(groups[2].replace(',', '.')) self.items_copied = int(groups[3]) self.items_to_copy = int(groups[4]) self.bytes_per_second = humanfriendly.parse_size("0" if "-" in groups[5] else groups[5].replace(',', '.')) self.items_per_second = float("0" if "-" in groups[6] else groups[6].replace(',', '.')) self.last_path_seen = groups[7]
class Config: web_core_api_url = os.getenv("LEANDA_WEB_CORE_API_URL") web_blob_api_url = os.getenv("LEANDA_WEB_BLOB_API_URL") web_socket_url = os.getenv("LEANDA_WEB_SOCKET_URL") identity_server_url = os.getenv("LEANDA_IDENTITY_SERVER_URL") file_upload_limit = os.getenv("LEANDA_FILE_UPLOAD_LIMIT") file_upload_limit_int = humanfriendly.parse_size( os.getenv("LEANDA_FILE_UPLOAD_LIMIT") or '50MB', binary=True) file_download_limit = os.getenv("LEANDA_FILE_DOWNLOAD_LIMIT") file_download_limit_int = humanfriendly.parse_size( os.getenv("LEANDA_FILE_DOWNLOAD_LIMIT") or '50MB', binary=True)
def chunksize(self): if not hasattr(self, '_chunksize'): try: transferchunk = self.annex.getconfig('transferchunk') self._chunksize = humanfriendly.parse_size(transferchunk) self.annex.debug("Using chunksize: {}".format(transferchunk)) except humanfriendly.InvalidSize: self.annex.debug( "No valid chunksize specified. Using default value: {}". format(self.DEFAULT_CHUNKSIZE)) self._chunksize = humanfriendly.parse_size( self.DEFAULT_CHUNKSIZE) return self._chunksize
def is_expired(status, since, size=0): now = datetime.datetime.now() min_bps = int(humanfriendly.parse_size("4MiB") / 8) if status == Tasks.building: return since < now - datetime.timedelta(hours=12) if status == Tasks.wiping_sdcard: return since < now - datetime.timedelta(minutes=30) if status in (Tasks.uploading, Tasks.downloading, Tasks.writing): size = humanfriendly.parse_size(size) return since < now - datetime.timedelta(seconds=int(size / min_bps))
def print_humanfriendly(fs_stats: Dict) -> None: print("Top PyPI Disk Users:") total_pypi_size = fs_stats["total_packages_size"] top_total_bytes = 0 for pkg_name, pkg_data in fs_stats["top_packages"].items(): top_total_bytes += pkg_data["size"] hfs = humanfriendly.format_size(humanfriendly.parse_size(str(pkg_data["size"]))) print(f"{pkg_name}: {hfs}") pct = int((top_total_bytes / total_pypi_size) * 100) ttb = humanfriendly.format_size(humanfriendly.parse_size(str(top_total_bytes))) print(f"\nTop Packages consume {ttb}\n- This is {pct}% of PyPI")
def memory_expand(tree, subnode, memory_start=0xbeef, prop_name='memory', verbose=0): # /* # * 1:1 map, it should match the memory regions # * specified under access below. # * # * It is in the form: # * memory = <address size address size ...> # */ try: mem = [] prop = subnode[prop_name] for i in range(0, len(prop)): mem.append(prop[i]) mem_list = [] for m in mem: try: start = str(m['start']) except: start = str(int(memory_start)) try: size = str(m['size']) except: size = str(int(0xbeef)) if 'flags' in m.keys(): flags = str(m['flags']) flags_names = LopperProp(prop_name + '-flags-names', value=str(flags)) subnode + flags_names #print( "memory expand: start/size as read: %s/%s" % (start,size)) start = humanfriendly.parse_size(start, True) size = humanfriendly.parse_size(size, True) #print( "memory expand: start/size as converted: %s/%s" % (start,size)) mem_list.append(int(start)) mem_list.append(int(size)) except Exception as e: mem_list = [0xdead, 0xffff] if verbose: # dump the memory as hex print('[DBG] memory: [{}]'.format(', '.join(hex(x) for x in mem_list))) property_set(prop_name, mem_list, subnode)
def list_dups(filepaths, file_suffix, min_size, hash_type, block_size, format, recursive, debug): logger.debug(f'hash-type: {hash_type}') hash_constructor = available_hashes[hash_type] min_sz = parse_size(min_size) blk_sz = parse_size(block_size, binary=True) dups = defaultdict(list) hardlinked = defaultdict(list) logger.debug(f'min-size: {min_sz}') logger.debug(f'block-size: {blk_sz}') glob_prefix = '**/' if recursive else '' glob_pattern = f'{glob_prefix}{file_suffix}' p1 = chain(*(Path(fp).expanduser().glob(glob_pattern) for fp in filepaths)) for f in tqdm(p1): if f.is_file() and not f.is_symlink(): s = f.stat() if (s.st_size > min_sz): #logger.debug(f'processing: {f}') if s.st_nlink > 1: logger.debug(f'hardlinks [{s.st_nlink}]: file: {f}') # Same Inodes: hardlinked[(s.st_ino, s.st_dev)].append(f) # FIXME: adaptive block_size?? hash_res = hash_file(f, block_size=blk_sz, hex_result=True, hash_constructor=hash_constructor) #click.echo(f'hash[{hash_type}]: {hash_res}') dups[hash_res].append(f) records = [[(filehash, path.resolve().as_posix(), path.stat().st_size, format_size(path.stat().st_size), str(path.stat().st_ino) if (path.stat().st_ino, path.stat().st_dev) in hardlinked else None, str(path.stat().st_dev) if (path.stat().st_ino, path.stat().st_dev) in hardlinked else None) for path in paths] for (filehash, paths) in dups.items() if len(paths) > 1] if len(records) > 0: cols = ['hash', 'file_path', 'size', 'human_size', 'inode', 'device'] df = (pd.concat(pd.DataFrame.from_records(r, columns=cols) for r in records) .sort_values(by=['size', 'inode', 'device'], ascending=False)) if debug: from IPython import embed embed() print(tabulate(df, headers=df.columns, tablefmt=format, showindex=False))
def info(self) -> ClusterStats: # pylint: disable=too-many-locals """Retrieve Kubernetes cluster statistics.""" pl_status = ClusterStats() node_list = pykube.Node.objects( self.api).filter(namespace=pykube.all).iterator() node_dict = {} # Get basic information from nodes for node in node_list: nss = NodeStats(node.name) nss.cores_total = float(node.obj['status']['allocatable']['cpu'] [1]) ## Bug found on GKE nss.memory_total = humanfriendly.parse_size( node.obj['status']['allocatable']['memory']) nss.labels = node.obj['metadata']['labels'] nss.status = 'online' node_dict[str(socket.gethostbyname(node.name))] = nss # Get information from all running pods, then accumulate to nodes pod_list = pykube.Pod.objects( self.api).filter(namespace=pykube.all).iterator() for pod in pod_list: try: host_ip = pod.obj['status']['hostIP'] except KeyError: continue nss = node_dict[host_ip] nss.container_count += 1 spec_cont = pod.obj['spec']['containers'][0] if 'resources' in spec_cont: if 'requests' in spec_cont['resources']: if 'memory' in spec_cont['resources']['requests']: memory = spec_cont['resources']['requests']['memory'] nss.memory_reserved = nss.memory_reserved + humanfriendly.parse_size( memory) if 'cpu' in spec_cont['resources']['requests']: cpu = spec_cont['resources']['requests']['cpu'] # ex: cpu could be 100m or 0.1 cpu_splitted = cpu.split('m') if len(cpu_splitted) > 1: cpu_float = int(cpu_splitted[0]) / 1000 else: cpu_float = int(cpu_splitted[0]) nss.cores_reserved = round( nss.cores_reserved + cpu_float, 3) for node_ip in node_dict: pl_status.nodes.append(node_dict[node_ip]) return pl_status
def info(self) -> SwarmStats: """Retrieve Swarm statistics. The Docker API returns a mess difficult to parse.""" info = self.cli.info() pl_status = SwarmStats() pl_status.container_count = info["Containers"] pl_status.image_count = info["Images"] pl_status.memory_total = info["MemTotal"] pl_status.cores_total = info["NCPU"] # DriverStatus is a list... idx = 1 assert 'Strategy' in info["DriverStatus"][idx][0] pl_status.placement_strategy = info["DriverStatus"][idx][1] idx = 2 assert 'Filters' in info["DriverStatus"][idx][0] pl_status.active_filters = [x.strip() for x in info["DriverStatus"][idx][1].split(", ")] idx = 3 assert 'Nodes' in info["DriverStatus"][idx][0] node_count = int(info["DriverStatus"][idx][1]) idx = 4 for node in range(node_count): idx2 = 0 node_stats = SwarmNodeStats(info["DriverStatus"][idx + node][0]) node_stats.docker_endpoint = info["DriverStatus"][idx + node][1] idx2 += 1 if 'Status' in info["DriverStatus"][idx + node + idx2][0]: # new docker version node_stats.status = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 node_stats.container_count = int(info["DriverStatus"][idx + node + idx2][1]) idx2 += 1 node_stats.cores_reserved = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[0]) node_stats.cores_total = int(info["DriverStatus"][idx + node + idx2][1].split(' / ')[1]) idx2 += 1 node_stats.memory_reserved = info["DriverStatus"][idx + node + idx2][1].split(' / ')[0] node_stats.memory_total = info["DriverStatus"][idx + node + idx2][1].split(' / ')[1] idx2 += 1 node_stats.labels = info["DriverStatus"][idx + node + idx2][1:] idx2 += 1 node_stats.error = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 node_stats.last_update = info["DriverStatus"][idx + node + idx2][1] idx2 += 1 node_stats.server_version = info["DriverStatus"][idx + node + idx2][1] node_stats.memory_reserved = humanfriendly.parse_size(node_stats.memory_reserved) node_stats.memory_total = humanfriendly.parse_size(node_stats.memory_total) pl_status.nodes.append(node_stats) idx += idx2 pl_status.timestamp = time.time() return pl_status
def constructResources(resources, config): # add CPU cpu_resource = resources.add() cpu_resource.name = "cpus" cpu_resource.type = messages_pb2.Value.SCALAR cpu_list = psutil.cpu_percent(interval=1, percpu=True) cpu_value = 0 for cpu in cpu_list: cpu_value += (100 - cpu) / 100 if 'maxCPUs' in config: if config['maxCPUs'] < cpu_value: cpu_value = config['maxCPUs'] cpu_resource.scalar.value = cpu_value # add MEMORY mem_resource = resources.add() mem_resource.name = "mem" mem_resource.type = messages_pb2.Value.SCALAR mem = psutil.virtual_memory().available if 'maxMem' in config: maxmem = humanfriendly.parse_size(config['maxMem']) if maxmem < mem: mem = maxmem mem_resource.scalar.value = mem # add DISK disk_resource = resources.add() disk_resource.name = "disk" disk_resource.type = messages_pb2.Value.SCALAR disk = os.statvfs('./').f_frsize * os.statvfs('./').f_bavail if 'maxDisk' in config: maxdisk = humanfriendly.parse_size(config['maxDisk']) if maxdisk < disk: disk = maxdisk disk_resource.scalar.value = disk # add devices if config and 'devices' in config: for device in config['devices']: device_resource = resources.add() device_resource.name = device['name'] device_resource.type = messages_pb2.Value.DEVICE device_resource.text.value = device['id'] if 'shared' in device: device_resource.shared = device['shared'] else: device_resource.shared = False
def info(self) -> ClusterStats: """Retrieve Swarm statistics. The Docker API returns a mess difficult to parse.""" info = self.cli.info() pl_status = ClusterStats() # SystemStatus is a list... idx = 0 # Role, skip idx += 1 assert 'Strategy' in info["SystemStatus"][idx][0] pl_status.placement_strategy = info["SystemStatus"][idx][1] idx += 1 assert 'Filters' in info["SystemStatus"][idx][0] pl_status.active_filters = [x.strip() for x in info["SystemStatus"][idx][1].split(", ")] idx += 1 assert 'Nodes' in info["SystemStatus"][idx][0] node_count = int(info["SystemStatus"][idx][1]) idx += 1 # At index 4 the nodes begin for node in range(node_count): idx2 = 0 node_stats = NodeStats(info["SystemStatus"][idx + node][0].strip()) node_stats.docker_endpoint = info["SystemStatus"][idx + node][1] idx2 += 1 # ID, skip idx2 += 1 # Status if info["SystemStatus"][idx + node + idx2][1] == 'Healthy': node_stats.status = 'online' else: node_stats.status = 'offline' idx2 += 1 # Containers node_stats.container_count = int(info["SystemStatus"][idx + node + idx2][1].split(' ')[0]) idx2 += 1 # CPUs node_stats.cores_reserved = int(info["SystemStatus"][idx + node + idx2][1].split(' / ')[0]) node_stats.cores_total = int(info["SystemStatus"][idx + node + idx2][1].split(' / ')[1]) idx2 += 1 # Memory node_stats.memory_reserved = info["SystemStatus"][idx + node + idx2][1].split(' / ')[0] node_stats.memory_total = info["SystemStatus"][idx + node + idx2][1].split(' / ')[1] idx2 += 1 # Labels node_stats.labels = info["SystemStatus"][idx + node + idx2][1].split(', ') idx2 += 1 # Last update node_stats.last_update = info["SystemStatus"][idx + node + idx2][1] idx2 += 1 # Docker version node_stats.server_version = info["SystemStatus"][idx + node + idx2][1] node_stats.memory_reserved = humanfriendly.parse_size(node_stats.memory_reserved) node_stats.memory_total = humanfriendly.parse_size(node_stats.memory_total) pl_status.nodes.append(node_stats) idx += idx2 pl_status.timestamp = time.time() return pl_status
def __init__(self, C, L, K, N, BIT_SIZE, input_label): self.C = int(humanfriendly.parse_size(C, binary=True)) self.L = int(humanfriendly.parse_size(L, binary=True)) self.K = int(K) self.N = int(N) if self.C != self.L * self.K * self.N: raise RuntimeError('Invalid matching L, K, N, C parameters') self.input_label = input_label self.BIT_SIZE = BIT_SIZE self.BYTE_SELECT = int(math.log(self.L, 2)) self.CACHE_INDEX = int(math.log(self.N, 2)) self.CACHE_TAG = self.BIT_SIZE - self.BYTE_SELECT - self.CACHE_INDEX
def extract_nand_info(self): fmt = "mnand_read_id: CS{:d} NAND {} {} {} size({:d}) writesize({:d}) oobsize({:d}) erasesize({:d})" chips = [] _id, lines = self.terminal.wait("mnand_read_id: flash id {:x}") if _id[0] > 0: params, lines = self.terminal.wait(fmt) chips.append(params) _id, lines = self.terminal.wait("mnand_read_id: flash id {:x}") if _id[0] > 0: params, lines = self.terminal.wait(fmt) chips.append(params) size = 0 part = "" for c in chips: size += humanfriendly.parse_size(c[1]) if len(chips) > 1: l = "(x2)" else: l = "(x1)" attrs = { "size": size, "part": f"NAND {chips[0][1]} {chips[0][2]} {chips[0][3]} {l}", "read_size": chips[0][4], "write_size": chips[0][5], "oob_size": chips[0][6], "erase_size": chips[0][7], } self.nand_chips = chips return attrs
def __init__(self, input_fasta, database, num_parts = None, # How many fasta pieces should we make part_size = None, # What size in MB should a fasta piece be seqs_per_part = None, # How many sequences in one fasta piece slurm_params = None, # Additional parameters for possible SLURM jobs parts_dir = None, # If you want a special direcotry for the fasta pieces **kwargs): # Determine number of parts # self.num_parts = None # Three possible options # if num_parts: self.num_parts = num_parts if part_size: self.bytes_target = humanfriendly.parse_size(part_size) self.num_parts = int(math.ceil(input_fasta.count_bytes / self.bytes_target)) if seqs_per_part: self.num_parts = int(math.ceil(input_fasta.count / seqs_per_part)) # Default case # if self.num_parts is None: self.num_parts = kwargs.get('num_threads', min(multiprocessing.cpu_count(), 32)) # In case the user has some special slurm params # self.slurm_params = slurm_params # In case the user wants a special parts directory # self.parts_dir = parts_dir # Super # SeqSearch.__init__(self, input_fasta, database, **kwargs)
def build( self, bloomfilters: hug.types.multiple, samples: hug.types.multiple = [], config: hug.types.text = None, ): config = get_config_from_file(config) if samples: assert len(samples) == len(bloomfilters) else: samples = bloomfilters if config.get("max_build_mem_bytes"): max_memory_bytes = humanfriendly.parse_size( config["max_build_mem_bytes"]) else: max_memory_bytes = None return build( config=config, bloomfilter_filepaths=bloomfilters, samples=samples, max_memory=max_memory_bytes, )
def __init__( self, fs: Union[int, str] = 16000, frontend_conf: Optional[dict] = get_default_kwargs(Frontend), download_dir: str = None, multilayer_feature: bool = False, ): assert check_argument_types() super().__init__() if isinstance(fs, str): fs = humanfriendly.parse_size(fs) if download_dir is not None: torch.hub.set_dir(download_dir) self.multilayer_feature = multilayer_feature self.upstream, self.featurizer = self._get_upstream(frontend_conf) if getattr( self.upstream, "model", None ) is not None and self.upstream.model.__class__.__name__ in [ "Wav2Vec2Model", "HuberModel", ]: self.upstream.model.encoder.layerdrop = 0.0 self.pretrained_params = copy.deepcopy(self.upstream.state_dict()) self.output_dim = self.featurizer.output_dim
def __init__( self, name, modules=None, cpus=None, memory=None, use_temporary_directory=None, temporary_directory_path=None, hard_resources=None, soft_resources=None, inputs=None, outputs=None, parameters=None, script=None): self.name = name self.modules = modules or [] self.cpus = cpus or 1 self.memory = memory or humanfriendly.parse_size('1GB') self.use_temporary_directory = use_temporary_directory or False self.temporary_directory_path = temporary_directory_path self.hard_resources = hard_resources or [] self.soft_resources = soft_resources or [] self.inputs = inputs or collections.OrderedDict() self.outputs = outputs or {} self.parameters = parameters or {} self.script = script or ''
def _parse_field(self, field_metric, field_key, field_value): try: # sizes if (field_metric, field_key) in self._size_fields: return humanfriendly.parse_size(field_value) # time spans elif (field_metric, field_key) in self._time_span_fields: return sum( humanfriendly.parse_timespan(v) if v else 0 for v in re.findall(r'([\d,\.]+\s*\D+)', field_value)) # percentages elif (field_metric, field_key) in self._percentage_fields: return int(field_value.replace('%', '').strip()) # floats elif (field_metric, field_key) in self._float_fields: return float(decimal.Decimal(field_value)) # integers elif (field_metric, field_key) in self._integer_fields: return int(decimal.Decimal(field_value)) except ValueError: pass
def parse_row(row): details = row.select('font.detDesc')[0].text # Parse category try: typ = self.parse_category(row.select('td')[0].text) except _CategoryUnknowError as e: typ = 'other' msg = "Unknow category: '{category}'" msg = msg.format(category=e.args[0]) print(msg, file=sys.stderr) # Parse size size = re.findall(r'([0-9\.]+\s*[GMK]i?B)', details, re.IGNORECASE)[0] size = humanfriendly.parse_size(size) # Parse created try: created = self.parse_timestamp(row.select('.detDesc')[0].text) except IndexError: created = 'other' return { 'name': row.findAll('a')[2].text, 'uri': row.findAll('a')[3]['href'], 'type': typ, 'size': size, 'created': created, 'seeds': int(row.findAll('td')[-2].text), 'leechers': int(row.findAll('td')[-1].text) }
def get_memory_info(self, arc_size): with open("/proc/meminfo") as f: meminfo = { s[0]: humanfriendly.parse_size(s[1], binary=True) for s in [line.split(":", 1) for line in f.readlines()] } classes = {} classes["page_tables"] = meminfo["PageTables"] classes["swap_cache"] = meminfo["SwapCached"] classes["slab_cache"] = meminfo["Slab"] classes["cache"] = meminfo["Cached"] classes["buffers"] = meminfo["Buffers"] classes["unused"] = meminfo["MemFree"] classes["arc"] = arc_size classes["apps"] = meminfo["MemTotal"] - sum(classes.values()) extra = { "inactive": meminfo["Inactive"], "committed": meminfo["Committed_AS"], "active": meminfo["Active"], "vmalloc_used": meminfo["VmallocUsed"], "mapped": meminfo["Mapped"], } swap = { "used": meminfo["SwapTotal"] - meminfo["SwapFree"], "total": meminfo["SwapTotal"], } return { "classes": classes, "extra": extra, "swap": swap, }
def _parse_field(self, field_metric, field_key, field_value): try: # sizes if (field_metric, field_key) in self._size_fields: return humanfriendly.parse_size(field_value) # time spans elif (field_metric, field_key) in self._time_span_fields: return sum(humanfriendly.parse_timespan(v) if v else 0 for v in re.findall(r'([\d,\.]+\s*\D+)', field_value)) # percentages elif (field_metric, field_key) in self._percentage_fields: return int(field_value.replace('%', '').strip()) # floats elif (field_metric, field_key) in self._float_fields: return float(decimal.Decimal(field_value)) # integers elif (field_metric, field_key) in self._integer_fields: return int(decimal.Decimal(field_value)) except ValueError: pass
def download_member_project_data(cls, member_data, target_member_dir, max_size=MAX_SIZE_DEFAULT, id_filename=False): """ Download files to sync a local dir to match OH member project data. :param member_data: This field is data related to member in a project. :param target_member_dir: This field is the target directory where data will be downloaded. :param max_size: This field is the maximum file size. It's default value is 128m. """ logging.debug('Download member project data...') sources_shared = member_data['sources_shared'] file_data = cls._get_member_file_data(member_data, id_filename=id_filename) for basename in file_data: # This is using a trick to identify a project's own data in an API # response, without knowing the project's identifier: if the data # isn't a shared data source, it must be the project's own data. if file_data[basename]['source'] in sources_shared: continue target_filepath = os.path.join(target_member_dir, basename) download_file(download_url=file_data[basename]['download_url'], target_filepath=target_filepath, max_bytes=parse_size(max_size))
def _convert_memory_size(value): if isinstance(value, str): return humanfriendly.parse_size(value) elif callable(value): return value(_workflow_builder_stack.peek().execution_count) else: raise Exception('Unsupported')
def _get_disk_size(self): """Returns data used by Docker in bytes.""" # docker.info() returns an awful data structure... with self.allocate() as docker: for field in docker.info()['DriverStatus']: if field[0]=='Data Space Used': return parse_size(field[1]) # Value in bytes logging.error('"Data Space Used" field was not found in the data returned by Docker.')
def from_du(cls, line): self = cls() if '\t' not in line: return a, b = line.split("\t", 1) self.size = humanfriendly.parse_size(a.strip()) self.path = b.strip() return self
def runTests(self): """ Run the tests prescribed in the configuration """ sites = self.get_sites() templates_dir = os.path.join(sys.prefix, "etc/stashcache-tester/templates") # Parse the size of the test in bytes raw_testsize = humanfriendly.parse_size(get_option("testsize")) md5sum = self.createTestFile(raw_testsize, get_option("stashdir")) # Create the site specific tests env = Environment(loader=FileSystemLoader(templates_dir)) env.globals = { "config_location": self.config_location, "stash_test_location": os.path.abspath(sys.argv[0]), "pythonpath": ":".join(sys.path), "testurl": get_option("testurl"), "localpath": get_option("stashdir"), "testsize": raw_testsize, "humantestsize": humanfriendly.format_size(raw_testsize) } test_dirs = [] testingdir = get_option("testingdir") for site in sites: tmp_site = Site(site) test_dir = tmp_site.createTest(testingdir, env) test_dirs.append(test_dir) # Create the DAG from the template dag_template = env.get_template("dag.tmpl") test_dag = os.path.join(testingdir, "submit.dag") with open(test_dag, 'w') as f: f.write(dag_template.render(sites=sites, md5sum=md5sum)) reduce_template = env.get_template("test_reduce.tmpl") reduce_submit = os.path.join(testingdir, "reduce.submit") with open(reduce_submit, 'w') as f: f.write(reduce_template.render()) shutil.copyfile(os.path.join(templates_dir, "site_post.py"), os.path.join(get_option("testingdir"), "site_post.py")) os.chmod(os.path.join(get_option("testingdir"), "site_post.py"), 0755) # Start the DAG (stdout, stderr) = RunExternal("cd %s; condor_submit_dag submit.dag" % testingdir) logging.debug("output from condor_submit_dag: %s" % stdout) if stderr is not None or stderr is not "": logging.error("Error from condor_submit_dag: %s" % stderr)
def test_parse_size(self): self.assertEqual(0, humanfriendly.parse_size('0B')) self.assertEqual(42, humanfriendly.parse_size('42')) self.assertEqual(42, humanfriendly.parse_size('42B')) self.assertEqual(1024, humanfriendly.parse_size('1k')) self.assertEqual(1024, humanfriendly.parse_size('1 KB')) self.assertEqual(1024, humanfriendly.parse_size('1 kilobyte')) self.assertEqual(1024 ** 3, humanfriendly.parse_size('1 GB')) self.assertEqual(1024 ** 3 * 1.5, humanfriendly.parse_size('1.5 GB')) self.assertRaises(humanfriendly.InvalidSize, humanfriendly.parse_size, '1z') self.assertRaises(humanfriendly.InvalidSize, humanfriendly.parse_size, 'a')
def main(): args = parser.parse_args() size_kb = humanfriendly.parse_size(args.size) / 1024 read_units_per_second = humanfriendly.parse_timespan(args.time_span) # Scan operation is eventually consistent, which is half the cost (hence /2 ) # Each 4kb consumes one capacity unit. read_capacity_units_needed = size_kb / 2 / 4 time_needed = read_capacity_units_needed / read_units_per_second print 'It will take (roughly): {0}'.format(humanfriendly.format_timespan(time_needed))
def __init__(self, configFiles): # First, read in the configuration set_config_file(configFiles) self.config_location = os.path.abspath(configFiles) loglevel = get_option("loglevel", default="warning", section="logging") logdirectory = get_option("logdirectory", default="log", section="logging") self._setLogging(loglevel, logdirectory) raw_testsize = humanfriendly.parse_size(get_option("testsize")) set_option("raw_testsize", raw_testsize)
def abstract_test_chargeback_cost( rate_key, obj_type, interval, chargeback_report_data, compute_rate, soft_assert): """This is an abstract test function for testing rate costs. It's comparing the expected value that calculated by the rate to the value in the chargeback report Args: :py:type:`str` rate_key: The rate key as it appear in the CHARGEBACK_HEADER_NAMES keys. :py:type:`str` obj_type: Object being tested; only 'Project' and 'Image' are supported :py:type:`str` interval: The rate interval, (Hourly/Daily/Weekly/Monthly) :py:class:`Report` chargeback_report_data: The chargeback report data. :py:class:`ComputeRate` compute_rate: The compute rate object. :var soft_assert: soft_assert fixture. """ report_headers = CHARGEBACK_HEADER_NAMES[rate_key] found_something_to_test = False for row in chargeback_report_data.rows: if row['Chargeback Rates'].lower() != compute_rate.description.lower(): continue found_something_to_test = True fixed_rate = float(compute_rate.fields[report_headers.rate_name]['fixed_rate']) variable_rate = float(compute_rate.fields[report_headers.rate_name].get('variable_rate', 0)) # Calculate numerical metric if rate_key == 'Memory': size_, unit_ = tokenize(row[report_headers.metric_name].upper()) metric = round(parse_size(str(size_) + unit_, binary=True) / 1048576.0, 2) else: metric = parse_number(row[report_headers.metric_name]) # Calculate fixed product and cost num_hours = parse_number(row[CHARGEBACK_HEADER_NAMES['Fixed1'].metric_name]) num_intervals = num_hours / hours_count_lut[interval] fixed_cost = num_intervals * fixed_rate variable_cost = num_intervals * metric * variable_rate # Calculate expected cost expected_cost = round(variable_cost + fixed_cost, 2) found_cost = round(parse_number(row[report_headers.cost_name]), 2) match_threshold = TEST_MATCH_ACCURACY * expected_cost soft_assert( abs(found_cost - expected_cost) <= match_threshold, 'Unexpected Chargeback: {}'.format(dump_args( charge_for=obj_type, rate_key=rate_key, metric=metric, num_hours=num_hours, num_intervals=num_intervals, fixed_rate=fixed_rate, variable_rate=variable_rate, fixed_cost=fixed_cost, variable_cost=variable_cost, expected_full_cost=expected_cost, found_full_cost=found_cost )) ) assert found_something_to_test, \ 'Could not find {} with the assigned rate: {}'.format(obj_type, compute_rate.description)
def __init__(self, path, part_size, base_dir=None): # Basic # self.path = path # Directory # if base_dir is None: self.base_dir = path + '.parts/' else: self.base_dir = base_dir if not os.path.exists(self.base_dir): os.makedirs(self.base_dir) # Evaluate size # self.bytes_target = humanfriendly.parse_size(part_size) # Chose number of parts # self.num_parts = int(math.ceil(self.count_bytes / self.bytes_target)) # Make parts # self.parts = [FASTA(self.make_part_paths(i)) for i in range(self.num_parts)]
def max_size(directory): """Test directory for maximum size returns true if directory size is larger than spec property "max_size" """ try: max_size = humanfriendly.parse_size(directory.spec['max_size']) except KeyError: raise UnregulatableError('No "max_size" configured in specification!') size = directory.size logger.debug("Directory size is %s", humanfriendly.format_size(size)) logger.debug("max_size set to: %s", humanfriendly.format_size(max_size)) return size > max_size
def test_parse_size(self): self.assertEqual(42, humanfriendly.parse_size('42')) self.assertEqual(1024, humanfriendly.parse_size('1k')) self.assertEqual(1024, humanfriendly.parse_size('1 KB')) self.assertEqual(1024, humanfriendly.parse_size('1 kilobyte')) self.assertEqual(1024 ** 3, humanfriendly.parse_size('1 GB')) try: humanfriendly.parse_size('1z') self.assertTrue(False) except Exception, e: self.assertTrue(isinstance(e, humanfriendly.InvalidSize))
def clean_fs_size(): try: cron_cfg = config.frontend_config['cron_clean_file_size'] max_size = cron_cfg['clean_fs_max_size'] # 0 means disabled if max_size == '0': log.debug("disabled by config") return 0 max_size_bytes = humanfriendly.parse_size(max_size, binary=True) nb_files = file_ctrl.remove_files_size(max_size_bytes) log.info("removed %d files", nb_files) return nb_files except (IrmaDatabaseError, IrmaFileSystemError) as e: log.exception(e)
def create(ctx, name, size, description, storage_profile, iops): try: restore_session(ctx, vdc_required=True) client = ctx.obj['client'] vdc_href = ctx.obj['profiles'].get('vdc_href') vdc = VDC(client, href=vdc_href) disk_resource = vdc.create_disk( name=name, size=humanfriendly.parse_size(size), description=description, storage_profile_name=storage_profile, iops=iops) stdout(disk_resource.Tasks.Task[0], ctx) except Exception as e: stderr(e, ctx)
def __init__(self, root=None, settings=None): self.fs_root = root self.fs_path = join(root, 'drops') self.fs_submission_queue = join(root, 'submissions') self.fs_scratch = join(root, 'scratch') # initialise settings from disk and parameters # settings provided as init parameter take precedence over values on-disk # which in turn take precedence over default values self.settings = dict( attachment_size_threshold=u'2Mb', ) self.settings.update(**self.parse_settings()) if settings is not None: self.settings.update(**settings) # set archive paths self.fs_archive_cleansed = self.settings.get('dropbox_cleansed_archive_path', join(root, 'archive_cleansed')) self.fs_archive_dirty = self.settings.get('dropbox_dirty_archive_path', join(root, 'archive_dirty')) self.fs_archive = dict( clean=self.fs_archive_cleansed, dirty=self.fs_archive_dirty, ) # set smtp instance defensively, to not overwrite mocked version from test settings: if 'smtp' not in self.settings: self.settings['smtp'] = setup_smtp_factory(**self.settings) # setup GPG self.gpg_context = gnupg.GPG( gnupghome=self.settings['fs_pgp_pubkeys'], gpgbinary=self.settings.get('fs_gpg_path', 'gpg'), ) # convert human readable size to bytes self.settings['attachment_size_threshold'] = parse_size(self.settings['attachment_size_threshold']) # ensure directories exist for directory in [ self.fs_root, self.fs_path, self.fs_submission_queue, self.fs_archive_cleansed, self.fs_archive_dirty, self.fs_scratch]: if not exists(directory): makedirs(directory)
def process_module_programming(module, lines, specific, source_path): log("Processing programming module") # Hledame vzorovy kod v zadani line = 0 while (line < len(lines)) and (not re.match(r"^```~python", lines[line])): line += 1 if line == len(lines): return lines # Hledame konec kodu end = line+1 while (end < len(lines)) and (not re.match(r"^```", lines[end])): end += 1 code = ''.join(lines[line+1:end]) # Pridame vzorovy kod do \module.data data = {} #old_data = json.loads(module.data) if module.data else None #data['programming'] = old_data['programming'] if (old_data) and ('programming' in old_data) else {} data['programming'] = {} data['programming']['default_code'] = code # Zkopirujeme skripty do prislusnych adresaru target_path = "data/modules/" + str(module.id) + "/" files = [ "eval.py", "merge.py", "post.py", "stdin.txt" ] if 'available' in specific: files.extend(specific['available']) if not os.path.isdir(target_path): os.makedirs(target_path) for f in files: if os.path.isfile(source_path+"/"+f): shutil.copy2(source_path+"/"+f, target_path+f) data['programming']['merge_script'] = target_path + "merge.py" data['programming']['stdin'] = target_path + "stdin.txt" if not os.path.isfile(source_path+"/stdin.txt"): open(target_path + "stdin.txt", "a").close() # create empty stdin if os.path.isfile(source_path+"/post.py"): data['programming']['post_trigger_script'] = target_path + "post.py" data['programming']['check_script'] = target_path + "eval.py" # direktivy z module.json if 'timeout' in specific: data['programming']['timeout'] = specific['timeout'] if 'args' in specific: data['programming']['args'] = specific['args'] if 'heaplimit' in specific: data['programming']['heaplimit'] = parse_size(specific['heaplimit']) log("Heaplimit parsed as " + str(data['programming']['heaplimit']) + " bytes") module.data = json.dumps(data, indent=2) return lines[:line]
def __init__(self, path, num_parts=None, part_size=None, base_dir=None): # Basic # self.path = path # Directory # if base_dir is None: self.base_dir = path + '.parts/' else: self.base_dir = base_dir # Num parts # if num_parts is not None: self.num_parts = num_parts # Evaluate size # if part_size is not None: self.bytes_target = humanfriendly.parse_size(part_size) self.num_parts = int(math.ceil(self.count_bytes / self.bytes_target)) # Make parts # self.make_name = lambda i: self.base_dir + "%03d/part.fasta" % i self.parts = [FASTA(self.make_name(i)) for i in range(1, self.num_parts+1)] # Give a number to each part # for i, part in enumerate(self.parts): part.num = i
def download(source, username, directory, max_size): """ Download data from Open Humans. """ signal.signal(signal.SIGINT, signal_handler_cb) max_bytes = parse_size(max_size) options = {} if source: options['source'] = source if username: options['username'] = username page = '{}?{}'.format(BASE_URL_API, urlencode(options)) results = [] counter = 1 print 'Retrieving metadata' while True: print 'Retrieving page {}'.format(counter) response = get_page(page) results = results + response['results'] if response['next']: page = response['next'] else: break counter += 1 print 'Downloading {} files'.format(len(results)) download_url_partial = partial(download_url, directory=directory, max_bytes=max_bytes) with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: for value in executor.map(download_url_partial, results): if value: print value
def _scrape_movie(url): request = YtsRequest(url) response = ScraperBase._try_open_stream(request) soup = BeautifulSoup(response, YtsScraper.HtmlLibrary) movie_wrapper = soup.find("div", {'id': YtsScraper.MovieWrapperClass}) image_url = movie_wrapper.find("div", YtsScraper.MovieImageClass).find("img")["src"] movie_info_section = movie_wrapper.find("div", YtsScraper.MovieInfoClass) full_title = movie_info_section.find("h1").getText() title_re = re.search('^(.+)\s+\((\d+)\).*$', full_title) title = str(title_re.group(1)) year = int(title_re.group(2)) info_dict = {str(info_rex.group(1)).lower().replace(' ', ''): str(info_rex.group(2)) for info_rex in [re.search('^(.+?):\s*(.+)$', info.getText().replace('\n', '').strip()) for info in movie_info_section.findAll("p")] if info_rex is not None} genres = [genre.strip() for genre in info_dict["genre"].split('|')] size = humanfriendly.parse_size(info_dict["size"]) rating = float(info_dict["imdbrating"].split("/")[0]) quality = MovieQuality.parse_p_value(info_dict["quality"]) tex = re.search('^(?:(\d+)(?:hr\s*))?(\d+)\s+min$', info_dict["runtime"]) hours = int(tex.group(1)) if tex.group(1) is not None else 0 minutes = int(tex.group(2)) runtime = hours * 60 + minutes description_section = movie_wrapper.find("div", YtsScraper.MovieDescriptionClass) synopsis_section = description_section.find("div", YtsScraper.MovieDescriptionInnerClass) synopsis = synopsis_section.find('p').getText().replace('\n', '') page_links = [urllib.parse.urlparse(anchor['href']) for anchor in movie_info_section.findAll('a')] imdb = youtube_id = magnet = "" for page_link in page_links: if page_link.netloc == "www.imdb.com": imdb = re.search('.+?/(.+)/$', page_link.path).group(1) elif page_link.netloc == "www.youtube.com": youtube_id = page_link.params['v'] elif page_link.scheme == "magnet": magnet = page_link.geturl() return Movie(MediaSource.Yts, title, year, url, magnet, imdb, rating, genres, quality, size, runtime, synopsis, image_url, youtube_id)
def metrics_series(self): ''' Aggregate check results metrics series attached to this state. ''' results = CheckResult.objects.get_state_log(self, max_results=1000) metrics = defaultdict(list) for result in reversed(results): for name, value in result.metrics.items(): if isinstance(value, six.string_types): try: value = humanfriendly.parse_size(value) except humanfriendly.InvalidSize: continue metrics[name].append((result.timestamp, value)) # Disable default_factory in defaultdict so it can be used in django # templates metrics.default_factory = None return metrics
def render_container_spec(app_name, proc): c = ContainerSpec() c.Image = proc.image c.Env = copy.deepcopy(proc.env) c.set_env("TZ", 'Asia/Shanghai') c.User = '' if not hasattr(proc, 'user') else proc.user c.WorkingDir = '' if not hasattr(proc, 'working_dir') else proc.working_dir c.DnsSearch = [] if not hasattr(proc, 'dns_search') else copy.deepcopy(proc.dns_search) c.Volumes = copy.deepcopy(proc.volumes) c.SystemVolumes = copy.deepcopy(proc.system_volumes) + get_system_volumes_from_etcd(app_name) c.CloudVolumes = render_cloud_volumes(proc.cloud_volumes) c.Command = proc.cmd c.Entrypoint = proc.entrypoint c.CpuLimit = proc.cpu c.MemoryLimit = humanfriendly.parse_size(proc.memory) c.Expose = 0 if not proc.port else proc.port.keys()[0] c.LogConfig = None return c
def main(): set_stdout_encoding() parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter, description='' ) parser.add_argument('fetch_source') parser.add_argument('--host', default='localhost', help='Host interface to listen on') parser.add_argument('-p', '--port', type=int, default=8080) parser.add_argument('-l', '--loglevel', '--log-level', help='Logging verbosity level threshold (to stderr)', default='info') parser.add_argument('--static-path', default='.', help='Path where HTML files are stored') parser.add_argument('--store-path', default='.', help='Path where Communications should be stored') parser.add_argument('--max-file-size', type=str, default='1GiB', help="Maximum size of (non-ZIP) files that can be read into memory " "(e.g. '2G', '300MB')") args = parser.parse_args() logging.basicConfig(format='%(asctime)-15s %(levelname)s: %(message)s', level=args.loglevel.upper()) comm_container = {} if os.path.isdir(args.fetch_source): comm_container = DirectoryBackedCommunicationContainer(args.fetch_source) elif zipfile.is_zipfile(args.fetch_source): comm_container = ZipFileBackedCommunicationContainer(args.fetch_source) else: max_file_size = humanfriendly.parse_size(args.max_file_size, binary=True) comm_container = MemoryBackedCommunicationContainer(args.fetch_source, max_file_size=max_file_size) fetch_handler = CommunicationContainerFetchHandler(comm_container) store_handler = DirectoryBackedStoreHandler(args.store_path) logging.info('Fetch endpoint: http://{}:{}/fetch_http_endpoint/'.format(args.host, args.port)) logging.info('Store endpoint: http://{}:{}/store_http_endpoint/'.format(args.host, args.port)) ahs = AccessHTTPServer(args.host, args.port, args.static_path, fetch_handler, store_handler) ahs.serve()
def scan(self): with self.getdbcursor() as (db, cursor): selected = cursor.execute(select1).fetchall() for pkg, path in selected: print Fore.BLUE + 'Uploading %s for scan ...' % pkg, path = os.path.abspath(os.path.join(self.basedir, path)) if os.path.getsize(path) >= hf.parse_size('32M'): cursor.execute(insert, (pkg, "file-too-big", "", -100)) print Fore.RED + 'File too big for VirusTotal' continue try: id = self.api.scan(path) cursor.execute(insert, (pkg, id, time(), -1)) print Fore.GREEN + 'OK' except: cursor.execute(insert, (pkg, "api-scan-error", "", -100)) print Fore.RED + 'API Error' db.commit() return len(selected)
def update(ctx, name, new_name, size, description, storage_profile, iops): try: restore_session(ctx, vdc_required=True) client = ctx.obj['client'] vdc_href = ctx.obj['profiles'].get('vdc_href') vdc = VDC(client, href=vdc_href) new_size = None if size is not None: new_size = humanfriendly.parse_size(size) name, id = extract_name_and_id(name) task = vdc.update_disk( name=name, disk_id=id, new_name=new_name, new_size=new_size, new_description=description, new_storage_profile_name=storage_profile, new_iops=iops) stdout(task, ctx) except Exception as e: stderr(e, ctx)
def create(filename, size): size_in_bytes = humanfriendly.parse_size(size) with open(filename, "wb") as out: out.truncate(size_in_bytes) cmd = "cryptsetup -y -v luksFormat %s" % filename subprocess.call(cmd.split()) mapping_name = os.path.basename(filename) cmd = "cryptsetup luksOpen %s %s" % (filename, mapping_name) subprocess.call(cmd.split()) cmd = "dd if=/dev/zero of=/dev/mapper/%s" % mapping_name subprocess.call(cmd.split()) cmd = "mkfs.ext4 /dev/mapper/%s" % mapping_name subprocess.call(cmd.split()) cmd = "cryptsetup luksClose %s" % mapping_name subprocess.call(cmd.split())
def parse_result(self, tree): rows = tree.xpath(CN_XPATH_RESULTS_ROWS) if len(rows) >= 1: logging.debug('found %d rows' % len(rows)) results = [] for row in rows: result = {} tds = row.getchildren() a = tds[1].find('.//a[@href]') ep_found = 1 for regex in EP_PATTERN_REGEX: r = re.match(regex, a.text_content()) if r: result['season'] = r.group('season') result['ep_start'] = r.group('start') result['ep_end'] = result['ep_start'] if 'end' in regex: result['ep_end'] = r.group('end') ep_found = (int(result['ep_end']) - int(result['ep_start'])) if ep_found == 0: ep_found = 1 break result['text'] = a.text_content() result['link'] = a.attrib['href'] size = humanfriendly.parse_size(tds[2].text_content()) result['size'] = size / ep_found result['data'] = time.strptime(tds[4].text_content(), "%d.%m.%y") result['seeds'] = int(tds[5].text_content()) result['leech'] = int(tds[6].text_content()) item_tree = html.parse(urllib2.urlopen(result['link'])) links = item_tree.xpath('//a[@href]') for link in links: if link.attrib['href'].startswith('magnet:?'): result['torrent'] = link.attrib['href'] results.append(result) return(results) else: return([])
def test_cli(self): """Test the command line interface.""" # Test that the usage message is printed by default. returncode, output = main() assert 'Usage:' in output # Test that the usage message can be requested explicitly. returncode, output = main('--help') assert 'Usage:' in output # Test handling of invalid command line options. returncode, output = main('--unsupported-option') assert returncode != 0 # Test `humanfriendly --format-number'. returncode, output = main('--format-number=1234567') assert output.strip() == '1,234,567' # Test `humanfriendly --format-size'. random_byte_count = random.randint(1024, 1024 * 1024) returncode, output = main('--format-size=%i' % random_byte_count) assert output.strip() == humanfriendly.format_size(random_byte_count) # Test `humanfriendly --format-table'. returncode, output = main('--format-table', '--delimiter=\t', input='1\t2\t3\n4\t5\t6\n7\t8\t9') assert output.strip() == dedent(''' ------------- | 1 | 2 | 3 | | 4 | 5 | 6 | | 7 | 8 | 9 | ------------- ''').strip() # Test `humanfriendly --format-timespan'. random_timespan = random.randint(5, 600) returncode, output = main('--format-timespan=%i' % random_timespan) assert output.strip() == humanfriendly.format_timespan(random_timespan) # Test `humanfriendly --parse-size'. returncode, output = main('--parse-size=5 KB') assert int(output) == humanfriendly.parse_size('5 KB') # Test `humanfriendly --run-command'. returncode, output = main('--run-command', 'bash', '-c', 'sleep 2 && exit 42') assert returncode == 42
def validate_parameters(cpu, memory, numinstances): if all([cpu is None, memory is None, numinstances is None]): warn("please input at least one param in cpu/memory/numinstances") exit(1) if numinstances is not None: try: numinstances = int(numinstances) except ValueError: warn('invalid parameter: num_instances (%s) should be integer'%numinstances) exit(1) if numinstances <= 0: warn('invalid parameter: num_instances (%s) should > 0'%numinstances) exit(1) if cpu is not None: try: cpu = int(cpu) except ValueError: warn('invalid parameter: cpu (%s) should be integer'%cpu) exit(1) if cpu < 0: warn('invalid parameter: cpu (%s) should >= 0'%cpu) exit(1) if memory is not None: memory = str(memory) try: if humanfriendly.parse_size(memory) < 4194304: error('invalid parameter: memory (%s) should >= 4M'%memory) exit(1) except humanfriendly.InvalidSize: error('invalid parameter: memory (%s) humanfriendly.parse_size(memory) failed'%memory) exit(1) return cpu, memory, numinstances