def humanize(self, **kwargs): """Gets the human-friendly representation of file size. :param kwargs: All keyword arguments will be passed to :func:`humanize.filesize.naturalsize`. """ return naturalsize(self, **kwargs)
def async_post(self, *_, **__): data = [] # filter backups by criteria filter_func = False folder = self.get_folder() if 'filter' in self.request_params: filter_func = True for file_name in os.listdir(folder): file_path = os.path.join(folder, file_name) if os.path.isfile(file_path): info = os.stat(file_path) record = { 'name': file_name, 'date': datetime_to_utc(datetime.fromtimestamp(info.st_ctime)), 'size': info.st_size } if not filter_func or self.filter_func(record): data.append(record) totalCount = len(data) if 'sort' in self.request_params: for sort_criteria in self.request_params['sort']: sort_property = sort_criteria['property'] reverse = True if sort_criteria['direction'].lower( ) == 'asc' else False data.sort(key=lambda record: record[sort_property], reverse=reverse) bottom = (self.request_params['page'] - 1) * self.request_params['limit'] top = bottom + self.request_params['limit'] data = data[bottom:top] for record in data: record['size'] = naturalsize(record['size']) return {'totalCount': totalCount, 'data': data}
def print(self): for p in self.arr: if p.number_of_reports_with_swapping( ) * args.interval > args.hl_analyze * 60: print(Fore.RED + _( "{} [{}] has {} reports with {} of swap variation average and {} of swap average. They took {}" ).format(p.datetime(), p.name(), int(p.length()), filesize.naturalsize(int(p.average_diff())), filesize.naturalsize(int(p.average_swap())), p.duration()) + Fore.RESET) else: if args.hl_only == False: print( _("{} [{}] has {} reports with {} of swap variation average and {} of swap average. They took {}" ).format(p.datetime(), p.name(), int(p.length()), filesize.naturalsize(int(p.average_diff())), filesize.naturalsize(int(p.average_swap())), p.duration()))
def __repr__(self): return '<History [' \ 'id={short_id}, ' \ 'created={created}, ' \ 'size={size}]>'.format( short_id=self.short_id, created=self.created.isoformat(), size=filesize.naturalsize(self.size) )
def size(): """Return how much memory we're taking. Doesn't work if you're running with Gunicorn because it makes child processes. """ process = psutil.Process(os.getpid()) n_bytes = process.memory_info().rss # in bytes return jsonify( n_bytes=n_bytes, n_bytes_human=naturalsize(n_bytes), )
def my_view(request): up_since = index["up_since"] up_time = datetime.now() - up_since cache_stats = cache.stats() return { 'entries': cache.cache_len(), 'cache_disk_size': filesize.naturalsize(cache.cache_disk_size()), 'up_date': time.naturaldate(up_since), 'up_duration': time.naturaldelta(up_time), 'cache_hits': str(cache_stats['hit']), 'cache_misses': str(cache_stats['misses']) }
def __repr__(self): repo = tag = '<none>' # just tak the first tag if there is one if self.tags: repo, tag = self.tags[0].split(':') return '<Image [' \ 'repo={repo}, ' \ 'tag={tag}, ' \ 'id={short_id}, ' \ 'vsize={vsize}]>'.format( repo=repo, tag=tag, short_id=self.short_id, vsize=filesize.naturalsize(self.virtual_size) )
def plot(results, rate=False): xs = np.array(sizes) ys_aesaes, std_aesaes, ys_twoaes, std_twoaes = map(np.array, zip(*results)) plt.xscale('log') plt.xlabel('file size') plt.xlim(xs[0], xs[-1]) plt.xticks(xs, [naturalsize(x, binary=True) for x in xs], rotation=90) plt.ylabel('re-encryption ' + ('rate ($MB/s$)' if rate else 'time ($s$)')) plt.yscale('linear' if rate else 'log') plt.ylim(0, max(max(ys_aesaes), max(ys_twoaes)) * 1.2) plt.errorbar(xs, ys_aesaes, yerr=std_aesaes, fmt='r^--', label='AES+AES') plt.errorbar(xs, ys_twoaes, yerr=std_twoaes, fmt='b-', label='TWOAES') plt.fill_between(xs, ys_aesaes, ys_twoaes, where=ys_aesaes>ys_twoaes, facecolor='g', alpha=.2, interpolate=True) plt.fill_between(xs, ys_aesaes, ys_twoaes, where=ys_aesaes<ys_twoaes, facecolor='r', alpha=.2, interpolate=True) plt.legend(loc='lower right') plt.tight_layout()
def test(size, use_aesni, runs, out, tempdir, rate=False): a1 = AES.new(k1, mode=AES.MODE_ECB, use_aesni=use_aesni) a2 = AES.new(k2, mode=AES.MODE_ECB, use_aesni=use_aesni) if use_aesni: ta = TWOAES.new(k1+k2, mode=TWOAES.MODE_ECB, use_aesni=True, window=10) else: ta = TWOAES.new(k1+k2, mode=TWOAES.MODE_ECB, use_aesni=False) def single_test(size, recrypt): # setup plain = bytegenerator(size) cipher = a1.encrypt(plain) # test start = default_timer() cipher2 = recrypt(cipher) time = default_timer() - start assert a2.decrypt(cipher2) == plain return (size / (2. ** 20)) / time if rate else time def recrypt_aesaes(cipher): plain = a1.decrypt(cipher) if tempdir: with NamedTemporaryFile(dir=tempdir) as temp: temp.write(plain) temp.flush() temp.seek(0) plain = temp.read() return a2.encrypt(plain) results = ( repeat(partial(single_test, size, recrypt_aesaes), runs, out) + repeat(partial(single_test, size, ta.encrypt), runs, out)) print(pattern.format(naturalsize(size,binary=True), results[0], results[2])) return results
def bundle(self, firmware_hash: str, datasets: List[Dataset], *, file: Union[str, BinaryIO, IO[bytes]], shard_spec: ShardSpec = None, delta_to: Dict[str, str] = None, overwrite: bool = False) -> List[ObjectInfo]: """ Builds a data bundle (*.tar.gz) for a firmware hash, including content from the specified datasets (FWAN plugin output locations) :param firmware_hash: The firmware hash to bundle :param datasets: The datasets to include in the bundle :param file: The output path or file-like-object to which the *.tar.gz output should be written :param shard_spec: If provided, only the specified shard of file hashes will appear in the bundle :param delta_to: A dictionary of path->etag values, which if supplied will cause the bundle to be built as a delta to that set, meaning only new objects or objects with modified etags will appear in the bundle. :param overwrite: The output path will not be overwritten, to prevent accidental data loss, unless this is set :return: A list of the object included in the bundle """ if not firmware_hash: raise ValueError('firmware_hash must be specified') if not datasets: raise ValueError('datasets must be specified, and non-empty') if delta_to is None: delta_to = {} logger.info( f"Building {'delta' if delta_to else ''} bundle for {firmware_hash}" ) contents: List[ObjectInfo] = [] with mgzip.open(filename=file, mode='w' if overwrite else 'x') as gz, tarfile.open( fileobj=gz, mode='w', bufsize=tarfile.RECORDSIZE * 4) as tar: # Fetch and process the file tree, using that as the basis for all other paths that need to be bundled. file_tree_path = f'file_tree/{firmware_hash}.jsonl' with CodeTimer('Read firmware file tree from object storage'): try: file_tree_result = fetch_object( bucket=self.firmware_metadata_bucket, key=file_tree_path) except ClientError as e: raise Exception( 'Firmware file tree could not be read') from e with CodeTimer('Extract file hashes from file tree'): try: file_hashes = extract_file_hashes(file_tree_result.payload) except json.JSONDecodeError as e: raise Exception( 'Firmware file tree could not be parsed') from e if not file_hashes: raise Exception('Firmware file tree is empty') file_tree_in_bundle = False if is_dataset_in_shard( dataset=FILE_TREE_DATASET, shard_spec=shard_spec ) and file_tree_result.info.etag != delta_to.get(file_tree_path): with CodeTimer('Add file tree to bundle'): add_to_tarfile(tar, file_tree_result) contents.append(file_tree_result.info) file_tree_in_bundle = True else: logger.info( 'File tree is unchanged or not part of this shard and will not be included in the bundle' ) file_tree_size = file_tree_result.info.size logger.info( 'File tree num distinct file hashes = {hash_count}; size = {size}' .format(hash_count=len(file_hashes), size=naturalsize(file_tree_size))) if shard_spec: with CodeTimer( f'Limiting file hashes to shard {shard_spec.index}'): def is_in_shard(file_hash: str) -> bool: return int(file_hash, 16) % shard_spec.count == shard_spec.index file_hashes = [ file_hash for file_hash in file_hashes if is_in_shard(file_hash) ] logger.info(f'Sharded num file hashes = {len(file_hashes)}') file_tree_result = None # Build paths to be bundled with CodeTimer('Build paths for bundle'): bundle_datasets = [ ds for ds in datasets if is_dataset_in_shard(dataset=ds, shard_spec=shard_spec) ] paths = self.build_paths(firmware_hash=firmware_hash, datasets=bundle_datasets, file_hashes=file_hashes, delta_to=delta_to) or [] path_count = len(paths) # Validate the paths (check for duplicates) with CodeTimer('Validate paths'): duplicates = [ path for path, count in collections.Counter(paths).items() if count > 1 ] if duplicates: raise Exception( f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}' ) total_path_count = path_count + 1 if file_tree_in_bundle else 0 logger.info( f'Bundle will include at most {total_path_count} paths from object storage' ) fetch_count = 0 miss_count = 0 skip_count = 0 fetch_bytes = 0 with CodeTimer('Bundle objects'): with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as executor: fetch_start = datetime.datetime.now() with CodeTimer( 'Submit object storage path retrieval tasks'): # Randomize path ordering to improve the performance of fetches from object storage, # so that a diversity of object key prefixes is being fetched at any one time. random.shuffle(paths) futures = [ executor.submit( fetch_object, bucket=self.firmware_metadata_bucket, key=path, compare_etag=delta_to.get(path), ) for path in paths ] for future in concurrent.futures.as_completed(futures): try: result = future.result() if result: add_to_tarfile(tar, result) result.payload = None fetch_count += 1 fetch_bytes += result.info.size contents.append(result.info) else: skip_count += 1 if fetch_count % 1000 == 0: logger.info( 'Bundled {} objects ({}) in {}'.format( fetch_count, naturalsize(fetch_bytes), naturaldelta(datetime.datetime.now() - fetch_start))) except ClientError as e: error_code = e.response.get('Error', {}).get('Code') if 'NoSuchKey' in error_code: miss_count += 1 elif '304' == error_code: # The ETag on this object was not modified, so it was not returned skip_count += 1 else: raise e if skip_count: logger.info(f'Skipped {skip_count} unmodified objects') if miss_count: logger.info( f"Made {miss_count} attempts to access object storage paths that didn't exist" ) logger.info('Bundled {} objects ({})'.format( fetch_count + (1 if file_tree_in_bundle else 0), naturalsize(fetch_bytes + (file_tree_size if file_tree_in_bundle else 0)))) # Validate fetched paths (check that each path was uniquely processed) with CodeTimer('Validate fetched paths'): fetched_path_counter = collections.Counter( [obj.path for obj in contents]) duplicates = [ path for path, count in fetched_path_counter.items() if count > 1 ] if duplicates: raise Exception( f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}' ) with CodeTimer('Finalize output'): contents = sorted(contents, key=lambda obj: obj.path) return contents
def humanize(cls, value): return naturalsize(value, binary=True)
def overview(request, format=None): #External Ceph REST API calls status_response, cluster_status = get_data.status(body='json') health_response, cluster_health = get_data.health(body='json') pg_response, pg_stat = get_data.pg_stat(body='json') osd_response, osd_dump = get_data.osd_dump(body='json') osd_perf_response, osd_perf = get_data.osd_perf(body='json') #General Health status lookup health_lookup = {'HEALTH_OK':"OK", 'HEALTH_WARN':"WARN"} # pg states lookups pg_warn_status = re.compile("(creating|degraded|replay|splitting|scrubbing|repair|recovering|backfill|wait-backfill|remapped)") pg_crit_status = re.compile("(down|inconsistent|incomplete|stale|peering)") pg_ok_status = re.compile("(active+clean)") pg_health_lookup = lambda x: "ok" if pg_ok_status.search(x) else "warn" if pg_warn_status.search(x) else "crit" if pg_warn_status.search(x) else "na" #osd states lookups osd_up = re.compile("(?=.*exists)(?=.*up)") osd_down = re.compile("(?=.*exists)(?=.*autoout)") osd_health_lookup = lambda x: "ok" if osd_up.search(str(x)) else "warn" if osd_down.search(str(x)) else "crit" #pg activities lookups pg_activities_lookup = {'read_bytes_sec':"Read", 'write_bytes_sec':"write", 'op_per_sec': "Ops", 'recovering_objects_per_sec': "Recovering Objects", 'recovering_bytes_per_sec': 'Recovery Speed', 'recovering_keys_per_sec': 'Recovering Keys'} #for testing since I have no activity #pg_activities_lookup = {'read_bytes_sec':"Read", 'bytes_total' : "Bytes", 'degraded_objects' : "Degraded"} #mon status data mons_status = filter(lambda x: x['health'] in health_lookup, cluster_status['output']['health']['health']['health_services'][0]['mons']) mon_count = len(cluster_status['output']['monmap']['mons']) #pg status data pg_status = cluster_status['output']['pgmap']['pgs_by_state'] #osd status data osd_status = osd_dump['output']['osds'] #pg activities data pg_activities = cluster_status['output']['pgmap'] #usage data usage_bytes_used = cluster_status['output']['pgmap']['bytes_used'] usage_bytes_total = cluster_status['output']['pgmap']['bytes_total'] usage_data_total = filesize.naturalsize(usage_bytes_total, binary=True).split()[0] usage_data_scale = filesize.naturalsize(usage_bytes_total).split()[1] response = {'health':{ 'clusterHealth': { 'status': cluster_health['output']['overall_status'], 'statusDescription': (lambda x, y: "CRIT" if x not in y else y[x] )(cluster_health['output']['overall_status'], health_lookup) } }, 'status':{ 'mons': dict(({'ok':0,'warn':0,'crit':mon_count-len(mons_status)}).items() + Counter(map(lambda x: health_lookup[x['health']].lower(), mons_status)).items()) , 'pgs': dict(({'ok':0,'warn':0,'crit':0}).items() + reduce(lambda y,z: y.update(z), map(lambda x: {pg_health_lookup(x['state_name']).lower(): x['count']}, pg_status)).items()) , 'osds': dict(({'ok':0,'warn':0,'crit':mon_count-len(mons_status)}).items() + Counter(map(lambda x: osd_health_lookup(x['state']).lower(), osd_status)).items()) }, 'usage':{ 'clusterBytesUsed':usage_bytes_used, 'clusterBytesAvail':usage_bytes_total-usage_bytes_used, 'clusterBytesTotal':usage_bytes_total, 'clusterDataUsed': round(float(usage_bytes_used)/pow(1024, filesize.suffixes['decimal'].index(usage_data_scale)+1), 1), 'clusterDataAvail':round(float(usage_bytes_total-usage_bytes_used)/pow(1024, filesize.suffixes['decimal'].index(usage_data_scale)+1), 1), 'clusterDataTotal': usage_data_total, 'clusterDataScale': usage_data_scale }, 'activities':reduce(lambda y, z: dict(y.items()+z.items()), map(lambda x: {x: pg_activities.get(x)} if x in pg_activities else {},pg_activities_lookup)), 'pg_states': reduce(lambda y, z: dict(y.items()+z.items()), map(lambda x: {x['state_name']:x['count']}, pg_status)), 'osd_states': Counter(map(lambda x: '+'.join(x['state']), osd_status)) } return Response(response)
def home(request): """ Main dashboard, Overall cluster health and status """ ceph = wrapper.CephWrapper(endpoint=settings.CEPH_BASE_URL) cresp, cluster_health = ceph.health(body="json") sresp, cluster_status = ceph.status(body="json") # Monitors all_mons = cluster_status["output"]["monmap"]["mons"] up_mons = cluster_status["output"]["health"]["timechecks"]["mons"] total_mon_count = len(all_mons) mons_ok = 0 mons_warn = 0 mons_crit = 0 for mon in up_mons: if mon["health"] == "HEALTH_OK": mons_ok += 1 else: mons_warn += 1 mons_crit = total_mon_count - (mons_ok + mons_warn) # Activity pgmap = cluster_status["output"]["pgmap"] activities = {} if "read_bytes_sec" in pgmap: activities["Read"] = filesize.naturalsize(pgmap.get("read_bytes_sec")) if "write_bytes_sec" in pgmap: activities["Write"] = filesize.naturalsize(pgmap.get("write_bytes_sec")) if "op_per_sec" in pgmap: activities["Ops"] = pgmap.get("op_per_sec") if "recovering_objects_per_sec" in pgmap: activities["Recovering Objects"] = pgmap.get("recovering_objects_per_sec") if "recovering_bytes_per_sec" in pgmap: activities["Recovery Speed"] = filesize.naturalsize(pgmap.get("recovering_bytes_per_sec")) if "recovering_keys_per_sec" in pgmap: activities["Recovering Keys"] = pgmap.get("recovering_keys_per_sec") # Get a rough estimate of cluster free space. Is this accurate ? presp, pg_stat = ceph.pg_stat(body="json") bytes_total = cluster_status["output"]["pgmap"]["bytes_total"] bytes_used = cluster_status["output"]["pgmap"]["bytes_used"] data_avail, data_scale = filesize.naturalsize(bytes_total).split() scale = filesize.suffixes["decimal"].index(data_scale) + 1 data_used = round(float(bytes_used) / pow(1024, scale), 1) # pgs pg_statuses = cluster_status["output"]["pgmap"] pg_ok = 0 pg_warn = 0 pg_crit = 0 # pg states pg_warn_status = re.compile( "(creating|degraded|replay|splitting|scrubbing|repair|recovering|backfill|wait-backfill|remapped)" ) pg_crit_status = re.compile("(down|inconsistent|incomplete|stale|peering)") for state in pg_statuses["pgs_by_state"]: if state["state_name"] == "active+clean": pg_ok = pg_ok + state["count"] elif pg_warn_status.search(state["state_name"]): pg_warn = pg_warn + state["count"] elif pg_crit_status.search(state["state_name"]): pg_crit = pg_crit + state["count"] # pg statuses pg_states = dict() for state in pg_statuses["pgs_by_state"]: pg_states[state["state_name"]] = state["count"] # osds dresp, osd_dump = ceph.osd_dump(body="json") osd_state = osd_dump["output"]["osds"] osds_ok = 0 osds_warn = 0 osds_crit = 0 # Possible states are: exists, up, autoout, new, ??? osd_up = re.compile("(?=.*exists)(?=.*up)") osd_down = re.compile("(?=.*exists)(?=.*autoout)") for osd_status in osd_state: if osd_up.search(str(osd_status["state"])): osds_ok += 1 elif osd_down.search(str(osd_status["state"])): osds_warn += 1 else: osds_crit += 1 return render_to_response("dashboard.html", locals())
def overview(request, format=None): #External Ceph REST API calls status_response, cluster_status = get_data.status(body='json') health_response, cluster_health = get_data.health(body='json') pg_response, pg_stat = get_data.pg_stat(body='json') osd_response, osd_dump = get_data.osd_dump(body='json') osd_perf_response, osd_perf = get_data.osd_perf(body='json') #General Health status lookup health_lookup = {'HEALTH_OK': "OK", 'HEALTH_WARN': "WARN"} # pg states lookups pg_warn_status = re.compile( "(creating|degraded|replay|splitting|scrubbing|repair|recovering|backfill|wait-backfill|remapped)" ) pg_crit_status = re.compile("(down|inconsistent|incomplete|stale|peering)") pg_ok_status = re.compile("(active+clean)") pg_health_lookup = lambda x: "ok" if pg_ok_status.search( x) else "warn" if pg_warn_status.search( x) else "crit" if pg_warn_status.search(x) else "na" #osd states lookups osd_up = re.compile("(?=.*exists)(?=.*up)") osd_down = re.compile("(?=.*exists)(?=.*autoout)") osd_health_lookup = lambda x: "ok" if osd_up.search(str( x)) else "warn" if osd_down.search(str(x)) else "crit" #pg activities lookups pg_activities_lookup = { 'read_bytes_sec': "Read", 'write_bytes_sec': "write", 'op_per_sec': "Ops", 'recovering_objects_per_sec': "Recovering Objects", 'recovering_bytes_per_sec': 'Recovery Speed', 'recovering_keys_per_sec': 'Recovering Keys' } #for testing since I have no activity #pg_activities_lookup = {'read_bytes_sec':"Read", 'bytes_total' : "Bytes", 'degraded_objects' : "Degraded"} #mon status data mons_status = filter( lambda x: x['health'] in health_lookup, cluster_status['output'] ['health']['health']['health_services'][0]['mons']) mon_count = len(cluster_status['output']['monmap']['mons']) #pg status data pg_status = cluster_status['output']['pgmap']['pgs_by_state'] #osd status data osd_status = osd_dump['output']['osds'] #pg activities data pg_activities = cluster_status['output']['pgmap'] #usage data usage_bytes_used = cluster_status['output']['pgmap']['bytes_used'] usage_bytes_total = cluster_status['output']['pgmap']['bytes_total'] usage_data_total = filesize.naturalsize(usage_bytes_total, binary=True).split()[0] usage_data_scale = filesize.naturalsize(usage_bytes_total).split()[1] response = { 'health': { 'clusterHealth': { 'status': cluster_health['output']['overall_status'], 'statusDescription': (lambda x, y: "CRIT" if x not in y else y[x])( cluster_health['output']['overall_status'], health_lookup) } }, 'status': { 'mons': dict(({ 'ok': 0, 'warn': 0, 'crit': mon_count - len(mons_status) }).items() + Counter( map(lambda x: health_lookup[x['health']].lower(), mons_status)).items()), 'pgs': dict(({ 'ok': 0, 'warn': 0, 'crit': 0 }).items() + reduce( lambda y, z: y.update(z), map( lambda x: {pg_health_lookup(x['state_name']).lower(): x['count']}, pg_status)).items()), 'osds': dict(({ 'ok': 0, 'warn': 0, 'crit': mon_count - len(mons_status) }).items() + Counter( map(lambda x: osd_health_lookup(x['state']).lower(), osd_status)).items()) }, 'usage': { 'clusterBytesUsed': usage_bytes_used, 'clusterBytesAvail': usage_bytes_total - usage_bytes_used, 'clusterBytesTotal': usage_bytes_total, 'clusterDataUsed': round( float(usage_bytes_used) / pow(1024, filesize.suffixes['decimal'].index(usage_data_scale) + 1), 1), 'clusterDataAvail': round( float(usage_bytes_total - usage_bytes_used) / pow(1024, filesize.suffixes['decimal'].index(usage_data_scale) + 1), 1), 'clusterDataTotal': usage_data_total, 'clusterDataScale': usage_data_scale }, 'activities': reduce( lambda y, z: dict(y.items() + z.items()), map( lambda x: {x: pg_activities.get(x)} if x in pg_activities else {}, pg_activities_lookup)), 'pg_states': reduce(lambda y, z: dict(y.items() + z.items()), map(lambda x: {x['state_name']: x['count']}, pg_status)), 'osd_states': Counter(map(lambda x: '+'.join(x['state']), osd_status)) } return Response(response)
def home(request): """ Main dashboard, Overall cluster health and status """ ceph = wrapper.CephWrapper(endpoint=settings.CEPH_BASE_URL) cresp, cluster_health = ceph.health(body='json') sresp, cluster_status = ceph.status(body='json') # Monitors all_mons = cluster_status['output']['monmap']['mons'] up_mons = cluster_status['output']['health']['timechecks']['mons'] total_mon_count = len(all_mons) mons_ok = 0 mons_warn = 0 mons_crit = 0 for mon in up_mons: if mon['health'] == "HEALTH_OK": mons_ok += 1 else: mons_warn += 1 mons_crit = total_mon_count - (mons_ok + mons_warn) # Activity pgmap = cluster_status['output']['pgmap'] activities = {} if 'read_bytes_sec' in pgmap: activities['Read'] = filesize.naturalsize(pgmap.get('read_bytes_sec')) if 'write_bytes_sec' in pgmap: activities['Write'] = filesize.naturalsize( pgmap.get('write_bytes_sec')) if 'op_per_sec' in pgmap: activities['Ops'] = pgmap.get('op_per_sec') if 'recovering_objects_per_sec' in pgmap: activities['Recovering Objects'] = pgmap.get( 'recovering_objects_per_sec') if 'recovering_bytes_per_sec' in pgmap: activities['Recovery Speed'] = filesize.naturalsize( pgmap.get('recovering_bytes_per_sec')) if 'recovering_keys_per_sec' in pgmap: activities['Recovering Keys'] = pgmap.get('recovering_keys_per_sec') # Get a rough estimate of cluster free space. Is this accurate ? presp, pg_stat = ceph.pg_stat(body='json') bytes_total = cluster_status['output']['pgmap']['bytes_total'] bytes_used = cluster_status['output']['pgmap']['bytes_used'] data_avail, data_scale = filesize.naturalsize(bytes_total).split() scale = filesize.suffixes['decimal'].index(data_scale) + 1 data_used = round(float(bytes_used) / pow(1024, scale), 1) # pgs pg_statuses = cluster_status['output']['pgmap'] pg_ok = 0 pg_warn = 0 pg_crit = 0 # pg states pg_warn_status = re.compile( "(creating|degraded|replay|splitting|scrubbing|repair|recovering|backfill|wait-backfill|remapped)" ) pg_crit_status = re.compile("(down|inconsistent|incomplete|stale|peering)") for state in pg_statuses['pgs_by_state']: if state['state_name'] == "active+clean": pg_ok = pg_ok + state['count'] elif pg_warn_status.search(state['state_name']): pg_warn = pg_warn + state['count'] elif pg_crit_status.search(state['state_name']): pg_crit = pg_crit + state['count'] # pg statuses pg_states = dict() for state in pg_statuses['pgs_by_state']: pg_states[state['state_name']] = state['count'] # osds dresp, osd_dump = ceph.osd_dump(body='json') osd_state = osd_dump['output']['osds'] osds_ok = 0 osds_warn = 0 osds_crit = 0 # Possible states are: exists, up, autoout, new, ??? osd_up = re.compile("(?=.*exists)(?=.*up)") osd_down = re.compile("(?=.*exists)(?=.*autoout)") for osd_status in osd_state: if osd_up.search(str(osd_status['state'])): osds_ok += 1 elif osd_down.search(str(osd_status['state'])): osds_warn += 1 else: osds_crit += 1 return render_to_response('dashboard.html', locals())
def home(request): """ Main dashboard, Overall cluster health and status """ get_data = wrapper.CephWrapper(endpoint=settings.CEPH_BASE_URL) cresp, cluster_health = get_data.health(body='json') sresp, cluster_status = get_data.status(body='json') # Monitors all_mons = cluster_status['output']['monmap']['mons'] up_mons = cluster_status['output']['health']['timechecks']['mons'] total_mon_count = len(all_mons) mons_ok = 0 mons_warn = 0 mons_crit = 0 for mon in up_mons: if mon['health'] == "HEALTH_OK": mons_ok += 1 else: mons_warn += 1 mons_crit = total_mon_count - (mons_ok + mons_warn) # Activity pgmap = cluster_status['output']['pgmap'] activities = {} if 'read_bytes_sec' in pgmap: activities['Read'] = filesize.naturalsize(pgmap.get('read_bytes_sec')) if 'write_bytes_sec' in pgmap: activities['Write'] = filesize.naturalsize(pgmap.get('write_bytes_sec')) if 'op_per_sec' in pgmap: activities['Ops'] = pgmap.get('op_per_sec') if 'recovering_objects_per_sec' in pgmap: activities['Recovering Objects'] = pgmap.get('recovering_objects_per_sec') if 'recovering_bytes_per_sec' in pgmap: activities['Recovery Speed'] = filesize.naturalsize(pgmap.get('recovering_bytes_per_sec')) if 'recovering_keys_per_sec' in pgmap: activities['Recovering Keys'] = pgmap.get('recovering_keys_per_sec') # Get a rough estimate of cluster free space. Is this accurate ? presp, pg_stat = get_data.pg_stat(body='json') bytes_total = cluster_status['output']['pgmap']['bytes_total'] bytes_used = cluster_status['output']['pgmap']['bytes_used'] data_avail, data_scale = filesize.naturalsize(bytes_total).split() scale = filesize.suffixes['decimal'].index(data_scale)+1 data_used = round(float(bytes_used)/pow(1024, scale), 1) # pgs pg_statuses = cluster_status['output']['pgmap'] pg_ok = 0 pg_warn = 0 pg_crit = 0 # pg states pg_warn_status = re.compile("(creating|degraded|replay|splitting|scrubbing|repair|recovering|backfill|wait-backfill|remapped)") pg_crit_status = re.compile("(down|inconsistent|incomplete|stale|peering)") for state in pg_statuses['pgs_by_state']: if state['state_name'] == "active+clean": pg_ok = pg_ok + state['count'] elif pg_warn_status.search(state['state_name']): pg_warn = pg_warn + state['count'] elif pg_crit_status.search(state['state_name']): pg_crit = pg_crit + state['count'] # pg statuses pg_states = dict() for state in pg_statuses['pgs_by_state']: pg_states[state['state_name']] = state['count'] # osds dresp, osd_dump = get_data.osd_dump(body='json') osd_state = osd_dump['output']['osds'] osds_ok = 0 osds_warn = 0 osds_crit = 0 # Possible states are: exists, up, autoout, new, ??? osd_up = re.compile("(?=.*exists)(?=.*up)") osd_down = re.compile("(?=.*exists)(?=.*autoout)") for osd_status in osd_state: if osd_up.search(str(osd_status['state'])): osds_ok += 1 elif osd_down.search(str(osd_status['state'])): osds_warn += 1 else: osds_crit += 1 return render_to_response('dashboard.html', locals())
def run_benchmark(self, app_label): logger.info('start running benchmark for {}'.format(app_label)) # run from 1 to 500 sample block classes for num_models in xrange(0, self.num_models + 1, self.step_size): # starting at zero makes no sense at all but we want to have # the steps counted in an intuitive manner, so start with 1 here if num_models == 0: num_models = 1 logger.info('generating models') self.runner.generate_models(num_models) logger.info('creating tables') self.runner.syncdb() bm_result = self.runner.create_result(num_models) bm_result.start = time() start = time() logger.info('creating models') container = self.runner.query_wrapper.create_query( app_label, num_models) bm_result.create_time_complete = time() - start bm_result.create_time_sql = sum( [float(q.get('time', 0)) for q in connection.queries] ) query_times = [] processing_times = [] memory_rss = [] memory_vrt = [] for it in xrange(0, self.num_queries): # clear previously logged queries connection.queries = [] # reset the value for monitored memory usage self.runner.monitor.reset_values() start = time() self.runner.query_wrapper.select_query(container.id) processing_times.append((time() - start)) rss, vrt = self.runner.monitor.get_memory_values() memory_rss.append(rss) memory_vrt.append(vrt) query_time_sql = sum( [float(q.get('time', 0)) for q in connection.queries] ) query_times.append(query_time_sql) bm_result.end = time() num_queries = len(query_times) bm_result.query_time_sql = sum(query_times) / num_queries bm_result.query_time_complete = sum(processing_times) / num_queries bm_result.db_rss = sum(memory_rss) / len(memory_rss) bm_result.db_vrt = sum(memory_vrt) / len(memory_vrt) self.storage.save_result(bm_result) print ("{num_models}, {test_duration:.4f} s, " "{create_time_sql:.4f} s, {create_time_complete:.4f} s, " "{query_time_sql:.4f} s, {query_time_complete:.4f} s, " "{db_rss}, {db_vrt}").format( num_models=num_models, test_duration=bm_result.end - bm_result.start, create_time_sql=bm_result.create_time_sql, create_time_complete=bm_result.create_time_complete, query_time_sql=bm_result.query_time_sql, query_time_complete=bm_result.query_time_complete, db_rss=filesize.naturalsize(bm_result.db_rss), db_vrt=filesize.naturalsize(bm_result.db_vrt), ) self.runner.drop_all_tables()
#!/usr/bin/env python import os from datetime import datetime from humanize import filesize FOLDER = 'DATA' FILE_NAME = 'alice.txt' file_path = os.path.join(FOLDER, FILE_NAME) print("file path:", file_path) file_size = os.path.getsize(file_path) print("file size:", filesize.naturalsize(file_size)) print(filesize.naturalsize(13029093033)) raw_timestamp = os.path.getmtime(file_path) print("raw timestamp:", raw_timestamp) timestamp = datetime.fromtimestamp(raw_timestamp) print("timestamp:", timestamp) print(os.path.dirname(file_path)) print(os.path.basename(file_path)) print(os.path.abspath(file_path)) for x in 'alice.txt', 'wombat.txt', 'mary.txt', 'koalas.txt': print(x, os.path.exists(os.path.join('DATA', x))) print()