def humanize_naturaldelta(eval_ctx, value, months=True, minimum_unit="seconds", when=None): return naturaldelta(value, months=months, minimum_unit=minimum_unit, when=when)
def humanized_next_due_in(self): ''' Can be appended to the prefix "Next review ...". ''' if not self._review_applied: raise Exception("Must first apply the review.") due_in = self.card.due_at - datetime.utcnow() if due_in < timedelta(minutes=1): return "in less than a minute" return "in {}".format(naturaldelta(due_in, months=True))
def humanized_next_due_in(self): ''' Can be appended to the prefix "Next review ...". ''' if not self._review_applied: raise Exception("Must first apply the review.") due_in = self.card.due_at - datetime.utcnow() if due_in < timedelta(minutes=1): return "in less than a minute" return u"in {}".format(naturaldelta(due_in, months=True))
def my_view(request): up_since = index["up_since"] up_time = datetime.now() - up_since cache_stats = cache.stats() return { 'entries': cache.cache_len(), 'cache_disk_size': filesize.naturalsize(cache.cache_disk_size()), 'up_date': time.naturaldate(up_since), 'up_duration': time.naturaldelta(up_time), 'cache_hits': str(cache_stats['hit']), 'cache_misses': str(cache_stats['misses']) }
def test_naturaldelta_nomonths(self): now = datetime.now() test_list = [ timedelta(days=7), timedelta(days=31), timedelta(days=230), timedelta(days=400), ] result_list = [ '7 days', '31 days', '230 days', '1 year, 35 days', ] with patch('humanize.time._now') as mocked: mocked.return_value = now nd_nomonths = lambda d: time.naturaldelta(d, months=False) self.assertManyResults(nd_nomonths, test_list, result_list)
def naturalinterval(secs): string = naturaldelta(secs) string = string.replace("a ", "").replace("an ", "") return "every %s" % string
def bundle(self, firmware_hash: str, datasets: List[Dataset], *, file: Union[str, BinaryIO, IO[bytes]], shard_spec: ShardSpec = None, delta_to: Dict[str, str] = None, overwrite: bool = False) -> List[ObjectInfo]: """ Builds a data bundle (*.tar.gz) for a firmware hash, including content from the specified datasets (FWAN plugin output locations) :param firmware_hash: The firmware hash to bundle :param datasets: The datasets to include in the bundle :param file: The output path or file-like-object to which the *.tar.gz output should be written :param shard_spec: If provided, only the specified shard of file hashes will appear in the bundle :param delta_to: A dictionary of path->etag values, which if supplied will cause the bundle to be built as a delta to that set, meaning only new objects or objects with modified etags will appear in the bundle. :param overwrite: The output path will not be overwritten, to prevent accidental data loss, unless this is set :return: A list of the object included in the bundle """ if not firmware_hash: raise ValueError('firmware_hash must be specified') if not datasets: raise ValueError('datasets must be specified, and non-empty') if delta_to is None: delta_to = {} logger.info( f"Building {'delta' if delta_to else ''} bundle for {firmware_hash}" ) contents: List[ObjectInfo] = [] with mgzip.open(filename=file, mode='w' if overwrite else 'x') as gz, tarfile.open( fileobj=gz, mode='w', bufsize=tarfile.RECORDSIZE * 4) as tar: # Fetch and process the file tree, using that as the basis for all other paths that need to be bundled. file_tree_path = f'file_tree/{firmware_hash}.jsonl' with CodeTimer('Read firmware file tree from object storage'): try: file_tree_result = fetch_object( bucket=self.firmware_metadata_bucket, key=file_tree_path) except ClientError as e: raise Exception( 'Firmware file tree could not be read') from e with CodeTimer('Extract file hashes from file tree'): try: file_hashes = extract_file_hashes(file_tree_result.payload) except json.JSONDecodeError as e: raise Exception( 'Firmware file tree could not be parsed') from e if not file_hashes: raise Exception('Firmware file tree is empty') file_tree_in_bundle = False if is_dataset_in_shard( dataset=FILE_TREE_DATASET, shard_spec=shard_spec ) and file_tree_result.info.etag != delta_to.get(file_tree_path): with CodeTimer('Add file tree to bundle'): add_to_tarfile(tar, file_tree_result) contents.append(file_tree_result.info) file_tree_in_bundle = True else: logger.info( 'File tree is unchanged or not part of this shard and will not be included in the bundle' ) file_tree_size = file_tree_result.info.size logger.info( 'File tree num distinct file hashes = {hash_count}; size = {size}' .format(hash_count=len(file_hashes), size=naturalsize(file_tree_size))) if shard_spec: with CodeTimer( f'Limiting file hashes to shard {shard_spec.index}'): def is_in_shard(file_hash: str) -> bool: return int(file_hash, 16) % shard_spec.count == shard_spec.index file_hashes = [ file_hash for file_hash in file_hashes if is_in_shard(file_hash) ] logger.info(f'Sharded num file hashes = {len(file_hashes)}') file_tree_result = None # Build paths to be bundled with CodeTimer('Build paths for bundle'): bundle_datasets = [ ds for ds in datasets if is_dataset_in_shard(dataset=ds, shard_spec=shard_spec) ] paths = self.build_paths(firmware_hash=firmware_hash, datasets=bundle_datasets, file_hashes=file_hashes, delta_to=delta_to) or [] path_count = len(paths) # Validate the paths (check for duplicates) with CodeTimer('Validate paths'): duplicates = [ path for path, count in collections.Counter(paths).items() if count > 1 ] if duplicates: raise Exception( f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}' ) total_path_count = path_count + 1 if file_tree_in_bundle else 0 logger.info( f'Bundle will include at most {total_path_count} paths from object storage' ) fetch_count = 0 miss_count = 0 skip_count = 0 fetch_bytes = 0 with CodeTimer('Bundle objects'): with concurrent.futures.ThreadPoolExecutor( max_workers=self.max_workers) as executor: fetch_start = datetime.datetime.now() with CodeTimer( 'Submit object storage path retrieval tasks'): # Randomize path ordering to improve the performance of fetches from object storage, # so that a diversity of object key prefixes is being fetched at any one time. random.shuffle(paths) futures = [ executor.submit( fetch_object, bucket=self.firmware_metadata_bucket, key=path, compare_etag=delta_to.get(path), ) for path in paths ] for future in concurrent.futures.as_completed(futures): try: result = future.result() if result: add_to_tarfile(tar, result) result.payload = None fetch_count += 1 fetch_bytes += result.info.size contents.append(result.info) else: skip_count += 1 if fetch_count % 1000 == 0: logger.info( 'Bundled {} objects ({}) in {}'.format( fetch_count, naturalsize(fetch_bytes), naturaldelta(datetime.datetime.now() - fetch_start))) except ClientError as e: error_code = e.response.get('Error', {}).get('Code') if 'NoSuchKey' in error_code: miss_count += 1 elif '304' == error_code: # The ETag on this object was not modified, so it was not returned skip_count += 1 else: raise e if skip_count: logger.info(f'Skipped {skip_count} unmodified objects') if miss_count: logger.info( f"Made {miss_count} attempts to access object storage paths that didn't exist" ) logger.info('Bundled {} objects ({})'.format( fetch_count + (1 if file_tree_in_bundle else 0), naturalsize(fetch_bytes + (file_tree_size if file_tree_in_bundle else 0)))) # Validate fetched paths (check that each path was uniquely processed) with CodeTimer('Validate fetched paths'): fetched_path_counter = collections.Counter( [obj.path for obj in contents]) duplicates = [ path for path, count in fetched_path_counter.items() if count > 1 ] if duplicates: raise Exception( f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}' ) with CodeTimer('Finalize output'): contents = sorted(contents, key=lambda obj: obj.path) return contents
def __exit__(self, ty, val, tb): """ Stop the timer and display a logging message with elapsed time. """ LOGGER.info("%s started %s, ran in %s.", self.name, self.dtstart, naturaldelta(time.time() - self.start)) return False
def date_published_delta(self): with django_language(): return _(u'{0} ago').format(naturaldelta(self.date_published))
def naturaldelta(*args, **kwargs): """ Wrap `humanize.naturaldelta` into django_language(). """ with django_language(): return humanize_time.naturaldelta(*args, **kwargs)
def naturaldelta(*args, **kwargs): with django_language(): return humanize_time.naturaldelta(*args, **kwargs)
def naturalinterval(secs): string = naturaldelta(secs) string = string.replace('a ', '').replace("an ", "") return "every %s" % string
def time_str(time): if time < sys.maxsize: return naturaldelta(timedelta(seconds=time), minimum_unit="Microseconds") else: return f'{ number_str( time // 3600 // 24 // 365 ) } years'
def nd_nomonths(d): return time.naturaldelta(d, months=False)
def unixnaturaldelta(value): return naturaldelta(datetime.fromtimestamp(value))