def humanize_naturaldelta(eval_ctx,
value,
months=True,
minimum_unit="seconds",
when=None):
return naturaldelta(value,
months=months,
minimum_unit=minimum_unit,
when=when)
def humanized_next_due_in(self):
'''
Can be appended to the prefix "Next review ...".
'''
if not self._review_applied:
raise Exception("Must first apply the review.")
due_in = self.card.due_at - datetime.utcnow()
if due_in < timedelta(minutes=1):
return "in less than a minute"
return "in {}".format(naturaldelta(due_in, months=True))
def humanized_next_due_in(self):
'''
Can be appended to the prefix "Next review ...".
'''
if not self._review_applied:
raise Exception("Must first apply the review.")
due_in = self.card.due_at - datetime.utcnow()
if due_in < timedelta(minutes=1):
return "in less than a minute"
return u"in {}".format(naturaldelta(due_in, months=True))
def my_view(request):
up_since = index["up_since"]
up_time = datetime.now() - up_since
cache_stats = cache.stats()
return {
'entries': cache.cache_len(),
'cache_disk_size': filesize.naturalsize(cache.cache_disk_size()),
'up_date': time.naturaldate(up_since),
'up_duration': time.naturaldelta(up_time),
'cache_hits': str(cache_stats['hit']),
'cache_misses': str(cache_stats['misses'])
}
def test_naturaldelta_nomonths(self):
now = datetime.now()
test_list = [
timedelta(days=7),
timedelta(days=31),
timedelta(days=230),
timedelta(days=400),
]
result_list = [
'7 days',
'31 days',
'230 days',
'1 year, 35 days',
]
with patch('humanize.time._now') as mocked:
mocked.return_value = now
nd_nomonths = lambda d: time.naturaldelta(d, months=False)
self.assertManyResults(nd_nomonths, test_list, result_list)
def naturalinterval(secs):
string = naturaldelta(secs)
string = string.replace("a ", "").replace("an ", "")
return "every %s" % string
def bundle(self,
firmware_hash: str,
datasets: List[Dataset],
*,
file: Union[str, BinaryIO, IO[bytes]],
shard_spec: ShardSpec = None,
delta_to: Dict[str, str] = None,
overwrite: bool = False) -> List[ObjectInfo]:
"""
Builds a data bundle (*.tar.gz) for a firmware hash, including content from the
specified datasets (FWAN plugin output locations)
:param firmware_hash: The firmware hash to bundle
:param datasets: The datasets to include in the bundle
:param file: The output path or file-like-object to which the *.tar.gz output should be written
:param shard_spec: If provided, only the specified shard of file hashes will appear in the bundle
:param delta_to: A dictionary of path->etag values, which if supplied will cause the bundle to be built as a
delta to that set, meaning only new objects or objects with modified etags will appear in the bundle.
:param overwrite: The output path will not be overwritten, to prevent accidental data loss, unless this is set
:return: A list of the object included in the bundle
"""
if not firmware_hash:
raise ValueError('firmware_hash must be specified')
if not datasets:
raise ValueError('datasets must be specified, and non-empty')
if delta_to is None:
delta_to = {}
logger.info(
f"Building {'delta' if delta_to else ''} bundle for {firmware_hash}"
)
contents: List[ObjectInfo] = []
with mgzip.open(filename=file,
mode='w' if overwrite else 'x') as gz, tarfile.open(
fileobj=gz,
mode='w',
bufsize=tarfile.RECORDSIZE * 4) as tar:
# Fetch and process the file tree, using that as the basis for all other paths that need to be bundled.
file_tree_path = f'file_tree/{firmware_hash}.jsonl'
with CodeTimer('Read firmware file tree from object storage'):
try:
file_tree_result = fetch_object(
bucket=self.firmware_metadata_bucket,
key=file_tree_path)
except ClientError as e:
raise Exception(
'Firmware file tree could not be read') from e
with CodeTimer('Extract file hashes from file tree'):
try:
file_hashes = extract_file_hashes(file_tree_result.payload)
except json.JSONDecodeError as e:
raise Exception(
'Firmware file tree could not be parsed') from e
if not file_hashes:
raise Exception('Firmware file tree is empty')
file_tree_in_bundle = False
if is_dataset_in_shard(
dataset=FILE_TREE_DATASET, shard_spec=shard_spec
) and file_tree_result.info.etag != delta_to.get(file_tree_path):
with CodeTimer('Add file tree to bundle'):
add_to_tarfile(tar, file_tree_result)
contents.append(file_tree_result.info)
file_tree_in_bundle = True
else:
logger.info(
'File tree is unchanged or not part of this shard and will not be included in the bundle'
)
file_tree_size = file_tree_result.info.size
logger.info(
'File tree num distinct file hashes = {hash_count}; size = {size}'
.format(hash_count=len(file_hashes),
size=naturalsize(file_tree_size)))
if shard_spec:
with CodeTimer(
f'Limiting file hashes to shard {shard_spec.index}'):
def is_in_shard(file_hash: str) -> bool:
return int(file_hash,
16) % shard_spec.count == shard_spec.index
file_hashes = [
file_hash for file_hash in file_hashes
if is_in_shard(file_hash)
]
logger.info(f'Sharded num file hashes = {len(file_hashes)}')
file_tree_result = None
# Build paths to be bundled
with CodeTimer('Build paths for bundle'):
bundle_datasets = [
ds for ds in datasets
if is_dataset_in_shard(dataset=ds, shard_spec=shard_spec)
]
paths = self.build_paths(firmware_hash=firmware_hash,
datasets=bundle_datasets,
file_hashes=file_hashes,
delta_to=delta_to) or []
path_count = len(paths)
# Validate the paths (check for duplicates)
with CodeTimer('Validate paths'):
duplicates = [
path for path, count in collections.Counter(paths).items()
if count > 1
]
if duplicates:
raise Exception(
f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}'
)
total_path_count = path_count + 1 if file_tree_in_bundle else 0
logger.info(
f'Bundle will include at most {total_path_count} paths from object storage'
)
fetch_count = 0
miss_count = 0
skip_count = 0
fetch_bytes = 0
with CodeTimer('Bundle objects'):
with concurrent.futures.ThreadPoolExecutor(
max_workers=self.max_workers) as executor:
fetch_start = datetime.datetime.now()
with CodeTimer(
'Submit object storage path retrieval tasks'):
# Randomize path ordering to improve the performance of fetches from object storage,
# so that a diversity of object key prefixes is being fetched at any one time.
random.shuffle(paths)
futures = [
executor.submit(
fetch_object,
bucket=self.firmware_metadata_bucket,
key=path,
compare_etag=delta_to.get(path),
) for path in paths
]
for future in concurrent.futures.as_completed(futures):
try:
result = future.result()
if result:
add_to_tarfile(tar, result)
result.payload = None
fetch_count += 1
fetch_bytes += result.info.size
contents.append(result.info)
else:
skip_count += 1
if fetch_count % 1000 == 0:
logger.info(
'Bundled {} objects ({}) in {}'.format(
fetch_count, naturalsize(fetch_bytes),
naturaldelta(datetime.datetime.now() -
fetch_start)))
except ClientError as e:
error_code = e.response.get('Error',
{}).get('Code')
if 'NoSuchKey' in error_code:
miss_count += 1
elif '304' == error_code:
# The ETag on this object was not modified, so it was not returned
skip_count += 1
else:
raise e
if skip_count:
logger.info(f'Skipped {skip_count} unmodified objects')
if miss_count:
logger.info(
f"Made {miss_count} attempts to access object storage paths that didn't exist"
)
logger.info('Bundled {} objects ({})'.format(
fetch_count + (1 if file_tree_in_bundle else 0),
naturalsize(fetch_bytes +
(file_tree_size if file_tree_in_bundle else 0))))
# Validate fetched paths (check that each path was uniquely processed)
with CodeTimer('Validate fetched paths'):
fetched_path_counter = collections.Counter(
[obj.path for obj in contents])
duplicates = [
path for path, count in fetched_path_counter.items()
if count > 1
]
if duplicates:
raise Exception(
f'Bundle paths contained {len(duplicates)} duplicates: {duplicates}'
)
with CodeTimer('Finalize output'):
contents = sorted(contents, key=lambda obj: obj.path)
return contents
def __exit__(self, ty, val, tb):
""" Stop the timer and display a logging message with elapsed time. """
LOGGER.info("%s started %s, ran in %s.", self.name, self.dtstart,
naturaldelta(time.time() - self.start))
return False
def date_published_delta(self):
with django_language():
return _(u'{0} ago').format(naturaldelta(self.date_published))
def date_published_delta(self):
with django_language():
return _(u'{0} ago').format(naturaldelta(self.date_published))
def naturaldelta(*args, **kwargs):
""" Wrap `humanize.naturaldelta` into django_language(). """
with django_language():
return humanize_time.naturaldelta(*args, **kwargs)
def naturaldelta(*args, **kwargs):
with django_language():
return humanize_time.naturaldelta(*args, **kwargs)
def naturalinterval(secs):
string = naturaldelta(secs)
string = string.replace('a ', '').replace("an ", "")
return "every %s" % string
def time_str(time):
if time < sys.maxsize:
return naturaldelta(timedelta(seconds=time),
minimum_unit="Microseconds")
else:
return f'{ number_str( time // 3600 // 24 // 365 ) } years'
def nd_nomonths(d):
return time.naturaldelta(d, months=False)
def unixnaturaldelta(value):
return naturaldelta(datetime.fromtimestamp(value))
def __exit__(self, ty, val, tb):
""" Stop the timer and display a logging message with elapsed time. """
LOGGER.info("%s started %s, ran in %s.", self.name, self.dtstart,
naturaldelta(time.time() - self.start))
return False
def naturaldelta(*args, **kwargs):
""" Wrap `humanize.naturaldelta` into django_language(). """
with django_language():
return humanize_time.naturaldelta(*args, **kwargs)
