def get_storage(output):
manifest_path = output.output_dir / 'manifest.outputs.json'
if not manifest_path.exists():
if not output.output_dir.exists():
return 0
else:
manifest = save_outputs_manifest(output.output_dir)
else:
with manifest_path.open() as f:
try:
manifest = json.load(f)
except: # hello corrupted file
manifest = save_outputs_manifest(output.output_dir)
return total_storage(manifest)
def update_manifest(self, compute_hashes=True):
qatools_config = self.batch.ci_commit.project.data.get(
'qatools_config', {})
return save_outputs_manifest(self.output_dir,
config=qatools_config,
compute_hashes=compute_hashes)
print("Starting updates")
most_recent = datetime.datetime.now() - datetime.timedelta(days=7)
for idx, output in enumerate(query(most_recent)):
if output.is_pending:
continue
if output.data.get('storage'):
continue
manifest_path = output.output_dir / 'manifest.outputs.json'
if not manifest_path.exists():
if not output.output_dir.exists():
output.data['storage'] = 0
db_session.add(output)
flag_modified(output, "data")
continue
else:
manifest = save_outputs_manifest(output.output_dir)
else:
with manifest_path.open() as f:
manifest = json.load(f)
output.data['storage'] = total_storage(manifest)
flag_modified(output, "data")
db_session.add(output)
if idx % batch_size == 0:
print(f"{idx/output_total:.1%}", output, f"{output.data['storage']/1024/1024:.01f} MB")
db_session.commit()