def _upload(job_context: Dict) -> Dict:
""" Uploads the result file to S3 and notifies user. """
# There has been a failure already, don't try to upload anything.
if not job_context.get("output_file", None):
logger.error(
"Was told to upload a smash result without an output_file.")
return job_context
try:
if job_context.get("upload", True) and settings.RUNNING_IN_CLOUD:
s3_client = boto3.client('s3')
# Note that file expiry is handled by the S3 object lifecycle,
# managed by terraform.
s3_client.upload_file(job_context["output_file"],
RESULTS_BUCKET,
job_context["output_file"].split('/')[-1],
ExtraArgs={'ACL': 'public-read'})
result_url = ("https://s3.amazonaws.com/" + RESULTS_BUCKET + "/" +
job_context["output_file"].split('/')[-1])
job_context["result_url"] = result_url
logger.debug("Result uploaded!",
result_url=job_context["result_url"])
job_context["dataset"].s3_bucket = RESULTS_BUCKET
job_context["dataset"].s3_key = job_context["output_file"].split(
'/')[-1]
job_context["dataset"].size_in_bytes = calculate_file_size(
job_context["output_file"])
job_context["dataset"].sha1 = calculate_sha1(
job_context["output_file"])
job_context["dataset"].save()
# File is uploaded, we can delete the local.
try:
os.remove(job_context["output_file"])
except OSError:
pass
except Exception as e:
logger.exception("Failed to upload smash result file.",
file=job_context["output_file"])
job_context['job'].success = False
job_context['job'].failure_reason = "Failure reason: " + str(e)
# Delay failing this pipeline until the failure notify has been sent
# job_context['success'] = False
return job_context
def _update_result_objects(job_context: Dict) -> Dict:
"""Closes out the dataset object."""
dataset = job_context["dataset"]
dataset.s3_bucket = RESULTS_BUCKET
dataset.s3_key = job_context["output_file"].split("/")[-1]
dataset.size_in_bytes = calculate_file_size(job_context["output_file"])
dataset.sha1 = calculate_sha1(job_context["output_file"])
dataset.is_processing = False
dataset.is_processed = True
dataset.is_available = True
dataset.expires_on = timezone.now() + timedelta(days=7)
dataset.save()
if settings.RUNNING_IN_CLOUD and job_context.get("upload", True):
# File is uploaded and the metadata is updated, can delete the local.
try:
os.remove(job_context["output_file"])
except OSError:
pass
job_context["success"] = True
return job_context
def calculate_size(self) -> None:
""" Calculate the number of bytes in a given file.
"""
self.size_in_bytes = calculate_file_size(self.absolute_file_path)
return self.size_in_bytes
