def _upload(job_context: Dict) -> Dict: """ Uploads the result file to S3 and notifies user. """ # There has been a failure already, don't try to upload anything. if not job_context.get("output_file", None): logger.error( "Was told to upload a smash result without an output_file.") return job_context try: if job_context.get("upload", True) and settings.RUNNING_IN_CLOUD: s3_client = boto3.client('s3') # Note that file expiry is handled by the S3 object lifecycle, # managed by terraform. s3_client.upload_file(job_context["output_file"], RESULTS_BUCKET, job_context["output_file"].split('/')[-1], ExtraArgs={'ACL': 'public-read'}) result_url = ("https://s3.amazonaws.com/" + RESULTS_BUCKET + "/" + job_context["output_file"].split('/')[-1]) job_context["result_url"] = result_url logger.debug("Result uploaded!", result_url=job_context["result_url"]) job_context["dataset"].s3_bucket = RESULTS_BUCKET job_context["dataset"].s3_key = job_context["output_file"].split( '/')[-1] job_context["dataset"].size_in_bytes = calculate_file_size( job_context["output_file"]) job_context["dataset"].sha1 = calculate_sha1( job_context["output_file"]) job_context["dataset"].save() # File is uploaded, we can delete the local. try: os.remove(job_context["output_file"]) except OSError: pass except Exception as e: logger.exception("Failed to upload smash result file.", file=job_context["output_file"]) job_context['job'].success = False job_context['job'].failure_reason = "Failure reason: " + str(e) # Delay failing this pipeline until the failure notify has been sent # job_context['success'] = False return job_context
def _update_result_objects(job_context: Dict) -> Dict: """Closes out the dataset object.""" dataset = job_context["dataset"] dataset.s3_bucket = RESULTS_BUCKET dataset.s3_key = job_context["output_file"].split("/")[-1] dataset.size_in_bytes = calculate_file_size(job_context["output_file"]) dataset.sha1 = calculate_sha1(job_context["output_file"]) dataset.is_processing = False dataset.is_processed = True dataset.is_available = True dataset.expires_on = timezone.now() + timedelta(days=7) dataset.save() if settings.RUNNING_IN_CLOUD and job_context.get("upload", True): # File is uploaded and the metadata is updated, can delete the local. try: os.remove(job_context["output_file"]) except OSError: pass job_context["success"] = True return job_context
def calculate_size(self) -> None: """ Calculate the number of bytes in a given file. """ self.size_in_bytes = calculate_file_size(self.absolute_file_path) return self.size_in_bytes