def generate_file(self, agency_code=None): """ Generates a file based on the FileGeneration object and updates any Jobs referencing it """ raw_filename = (GEN_FILENAMES[self.file_type] if not self.file_generation else GEN_FILENAMES[self.file_type].format( self.file_generation.agency_type)) file_name = S3Handler.get_timestamped_filename(raw_filename) if self.is_local: file_path = "".join([CONFIG_BROKER['broker_files'], file_name]) else: file_path = "".join(["None/", file_name]) # Generate the file and upload to S3 log_data = { 'message': 'Finished file {} generation'.format(self.file_type), 'message_type': 'ValidatorInfo', 'file_type': self.file_type, 'file_path': file_path } if self.file_generation: self.generate_d_file(file_path) log_data.update({ 'agency_code': self.file_generation.agency_code, 'agency_type': self.file_generation.agency_type, 'start_date': self.file_generation.start_date, 'end_date': self.file_generation.end_date, 'file_generation_id': self.file_generation.file_generation_id }) elif self.job.file_type.letter_name in ['A', 'E', 'F']: log_data['job_id'] = self.job.job_id mark_job_status(self.job.job_id, 'running') if self.job.file_type.letter_name == 'A': if not agency_code: raise ResponseException( 'Agency code not provided for an A file generation') self.generate_a_file(agency_code, file_path) else: # Call self.generate_%s_file() where %s is e or f based on the Job's file_type file_type_lower = self.job.file_type.letter_name.lower() getattr(self, 'generate_%s_file' % file_type_lower)() mark_job_status(self.job.job_id, 'finished') else: e = 'No FileGeneration object for D file generation.' if self.file_type in ['D1', 'D2'] else \ 'Cannot generate file for {} file type.'.format(self.file_type if self.file_type else 'empty') raise ResponseException(e) logger.info(log_data)
def generate_from_job(self): """ Generates a file for a specified job """ # Mark Job as running mark_job_status(self.job.job_id, 'running') # Ensure this is a file generation job job_type = self.job.job_type.name if job_type != 'file_upload': raise ResponseException( 'Job ID {} is not a file generation job (job type is {})'.format(self.job.job_id, job_type), StatusCode.CLIENT_ERROR, None, ValidationError.jobError) # Ensure there is an available agency_code if not self.agency_code: raise ResponseException( 'An agency_code must be provided to generate a file'.format(self.job.job_id, job_type), StatusCode.CLIENT_ERROR, None, ValidationError.jobError) # Retrieve any FileRequest that may have started since the Broker sent the request to SQS skip_generation = None if self.job.file_type.letter_name in ['D1', 'D2']: skip_generation = retrieve_cached_file_request(self.job, self.agency_type, self.agency_code, self.is_local) if not skip_generation: # Generate timestamped file names raw_filename = CONFIG_BROKER["".join([str(self.job.file_type.name), "_file_name"])] self.job.original_filename = S3Handler.get_timestamped_filename(raw_filename) if self.is_local: self.job.filename = "".join([CONFIG_BROKER['broker_files'], self.job.original_filename]) else: self.job.filename = "".join([str(self.job.submission_id), "/", self.job.original_filename]) self.sess.commit() # Generate the file, and upload to S3 if self.job.file_type.letter_name in ['D1', 'D2']: # Update the validation Job if necessary update_validation_job_info(self.sess, self.job) self.generate_d_file() elif self.job.file_type.letter_name == 'A': self.generate_a_file() elif self.job.file_type.letter_name == 'E': self.generate_e_file() else: self.generate_f_file() mark_job_status(self.job.job_id, 'finished') logger.info({ 'message': 'Finished file {} generation'.format(self.job.file_type.letter_name), 'message_type': 'ValidatorInfo', 'job_id': self.job.job_id, 'agency_code': self.agency_code, 'file_type': self.job.file_type.letter_name, 'start_date': self.job.start_date, 'end_date': self.job.end_date, 'filename': self.job.original_filename })
def generate_from_job(self, job_id, agency_code): """ Generates a file for a specified job Args: job_id: ID of the upload Job agency_code: FREC or CGAC code to generate data from """ mark_job_status(job_id, 'running') with job_context(job_id, self.is_local) as context: sess, job = context # Ensure this is a file generation job if job.job_type.name != 'file_upload': raise ResponseException( 'Job ID {} is not a file generation job (job type is {})'. format(job.job_id, job.job_type.name), StatusCode.CLIENT_ERROR, None, ValidationError.jobError) # Ensure there is an available agency_code if not agency_code: if job.submission_id: agency_code = job.submission.frec_code if job.submission.frec_code else job.submission.cgac_code else: raise ResponseException( 'An agency_code must be provided to generate a file'. format(job.job_id, job.job_type.name), StatusCode.CLIENT_ERROR, None, ValidationError.jobError) # Generate timestamped file names old_filename = job.original_filename job.original_filename = S3Handler.get_timestamped_filename( CONFIG_BROKER["".join([str(job.file_type.name), "_file_name"])]) if self.is_local: job.filename = "".join( [CONFIG_BROKER['broker_files'], job.original_filename]) else: job.filename = "".join( [str(job.submission_id), "/", job.original_filename]) # Generate the file and upload to S3 if job.file_type.letter_name in ['D1', 'D2']: # Update the validation Job if necessary if job.submission_id: self.update_validation_job_info(job) generate_d_file(sess, job, agency_code, self.is_local, old_filename) elif job.file_type.letter_name == 'E': generate_e_file(sess, job, self.is_local) else: generate_f_file(sess, job, self.is_local)
def generate_file(self, agency_code=None): """ Generates a file based on the FileGeneration object and updates any Jobs referencing it """ raw_filename = (GEN_FILENAMES[self.file_type] if not self.file_generation else GEN_FILENAMES[self.file_type].format(self.file_generation.agency_type)) file_name = S3Handler.get_timestamped_filename(raw_filename) if self.is_local: file_path = "".join([CONFIG_BROKER['broker_files'], file_name]) else: file_path = "".join(["None/", file_name]) # Generate the file and upload to S3 log_data = {'message': 'Finished file {} generation'.format(self.file_type), 'message_type': 'ValidatorInfo', 'file_type': self.file_type, 'file_path': file_path} if self.file_generation: self.generate_d_file(file_path) log_data.update({ 'agency_code': self.file_generation.agency_code, 'agency_type': self.file_generation.agency_type, 'start_date': self.file_generation.start_date, 'end_date': self.file_generation.end_date, 'file_generation_id': self.file_generation.file_generation_id }) elif self.job.file_type.letter_name in ['A', 'E', 'F']: log_data['job_id'] = self.job.job_id mark_job_status(self.job.job_id, 'running') if self.job.file_type.letter_name == 'A': if not agency_code: raise ResponseException('Agency code not provided for an A file generation') self.generate_a_file(agency_code, file_path) else: # Call self.generate_%s_file() where %s is e or f based on the Job's file_type file_type_lower = self.job.file_type.letter_name.lower() getattr(self, 'generate_%s_file' % file_type_lower)() mark_job_status(self.job.job_id, 'finished') else: e = 'No FileGeneration object for D file generation.' if self.file_type in ['D1', 'D2'] else \ 'Cannot generate file for {} file type.'.format(self.file_type if self.file_type else 'empty') raise ResponseException(e) logger.info(log_data)