def generate_e_file(self): """ Write file E to an appropriate CSV. """ log_data = {'message': 'Starting file E generation', 'message_type': 'ValidatorInfo', 'job_id': self.job.job_id, 'submission_id': self.job.submission_id, 'file_type': 'executive_compensation'} logger.info(log_data) d1 = self.sess.query(AwardProcurement.awardee_or_recipient_uniqu).\ filter(AwardProcurement.submission_id == self.job.submission_id).\ distinct() d2 = self.sess.query(AwardFinancialAssistance.awardee_or_recipient_uniqu).\ filter(AwardFinancialAssistance.submission_id == self.job.submission_id).\ distinct() duns_set = {r.awardee_or_recipient_uniqu for r in d1.union(d2)} duns_list = list(duns_set) # get an order rows = [] for i in range(0, len(duns_list), 100): rows.extend(fileE.retrieve_rows(duns_list[i:i + 100])) # Add rows to database here. # TODO: This is a temporary solution until loading from SAM's SFTP has been resolved for row in rows: self.sess.merge(ExecutiveCompensation(**fileE.row_to_dict(row))) self.sess.commit() log_data['message'] = 'Writing E file CSV: {}'.format(self.job.original_filename) logger.info(log_data) write_csv(self.job.original_filename, self.job.filename, self.is_local, fileE.Row._fields, rows) log_data['message'] = 'Finished writing E file CSV: {}'.format(self.job.original_filename) logger.info(log_data)
def generate_f_file(sess, job, is_local): """Write rows from fileF.generate_f_rows to an appropriate CSV. Args: sess: database session job: upload Job is_local: True if in local development, False otherwise """ log_data = { 'message': 'Starting file F generation', 'message_type': 'ValidatorInfo', 'job_id': job.job_id, 'submission_id': job.submission_id, 'file_type': 'sub_award' } logger.info(log_data) rows_of_dicts = fileF.generate_f_rows(job.submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) log_data['message'] = 'Writing file F CSV' logger.info(log_data) write_csv(job.original_filename, job.filename, is_local, header, body) log_data['message'] = 'Finished file F generation' logger.info(log_data)
def generate_e_file(self): """ Write file E to an appropriate CSV. """ log_data = {'message': 'Starting file E generation', 'message_type': 'ValidatorInfo', 'job_id': self.job.job_id, 'submission_id': self.job.submission_id, 'file_type': 'executive_compensation'} logger.info(log_data) d1 = self.sess.query(AwardProcurement.awardee_or_recipient_uniqu).\ filter(AwardProcurement.submission_id == self.job.submission_id).\ distinct() d2 = self.sess.query(AwardFinancialAssistance.awardee_or_recipient_uniqu).\ filter(AwardFinancialAssistance.submission_id == self.job.submission_id).\ distinct() duns_set = {r.awardee_or_recipient_uniqu for r in d1.union(d2)} duns_list = list(duns_set) # get an order rows = [] for i in range(0, len(duns_list), 100): rows.extend(fileE.retrieve_rows(duns_list[i:i + 100])) # Add rows to database here. # TODO: This is a temporary solution until loading from SAM's SFTP has been resolved for row in rows: self.sess.merge(ExecutiveCompensation(**fileE.row_to_dict(row))) self.sess.commit() log_data['message'] = 'Writing file E CSV' logger.info(log_data) write_csv(self.job.original_filename, self.job.filename, self.is_local, fileE.Row._fields, rows)
def generate_e_file(submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write file E to an appropriate CSV.""" with job_context(job_id) as session: d1 = session.\ query(AwardProcurement.awardee_or_recipient_uniqu).\ filter(AwardProcurement.submission_id == submission_id).\ distinct() d2 = session.\ query(AwardFinancialAssistance.awardee_or_recipient_uniqu).\ filter(AwardFinancialAssistance.submission_id == submission_id).\ distinct() duns_set = {r.awardee_or_recipient_uniqu for r in d1.union(d2)} duns_list = list(duns_set) # get an order rows = [] for i in range(0, len(duns_list), 100): rows.extend(fileE.retrieve_rows(duns_list[i:i + 100])) # Add rows to database here. # TODO: This is a temporary solution until loading from SAM's SFTP has been resolved for row in rows: session.merge(ExecutiveCompensation(**fileE.row_to_dict(row))) session.commit() write_csv(timestamped_name, upload_file_name, is_local, fileE.Row._fields, rows)
def generate_f_file(submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write rows from fileF.generate_f_rows to an appropriate CSV. Args: submission_id - Submission ID for generation job_id - Job ID for upload job timestamped_name - Version of filename without user ID upload_file_name - Filename to use on S3 is_local - True if in local development, False otherwise """ log_data = { 'message': 'Starting file F generation', 'message_type': 'BrokerInfo', 'submission_id': submission_id, 'job_id': job_id, 'file_type': 'sub_award' } logger.info(log_data) with job_context(job_id): rows_of_dicts = fileF.generate_f_rows(submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) log_data['message'] = 'Writing file F CSV' logger.info(log_data) write_csv(timestamped_name, upload_file_name, is_local, header, body) log_data['message'] = 'Finished file F generation' logger.info(log_data)
def generate_f_file(task, submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write rows from fileF.generateFRows to an appropriate CSV.""" with job_context(task, job_id) as session: rows_of_dicts = fileF.generateFRows(session, submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) write_csv(timestamped_name, upload_file_name, is_local, header, body)
def generate_f_file(task, submission_id, job_id, interface_holder_class, timestamped_name, upload_file_name, is_local): """Write rows from fileF.generateFRows to an appropriate CSV. Here the third parameter, interface_holder_class, is a bit of a hack. Importing InterfaceHolder directly causes cyclic dependency woes, so we're passing in a class""" with job_context(task, interface_holder_class, job_id) as job_manager: rows_of_dicts = fileF.generateFRows(job_manager.session, submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) write_csv(timestamped_name, upload_file_name, is_local, header, body)
def generate_f_file(self): """ Write rows from fileF.generate_f_rows to an appropriate CSV. """ log_data = {'message': 'Starting file F generation', 'message_type': 'ValidatorInfo', 'job_id': self.job.job_id, 'submission_id': self.job.submission_id, 'file_type': 'sub_award'} logger.info(log_data) rows_of_dicts = fileF.generate_f_rows(self.job.submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) log_data['message'] = 'Writing file F CSV' logger.info(log_data) write_csv(self.job.original_filename, self.job.filename, self.is_local, header, body)
def load_d_file(self, url, upload_name, timestamped_name, job_id, isLocal): """ Pull D file from specified URL and write to S3 """ job_manager = self.interfaces.jobDb try: full_file_path = "".join( [CONFIG_BROKER['d_file_storage_path'], timestamped_name]) CloudLogger.log("DEBUG: Downloading file...", log_type="debug", file_name=self.smx_log_file_name) if not self.download_file(full_file_path, url): # Error occurred while downloading file, mark job as failed and record error message job_manager.markJobStatus(job_id, "failed") job = job_manager.getJobById(job_id) file_type = job_manager.getFileType(job_id) if file_type == "award": source = "ASP" elif file_type == "award_procurement": source = "FPDS" else: source = "unknown source" job.error_message = "A problem occurred receiving data from {}".format( source) raise ResponseException(job.error_message, StatusCode.CLIENT_ERROR) lines = self.get_lines_from_csv(full_file_path) write_csv(timestamped_name, upload_name, isLocal, lines[0], lines[1:]) CloudLogger.log("DEBUG: Marking job id of " + str(job_id) + " as finished", log_type="debug", file_name=self.smx_log_file_name) job_manager.markJobStatus(job_id, "finished") return {"message": "Success", "file_name": timestamped_name} except Exception as e: CloudLogger.log("ERROR: Exception caught => " + str(e), log_type="debug", file_name=self.smx_log_file_name) # Log the error JsonResponse.error(e, 500) job_manager.getJobById(job_id).error_message = str(e) job_manager.markJobStatus(job_id, "failed") job_manager.session.commit() raise e
def generate_e_file(submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write file E to an appropriate CSV. Args: submission_id - Submission ID for generation job_id - Job ID for upload job timestamped_name - Version of filename without user ID upload_file_name - Filename to use on S3 is_local - True if in local development, False otherwise """ log_data = { 'message': 'Starting file E generation', 'message_type': 'BrokerInfo', 'submission_id': submission_id, 'job_id': job_id, 'file_type': 'executive_compensation' } logger.info(log_data) with job_context(job_id) as session: d1 = session.query(AwardProcurement.awardee_or_recipient_uniqu).\ filter(AwardProcurement.submission_id == submission_id).\ distinct() d2 = session.query(AwardFinancialAssistance.awardee_or_recipient_uniqu).\ filter(AwardFinancialAssistance.submission_id == submission_id).\ distinct() duns_set = {r.awardee_or_recipient_uniqu for r in d1.union(d2)} duns_list = list(duns_set) # get an order rows = [] for i in range(0, len(duns_list), 100): rows.extend(fileE.retrieve_rows(duns_list[i:i + 100])) # Add rows to database here. # TODO: This is a temporary solution until loading from SAM's SFTP has been resolved for row in rows: session.merge(ExecutiveCompensation(**fileE.row_to_dict(row))) session.commit() log_data['message'] = 'Writing file E CSV' logger.info(log_data) write_csv(timestamped_name, upload_file_name, is_local, fileE.Row._fields, rows) log_data['message'] = 'Finished file E generation' logger.info(log_data)
def generate_f_file(submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write rows from fileF.generate_f_rows to an appropriate CSV.""" logger.debug('Starting file F generation') with job_context(job_id): logger.debug('Calling genearte_f_rows') rows_of_dicts = fileF.generate_f_rows(submission_id) header = [key for key in fileF.mappings] # keep order body = [] for row in rows_of_dicts: body.append([row[key] for key in header]) logger.debug('Writing file F CSV') write_csv(timestamped_name, upload_file_name, is_local, header, body) logger.debug('Finished file F generation')
def generate_e_file(task, submission_id, job_id, timestamped_name, upload_file_name, is_local): """Write file E to an appropriate CSV.""" with job_context(task, job_id) as session: d1 = session.\ query(AwardProcurement.awardee_or_recipient_uniqu).\ filter(AwardProcurement.submission_id == submission_id).\ distinct() d2 = session.\ query(AwardFinancialAssistance.awardee_or_recipient_uniqu).\ filter(AwardFinancialAssistance.submission_id == submission_id).\ distinct() dunsSet = {r.awardee_or_recipient_uniqu for r in d1.union(d2)} dunsList = list(dunsSet) # get an order rows = [] for i in range(0, len(dunsList), 100): rows.extend(fileE.retrieveRows(dunsList[i:i + 100])) write_csv(timestamped_name, upload_file_name, is_local, fileE.Row._fields, rows)