def handle(self, *args, **options): processing_start_datetime = datetime.now(timezone.utc) logger.info("Starting FABS data load script...") # "Reload all" supersedes all other processing options. reload_all = options["reload_all"] if reload_all: ids = None afa_ids = None start_datetime = None end_datetime = None else: ids = options["ids"] afa_ids = set(options["afa_ids"]) if options["afa_id_file"]: afa_ids = tuple( afa_ids | read_afa_ids_from_file(options["afa_id_file"])) start_datetime = options["start_datetime"] end_datetime = options["end_datetime"] # If no other processing options were provided than this is an incremental load. is_incremental_load = not any( (reload_all, ids, afa_ids, start_datetime, end_datetime)) if is_incremental_load: start_datetime = get_incremental_load_start_datetime() logger.info( f"Processing data for FABS starting from {start_datetime} (includes offset)" ) # We only perform deletes with incremental loads. with timer("obtaining delete records", logger.info): delete_records = retrieve_deleted_fabs_transactions( start_datetime, end_datetime) ids_to_delete = [ item for sublist in delete_records.values() for item in sublist if item ] ids_to_delete = get_delete_pks_for_afa_keys(ids_to_delete) logger.info(f"{len(ids_to_delete):,} delete ids found in total") with timer("retrieving IDs of FABS to process", logger.info): ids_to_upsert = get_fabs_transaction_ids(ids, afa_ids, start_datetime, end_datetime) update_award_ids = delete_fabs_transactions( ids_to_delete) if is_incremental_load else [] upsert_fabs_transactions(ids_to_upsert, update_award_ids) if is_incremental_load: logger.info( f"Storing {processing_start_datetime} for the next incremental run" ) update_last_load_date("fabs", processing_start_datetime) logger.info("FABS UPDATE FINISHED!")
def handle(self, *args, **options): processing_start_datetime = datetime.now(timezone.utc) logger.info("Starting FABS data load script...") do_not_log_deletions = options["do_not_log_deletions"] # "Reload all" supersedes all other processing options. reload_all = options["reload_all"] if reload_all: submission_ids = None afa_ids = None start_datetime = None end_datetime = None else: submission_ids = tuple(options["submission_ids"] ) if options["submission_ids"] else None afa_ids = read_afa_ids_from_file( options['afa_id_file']) if options['afa_id_file'] else None start_datetime = options["start_datetime"] end_datetime = options["end_datetime"] # If no other processing options were provided than this is an incremental load. is_incremental_load = not any((reload_all, submission_ids, afa_ids, start_datetime, end_datetime)) if is_incremental_load: last_load_date = get_last_load_date() submission_ids = get_new_submission_ids(last_load_date) logger.info("Processing data for FABS starting from %s" % last_load_date) if is_incremental_load and not submission_ids: logger.info("No new submissions. Exiting.") else: with timer("obtaining delete records", logger.info): ids_to_delete = get_fabs_records_to_delete( submission_ids, afa_ids, start_datetime, end_datetime) with timer("retrieving/diff-ing FABS Data", logger.info): ids_to_upsert = get_fabs_transaction_ids( submission_ids, afa_ids, start_datetime, end_datetime) update_award_ids = delete_fabs_transactions( ids_to_delete, do_not_log_deletions) upsert_fabs_transactions(ids_to_upsert, update_award_ids) if is_incremental_load: update_last_load_date("fabs", processing_start_datetime) logger.info("FABS UPDATE FINISHED!")
def delete_and_add_fabs_transaction_records(): from usaspending_api.broker.helpers.delete_fabs_transactions import delete_fabs_transactions from usaspending_api.broker.helpers.upsert_fabs_transactions import upsert_fabs_transactions with Timer("Insert/delete FABS transactions"): delete_ids = get_ids(TEMP_TRANSACTION_FABS_DELETE_IDS_TABLE) add_ids = get_ids(TEMP_TRANSACTION_FABS_ADD_IDS_TABLE) if not delete_ids and not add_ids: logger.info("No FABS transaction records to add or delete") return update_award_ids = delete_fabs_transactions(delete_ids) upsert_fabs_transactions(add_ids, update_award_ids)
def handle(self, *args, **options): processing_start_datetime = datetime.now(timezone.utc) logger.info("Starting FABS data load script...") # "Reload all" supersedes all other processing options. reload_all = options["reload_all"] if reload_all: afa_ids = None start_datetime = None end_datetime = None else: afa_ids = read_afa_ids_from_file( options["afa_id_file"]) if options["afa_id_file"] else None start_datetime = options["start_datetime"] end_datetime = options["end_datetime"] # If no other processing options were provided than this is an incremental load. is_incremental_load = not any( (reload_all, afa_ids, start_datetime, end_datetime)) if is_incremental_load: start_datetime = get_last_load_date() logger.info("Processing data for FABS starting from %s" % start_datetime) with timer("obtaining delete records", logger.info): delete_records = retrieve_deleted_fabs_transactions( start_datetime, end_datetime) ids_to_delete = [ item for sublist in delete_records.values() for item in sublist if item ] with timer("retrieving/diff-ing FABS Data", logger.info): ids_to_upsert = get_fabs_transaction_ids(afa_ids, start_datetime, end_datetime) update_award_ids = delete_fabs_transactions(ids_to_delete) upsert_fabs_transactions(ids_to_upsert, update_award_ids) if is_incremental_load: update_last_load_date("fabs", processing_start_datetime) logger.info("FABS UPDATE FINISHED!")
def handle(self, *args, **options): processing_start_datetime = datetime.now(timezone.utc) logger.info("Starting FABS data load script...") do_not_log_deletions = options["do_not_log_deletions"] # "Reload all" supersedes all other processing options. reload_all = options["reload_all"] if reload_all: submission_ids = None afa_ids = None start_datetime = None end_datetime = None else: submission_ids = tuple(options["submission_ids"]) if options["submission_ids"] else None afa_ids = read_afa_ids_from_file(options['afa_id_file']) if options['afa_id_file'] else None start_datetime = options["start_datetime"] end_datetime = options["end_datetime"] # If no other processing options were provided than this is an incremental load. is_incremental_load = not any((reload_all, submission_ids, afa_ids, start_datetime, end_datetime)) if is_incremental_load: last_load_date = get_last_load_date() submission_ids = get_new_submission_ids(last_load_date) logger.info("Processing data for FABS starting from %s" % last_load_date) if is_incremental_load and not submission_ids: logger.info("No new submissions. Exiting.") else: with timer("obtaining delete records", logger.info): ids_to_delete = get_fabs_records_to_delete(submission_ids, afa_ids, start_datetime, end_datetime) with timer("retrieving/diff-ing FABS Data", logger.info): ids_to_upsert = get_fabs_transaction_ids(submission_ids, afa_ids, start_datetime, end_datetime) update_award_ids = delete_fabs_transactions(ids_to_delete, do_not_log_deletions) upsert_fabs_transactions(ids_to_upsert, update_award_ids) if is_incremental_load: update_last_load_date("fabs", processing_start_datetime) logger.info("FABS UPDATE FINISHED!")