def load_fpds_incrementally(self,
date: Optional[datetime],
chunk_size: int = CHUNK_SIZE) -> None:
"""Process incremental loads based on a date range or full data loads"""
if date is None:
logger.info("Skipping deletes. Fetching all fpds transactions...")
else:
logger.info(f"Handling fpds transactions since {date}...")
stale_awards = delete_stale_fpds(date.date())
self.update_award_records(awards=stale_awards,
skip_cd_linkage=True)
with psycopg2.connect(dsn=get_broker_dsn_string()) as connection:
logger.info("Fetching records to update")
total_records = self.get_cursor_for_date_query(
connection, date, True).fetchall()[0][0]
records_processed = 0
logger.info("{} total records to update".format(total_records))
cursor = self.get_cursor_for_date_query(connection, date)
while True:
id_list = cursor.fetchmany(chunk_size)
if len(id_list) == 0:
break
logger.info(
"Loading batch (size: {}) from date query...".format(
len(id_list)))
self.modified_award_ids.extend(
load_fpds_transactions([row[0] for row in id_list]))
records_processed = records_processed + len(id_list)
logger.info("{} out of {} processed".format(
records_processed, total_records))
def generate_ids_from_broker(self):
sql = self.combine_sql()
with psycopg2.connect(dsn=get_broker_dsn_string()) as connection:
with connection.cursor("usaspending_data_transfer") as cursor:
cursor.execute(sql.strip("\n"))
while True:
id_list = [id[0] for id in cursor.fetchmany(size=self.chunk_size)]
if not id_list:
break
for broker_id in id_list:
yield broker_id