def retrieve_deleted_transactions(
regex: str,
date_fmt: str,
start_datetime: datetime,
end_datetime: Optional[datetime] = None) -> dict:
with Timer("Obtaining S3 Object list"):
logger.info(f"file regex pattern: {regex}")
logger.info(f"date format: {date_fmt}")
logger.info(f"start date: {start_datetime}")
logger.info(f"end date: {end_datetime}")
logger.info(
f"S3 bucket name: {settings.DELETED_TRANSACTION_JOURNAL_FILES}")
objects = access_s3_object_list(
settings.DELETED_TRANSACTION_JOURNAL_FILES, regex_pattern=regex)
if objects is None:
raise Exception("Problem accessing S3 for deleted records")
objects = limit_objects_to_date_range(objects, date_fmt,
start_datetime, end_datetime)
logger.info(f"{len(objects):,} files found in date range.")
deleted_records = defaultdict(list)
for obj in objects:
with Timer(f"Read delete ids from {obj.key}"):
object_data = access_s3_object(
settings.DELETED_TRANSACTION_JOURNAL_FILES, obj)
reader = csv.reader(
object_data.read().decode("utf-8").splitlines())
next(reader) # skip the header
transaction_id_list = [rows[0] for rows in reader]
logger.info(f"{len(transaction_id_list):,} delete ids found")
if transaction_id_list:
file_date = obj.key[:obj.key.find("_")]
deleted_records[file_date].extend(transaction_id_list)
return deleted_records
def retrieve_deleted_transactions(
regex: str,
start_datetime: datetime,
end_datetime: Optional[datetime] = None) -> dict:
with Timer("Obtaining S3 Object list"):
objects = retrieve_s3_bucket_object_list(
settings.DELETED_TRANSACTION_JOURNAL_FILES)
logger.info(
f"{len(objects):,} files found in bucket '{settings.DELETED_TRANSACTION_JOURNAL_FILES}'."
)
objects = [
o for o in objects if re.fullmatch(regex, o.key) is not None
]
logger.info(f"{len(objects):,} files match file pattern '{regex}'.")
objects = limit_objects_to_date_range(objects, regex, start_datetime,
end_datetime)
logger.info(
f"{len(objects):,} files found in date range {start_datetime} through {end_datetime or 'the end of time'}."
)
deleted_records = defaultdict(list)
for obj in objects:
object_data = access_s3_object(
settings.DELETED_TRANSACTION_JOURNAL_FILES, obj)
reader = csv.reader(object_data.read().decode("utf-8").splitlines())
next(reader) # skip the header
transaction_id_list = [rows[0] for rows in reader]
logger.info(
f"{len(transaction_id_list):,} delete ids found in {obj.key}")
if transaction_id_list:
file_date = obj.key[:obj.key.find("_")]
deleted_records[file_date].extend(transaction_id_list)
return deleted_records
def gather_deleted_ids(config):
"""
Connect to S3 and gather all of the transaction ids stored in CSV files
generated by the broker when transactions are removed from the DB.
"""
if not config["process_deletes"]:
logger.info(
format_log(f"Skipping the S3 CSV fetch for deleted transactions",
process="ES Delete"))
return
logger.info(
format_log(f"Gathering all deleted transactions from S3",
process="ES Delete"))
start = perf_counter()
bucket_objects = retrieve_s3_bucket_object_list(
bucket_name=config["s3_bucket"])
logger.info(
format_log(
f"{len(bucket_objects):,} files found in bucket '{config['s3_bucket']}'",
process="ES Delete"))
if config["verbose"]:
logger.info(
format_log(f"CSV data from {config['starting_date']} to now",
process="ES Delete"))
filtered_csv_list = [
x for x in bucket_objects
if (x.key.endswith(".csv") and not x.key.startswith("staging")
and x.last_modified >= config["starting_date"])
]
if config["verbose"]:
logger.info(
format_log(f"Found {len(filtered_csv_list)} csv files",
process="ES Delete"))
deleted_ids = {}
for obj in filtered_csv_list:
object_data = access_s3_object(bucket_name=config["s3_bucket"],
obj=obj)
# Ingests the CSV into a dataframe. pandas thinks some ids are dates, so disable parsing
data = pd.read_csv(object_data, dtype=str)
if "detached_award_proc_unique" in data:
new_ids = [
"CONT_TX_" + x.upper()
for x in data["detached_award_proc_unique"].values
]
elif "afa_generated_unique" in data:
new_ids = [
"ASST_TX_" + x.upper()
for x in data["afa_generated_unique"].values
]
else:
logger.info(
format_log(f"[Missing valid col] in {obj.key}",
process="ES Delete"))
for uid in new_ids:
if uid in deleted_ids:
if deleted_ids[uid]["timestamp"] < obj.last_modified:
deleted_ids[uid]["timestamp"] = obj.last_modified
else:
deleted_ids[uid] = {"timestamp": obj.last_modified}
if config["verbose"]:
for uid, deleted_dict in deleted_ids.items():
logger.info(
format_log(
f"id: {uid} last modified: {deleted_dict['timestamp']}",
process="ES Delete"))
logger.info(
format_log(
f"Gathering {len(deleted_ids):,} deleted transactions took {perf_counter() - start:.2f}s",
process="ES Delete",
))
return deleted_ids
def gather_deleted_ids(config):
"""
Connect to S3 and gather all of the transaction ids stored in CSV files
generated by the broker when transactions are removed from the DB.
"""
if not config["process_deletes"]:
printf({"msg": "Skipping the S3 CSV fetch for deleted transactions"})
return
printf({"msg": "Gathering all deleted transactions from S3"})
start = perf_counter()
bucket_objects = access_s3_object_list(bucket_name=config["s3_bucket"])
if bucket_objects is None:
raise Exception(f"Issue connecting to {config['s3_bucket']} in s3")
if config["verbose"]:
printf({"msg": f"CSV data from {config['starting_date']} to now"})
filtered_csv_list = [
x for x in bucket_objects
if (x.key.endswith(".csv") and not x.key.startswith("staging")
and x.last_modified >= config["starting_date"])
]
if config["verbose"]:
printf({"msg": f"Found {len(filtered_csv_list)} csv files"})
deleted_ids = {}
for obj in filtered_csv_list:
object_data = access_s3_object(bucket_name=config["s3_bucket"],
obj=obj)
# Ingests the CSV into a dataframe. pandas thinks some ids are dates, so disable parsing
data = pd.read_csv(object_data, dtype=str)
if "detached_award_proc_unique" in data:
new_ids = [
"CONT_TX_" + x.upper()
for x in data["detached_award_proc_unique"].values
]
elif "afa_generated_unique" in data:
new_ids = [
"ASST_TX_" + x.upper()
for x in data["afa_generated_unique"].values
]
else:
printf({"msg": f" [Missing valid col] in {obj.key}"})
for uid in new_ids:
if uid in deleted_ids:
if deleted_ids[uid]["timestamp"] < obj.last_modified:
deleted_ids[uid]["timestamp"] = obj.last_modified
else:
deleted_ids[uid] = {"timestamp": obj.last_modified}
if config["verbose"]:
for uid, deleted_dict in deleted_ids.items():
printf({
"msg":
"id: {} last modified: {}".format(
uid, str(deleted_dict["timestamp"]))
})
printf({
"msg":
"Gathering {} deleted transactions took {}s".format(
len(deleted_ids),
perf_counter() - start)
})
return deleted_ids