def parse_source(source, columns, download_job, working_dir, piid, assistance_id, zip_file_path, limit):
"""Write to csv and zip files using the source data"""
d_map = {
"d1": "contracts",
"d2": "assistance",
"treasury_account": "treasury_account",
"federal_account": "federal_account",
}
if download_job and download_job.monthly_download:
# Use existing detailed filename from parent file for monthly files
# e.g. `019_Assistance_Delta_20180917_%s.csv`
source_name = strip_file_extension(download_job.file_name)
elif source.is_for_idv or source.is_for_contract:
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
source_name = file_name_pattern.format(piid=slugify_text_for_file_names(piid, "UNKNOWN", 50))
elif source.is_for_assistance:
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
source_name = file_name_pattern.format(assistance_id=slugify_text_for_file_names(assistance_id, "UNKNOWN", 50))
else:
source_name = "{}_{}_{}".format(
source.agency_code, d_map[source.file_type], VALUE_MAPPINGS[source.source_type]["download_name"]
)
source_query = source.row_emitter(columns)
source.file_name = "{}.csv".format(source_name)
source_path = os.path.join(working_dir, source.file_name)
write_to_log(message="Preparing to download data as {}".format(source_name), download_job=download_job)
# Generate the query file; values, limits, dates fixed
temp_file, temp_file_path = generate_temp_query_file(source_query, limit, source, download_job, columns)
start_time = time.perf_counter()
try:
# Create a separate process to run the PSQL command; wait
psql_process = multiprocessing.Process(target=execute_psql, args=(temp_file_path, source_path, download_job))
psql_process.start()
wait_for_process(psql_process, start_time, download_job)
# Log how many rows we have
write_to_log(message="Counting rows in CSV", download_job=download_job)
try:
download_job.number_of_rows += count_rows_in_csv_file(filename=source_path, has_header=True)
except Exception:
write_to_log(message="Unable to obtain CSV line count", is_error=True, download_job=download_job)
download_job.save()
# Create a separate process to split the large csv into smaller csvs and write to zip; wait
zip_process = multiprocessing.Process(
target=split_and_zip_csvs, args=(zip_file_path, source_path, source_name, download_job)
)
zip_process.start()
wait_for_process(zip_process, start_time, download_job)
download_job.save()
except Exception as e:
raise e
finally:
# Remove temporary files
os.close(temp_file)
os.remove(temp_file_path)
def build_data_file_name(source, download_job, piid, assistance_id):
d_map = {
"d1": "Contracts",
"d2": "Assistance",
"treasury_account": "TAS",
"federal_account": "FA"
}
if download_job and download_job.monthly_download:
# For monthly archives, use the existing detailed zip filename for the data files
# e.g. FY(All)-012_Contracts_Delta_20191108.zip -> FY(All)-012_Contracts_Delta_20191108_%.csv
return strip_file_extension(download_job.file_name)
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
timestamp = datetime.strftime(datetime.now(timezone.utc),
"%Y-%m-%d_H%HM%MS%S")
if source.is_for_idv or source.is_for_contract:
data_file_name = file_name_pattern.format(
piid=slugify_text_for_file_names(piid, "UNKNOWN", 50))
elif source.is_for_assistance:
data_file_name = file_name_pattern.format(
assistance_id=slugify_text_for_file_names(assistance_id, "UNKNOWN",
50))
elif source.source_type == "disaster_recipient":
data_file_name = file_name_pattern.format(
award_category=source.award_category, timestamp=timestamp)
else:
if source.agency_code == "all":
agency = "All"
else:
agency = str(source.agency_code)
request = json.loads(download_job.json_request)
filters = request["filters"]
if request.get("limit"):
agency = ""
elif source.file_type not in ("treasury_account", "federal_account"):
agency = f"{agency}_"
data_file_name = file_name_pattern.format(
agency=agency,
data_quarters=construct_data_date_range(filters),
level=d_map[source.file_type],
timestamp=timestamp,
type=d_map[source.file_type],
)
return data_file_name
def create_unique_filename(json_request, origination=None):
timestamp = datetime.strftime(datetime.now(timezone.utc), "%Y-%m-%d_H%HM%MS%S%f")
request_agency = json_request.get("agency", "all")
if json_request.get("is_for_idv"):
download_name = f"IDV_{slugify_text_for_file_names(json_request.get('piid'), 'UNKNOWN', 50)}_{timestamp}.zip"
elif json_request.get("is_for_contract"):
download_name = f"CONT_{slugify_text_for_file_names(json_request.get('piid'), 'UNKNOWN', 50)}_{timestamp}.zip"
elif json_request.get("is_for_assistance"):
slug_text = slugify_text_for_file_names(json_request.get("assistance_id"), "UNKNOWN", 50)
download_name = f"ASST_{slug_text}_{timestamp}.zip"
elif json_request["request_type"] == "account":
file_name_template = obtain_zip_filename_format(json_request["download_types"])
agency = obtain_filename_prefix_from_agency_id(request_agency)
level = "FA" if json_request["account_level"] == "federal_account" else "TAS"
data_quarters = construct_data_date_range(json_request["filters"])
download_name = file_name_template.format(
agency=agency, data_quarters=data_quarters, level=level, timestamp=timestamp,
)
else: # "award" downloads
agency = ""
# Since Keyword Search using the "Bulk Download" Endpoint for unknown reasons
# Check for the specific filter to mimic the Advanced Search download filename
if origination == "bulk_download" and "elasticsearch_keyword" not in json_request["filters"]:
agency = obtain_filename_prefix_from_agency_id(request_agency) + "_"
award_type_name = create_award_level_string(json_request["download_types"])
download_name = f"{agency}{award_type_name}_{timestamp}.zip"
return download_name
def create_unique_filename(json_request, request_agency=None):
if json_request.get("is_for_idv"):
download_name = "IDV_" + slugify_text_for_file_names(
json_request.get("piid"), "UNKNOWN", 50)
elif json_request.get("is_for_contract"):
download_name = "CONT_" + slugify_text_for_file_names(
json_request.get("piid"), "UNKNOWN", 50)
elif json_request.get("is_for_assistance"):
download_name = "ASST_" + slugify_text_for_file_names(
json_request.get("assistance_id"), "UNKNOWN", 50)
else:
download_types = json_request["download_types"]
prefix = obtain_filename_prefix_from_agency_id(request_agency)
award_type_name = create_award_level_string(download_types)
download_name = "{}_{}".format(prefix, award_type_name)
timestamped_file_name = get_timestamped_filename(
"{}.zip".format(download_name))
return timestamped_file_name
def parse_source(source, columns, download_job, working_dir, piid,
assistance_id, zip_file_path, limit, extension):
"""Write to delimited text file(s) and zip file(s) using the source data"""
d_map = {
"d1": "Contracts",
"d2": "Assistance",
"treasury_account": "TAS",
"federal_account": "FA",
}
if download_job and download_job.monthly_download:
# For monthly archives, use the existing detailed zip filename for the data files
# e.g. FY(All)-012_Contracts_Delta_20191108.zip -> FY(All)-012_Contracts_Delta_20191108_%.csv
source_name = strip_file_extension(download_job.file_name)
elif source.is_for_idv or source.is_for_contract:
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
source_name = file_name_pattern.format(
piid=slugify_text_for_file_names(piid, "UNKNOWN", 50))
elif source.is_for_assistance:
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
source_name = file_name_pattern.format(
assistance_id=slugify_text_for_file_names(assistance_id, "UNKNOWN",
50))
else:
file_name_pattern = VALUE_MAPPINGS[source.source_type]["download_name"]
if source.agency_code == "all":
agency = "All"
else:
agency = str(source.agency_code)
request = json.loads(download_job.json_request)
filters = request["filters"]
if request.get("limit"):
agency = ""
elif source.file_type not in ("treasury_account", "federal_account"):
agency = f"{agency}_"
timestamp = datetime.strftime(datetime.now(timezone.utc),
"%Y-%m-%d_H%HM%MS%S")
source_name = file_name_pattern.format(
agency=agency,
data_quarters=construct_data_date_range(filters),
level=d_map[source.file_type],
timestamp=timestamp,
type=d_map[source.file_type],
)
source_query = source.row_emitter(columns)
source.file_name = f"{source_name}.{extension}"
source_path = os.path.join(working_dir, source.file_name)
write_to_log(message=f"Preparing to download data as {source.file_name}",
download_job=download_job)
# Generate the query file; values, limits, dates fixed
temp_file, temp_file_path = generate_temp_query_file(
source_query, limit, source, download_job, columns, extension)
start_time = time.perf_counter()
try:
# Create a separate process to run the PSQL command; wait
psql_process = multiprocessing.Process(target=execute_psql,
args=(temp_file_path,
source_path,
download_job))
psql_process.start()
wait_for_process(psql_process, start_time, download_job)
delim = FILE_FORMATS[extension]["delimiter"]
# Log how many rows we have
write_to_log(message="Counting rows in delimited text file",
download_job=download_job)
try:
download_job.number_of_rows += count_rows_in_delimited_file(
filename=source_path, has_header=True, delimiter=delim)
except Exception:
write_to_log(
message="Unable to obtain delimited text file line count",
is_error=True,
download_job=download_job)
download_job.save()
# Create a separate process to split the large data files into smaller file and write to zip; wait
zip_process = multiprocessing.Process(target=split_and_zip_data_files,
args=(zip_file_path, source_path,
source_name, extension,
download_job))
zip_process.start()
wait_for_process(zip_process, start_time, download_job)
download_job.save()
except Exception as e:
raise e
finally:
# Remove temporary files
os.close(temp_file)
os.remove(temp_file_path)
def test_slugify_text_for_file_names():
assert slugify_text_for_file_names("test") == "test"
assert slugify_text_for_file_names("test", "default") == "test"
assert slugify_text_for_file_names("test", "default", 50) == "test"
assert slugify_text_for_file_names("test", "default", 2) == "te"
assert slugify_text_for_file_names(None) is None
assert slugify_text_for_file_names(None, None) is None
assert slugify_text_for_file_names(None, "default") == "default"
assert slugify_text_for_file_names(None, "default", 50) == "default"
assert slugify_text_for_file_names(None, "default", 2) == "default"
assert slugify_text_for_file_names("---") is None
assert slugify_text_for_file_names("---", "default") == "default"
assert slugify_text_for_file_names("---", "default", 50) == "default"
assert slugify_text_for_file_names("---", "default", 2) == "default"
garbage = r")*(&*()THIS(#@$@^*IS<,>.?/:;\"'{[}]|\+=_-)(*&^%$#@!A><?>\":}TEST()(*&(*&*(*"
assert slugify_text_for_file_names(garbage) == "THIS_IS_A_TEST"
assert slugify_text_for_file_names(garbage, "default") == "THIS_IS_A_TEST"
assert slugify_text_for_file_names(garbage, "default",
50) == "THIS_IS_A_TEST"
assert slugify_text_for_file_names(garbage, "default", 2) == "TH"
assert slugify_text_for_file_names("áéíóúüñ") == "aeiouun"
assert slugify_text_for_file_names("áéíóúüñ", "default") == "aeiouun"
assert slugify_text_for_file_names("áéíóúüñ", "default", 50) == "aeiouun"
assert slugify_text_for_file_names("áéíóúüñ", "default", 2) == "ae"
assert slugify_text_for_file_names("buenos días") == "buenos_dias"
assert slugify_text_for_file_names("buenos días",
"default") == "buenos_dias"
assert slugify_text_for_file_names("buenos días", "default",
50) == "buenos_dias"
assert slugify_text_for_file_names("buenos días", "default", 2) == "bu"
# These all contain no ascii compatible characters.
assert slugify_text_for_file_names("Καλημέρα") is None
assert slugify_text_for_file_names("Καλημέρα", "default") == "default"
assert slugify_text_for_file_names("Καλημέρα", "default", 50) == "default"
assert slugify_text_for_file_names("Καλημέρα", "default", 2) == "default"
assert slugify_text_for_file_names("Доброе утро") is None
assert slugify_text_for_file_names("Доброе утро", "default") == "default"
assert slugify_text_for_file_names("Доброе утро", "default",
50) == "default"
assert slugify_text_for_file_names("Доброе утро", "default",
2) == "default"
assert slugify_text_for_file_names("早上好/ 早") is None
assert slugify_text_for_file_names("早上好/ 早", "default") == "default"
assert slugify_text_for_file_names("早上好/ 早", "default", 50) == "default"
assert slugify_text_for_file_names("早上好/ 早", "default", 2) == "default"