def test_build_file_description():
with NamedTemporaryFile() as f:
f.write(b"test")
f.flush()
fd = build_file_description(f.name, [])
assert fd == "test"
with NamedTemporaryFile() as f:
f.write(b"\n".join([b"this", b"is", b"a", b"test"]))
f.flush()
fd = build_file_description(f.name, [])
assert fd == "this\nis\na\ntest"
with NamedTemporaryFile() as f:
f.write(b"\n".join([b"this", b"is", b"#not", b"a", b"test"]))
f.flush()
fd = build_file_description(f.name, [])
assert fd == "this\nis\na\ntest"
with NamedTemporaryFile() as f:
source = namedtuple("str", "source_type file_name")
source.source_type = "replace_me"
source.file_name = "also"
f.write(b"\n".join(
[b"this", b"is", b"#not", b"{replace_me}", b"a", b"test"]))
f.flush()
fd = build_file_description(f.name, [source])
assert fd == "this\nis\nalso\na\ntest"
def finalize_zip_contents(self):
self.filepaths_to_delete.append(self.working_dir_path /
"Data_Dictionary_Crosswalk.xlsx")
add_data_dictionary_to_zip(str(self.zip_file_path.parent),
str(self.zip_file_path))
file_description = build_file_description(str(self.readme_path),
dict())
file_description_path = save_file_description(
str(self.zip_file_path.parent), self.readme_path.name,
file_description)
self.filepaths_to_delete.append(Path(file_description_path))
append_files_to_zip_file([file_description_path],
str(self.zip_file_path))
self.total_download_size = self.zip_file_path.stat().st_size
def generate_download(download_job: DownloadJob,
origination: Optional[str] = None):
"""Create data archive files from the download job object"""
# Parse data from download_job
json_request = json.loads(download_job.json_request)
columns = json_request.get("columns", None)
limit = json_request.get("limit", None)
piid = json_request.get("piid", None)
award_id = json_request.get("award_id")
assistance_id = json_request.get("assistance_id")
file_format = json_request.get("file_format")
file_name = start_download(download_job)
working_dir = None
try:
# Create temporary files and working directory
zip_file_path = settings.CSV_LOCAL_PATH + file_name
if not settings.IS_LOCAL and os.path.exists(zip_file_path):
# Clean up a zip file that might exist from a prior attempt at this download
os.remove(zip_file_path)
working_dir = os.path.splitext(zip_file_path)[0]
if not os.path.exists(working_dir):
os.mkdir(working_dir)
write_to_log(message=f"Generating {file_name}",
download_job=download_job)
# Generate sources from the JSON request object
sources = get_download_sources(json_request, origination)
for source in sources:
# Parse and write data to the file; if there are no matching columns for a source then add an empty file
source_column_count = len(source.columns(columns))
if source_column_count == 0:
create_empty_data_file(source, download_job, working_dir, piid,
assistance_id, zip_file_path,
file_format)
else:
download_job.number_of_columns += source_column_count
parse_source(source, columns, download_job, working_dir, piid,
assistance_id, zip_file_path, limit, file_format)
include_data_dictionary = json_request.get("include_data_dictionary")
if include_data_dictionary:
add_data_dictionary_to_zip(working_dir, zip_file_path)
include_file_description = json_request.get("include_file_description")
if include_file_description:
write_to_log(message="Adding file description to zip file")
file_description = build_file_description(
include_file_description["source"], sources)
file_description = file_description.replace(
"[AWARD_ID]", str(award_id))
file_description_path = save_file_description(
working_dir, include_file_description["destination"],
file_description)
append_files_to_zip_file([file_description_path], zip_file_path)
download_job.file_size = os.stat(zip_file_path).st_size
except InvalidParameterException as e:
exc_msg = "InvalidParameterException was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise InvalidParameterException(e)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise Exception(download_job.error_message) from e
finally:
# Remove working directory
if working_dir and os.path.exists(working_dir):
shutil.rmtree(working_dir)
_kill_spawned_processes(download_job)
try:
# push file to S3 bucket, if not local
if not settings.IS_LOCAL:
bucket = settings.BULK_DOWNLOAD_S3_BUCKET_NAME
region = settings.USASPENDING_AWS_REGION
start_uploading = time.perf_counter()
multipart_upload(bucket, region, zip_file_path,
os.path.basename(zip_file_path))
write_to_log(
message=
f"Uploading took {time.perf_counter() - start_uploading:.2f}s",
download_job=download_job)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to upload the file"
fail_download(download_job, e, exc_msg)
if isinstance(e, InvalidParameterException):
raise InvalidParameterException(e)
else:
raise Exception(download_job.error_message) from e
finally:
# Remove generated file
if not settings.IS_LOCAL and os.path.exists(zip_file_path):
os.remove(zip_file_path)
_kill_spawned_processes(download_job)
return finish_download(download_job)
def generate_csvs(download_job):
"""Derive the relevant file location and write CSVs to it"""
# Parse data from download_job
json_request = json.loads(download_job.json_request)
columns = json_request.get('columns', None)
limit = json_request.get('limit', None)
piid = json_request.get('piid', None)
file_name = start_download(download_job)
try:
# Create temporary files and working directory
zip_file_path = settings.CSV_LOCAL_PATH + file_name
working_dir = os.path.splitext(zip_file_path)[0]
if not os.path.exists(working_dir):
os.mkdir(working_dir)
write_to_log(message='Generating {}'.format(file_name), download_job=download_job)
# Generate sources from the JSON request object
sources = get_csv_sources(json_request)
for source in sources:
# Parse and write data to the file
download_job.number_of_columns = max(download_job.number_of_columns, len(source.columns(columns)))
parse_source(source, columns, download_job, working_dir, piid, zip_file_path, limit)
include_file_description = json_request.get('include_file_description')
if include_file_description:
write_to_log(message="Adding file description to zip file")
file_description = build_file_description(include_file_description["source"], sources)
file_description_path = save_file_description(
working_dir, include_file_description["destination"], file_description)
append_files_to_zip_file([file_description_path], zip_file_path)
download_job.file_size = os.stat(zip_file_path).st_size
except InvalidParameterException as e:
exc_msg = "InvalidParameterException was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise InvalidParameterException(e)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise Exception(download_job.error_message) from e
finally:
# Remove working directory
if os.path.exists(working_dir):
shutil.rmtree(working_dir)
try:
# push file to S3 bucket, if not local
if not settings.IS_LOCAL:
bucket = settings.BULK_DOWNLOAD_S3_BUCKET_NAME
region = settings.USASPENDING_AWS_REGION
start_uploading = time.perf_counter()
multipart_upload(bucket, region, zip_file_path, os.path.basename(zip_file_path))
write_to_log(message='Uploading took {} seconds'.format(time.perf_counter() - start_uploading),
download_job=download_job)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to upload the file"
fail_download(download_job, e, exc_msg)
if isinstance(e, InvalidParameterException):
raise InvalidParameterException(e)
else:
raise Exception(download_job.error_message) from e
finally:
# Remove generated file
if not settings.IS_LOCAL and os.path.exists(zip_file_path):
os.remove(zip_file_path)
return finish_download(download_job)
def generate_download(download_job: DownloadJob,
origination: Optional[str] = None):
"""Create data archive files from the download job object"""
# Parse data from download_job
json_request = json.loads(download_job.json_request)
columns = json_request.get("columns", None)
limit = json_request.get("limit", None)
piid = json_request.get("piid", None)
award_id = json_request.get("award_id")
assistance_id = json_request.get("assistance_id")
file_format = json_request.get("file_format")
request_type = json_request.get("request_type")
span = tracer.current_span()
if span and request_type:
span.resource = request_type
file_name = start_download(download_job)
working_dir = None
try:
# Create temporary files and working directory
zip_file_path = settings.CSV_LOCAL_PATH + file_name
if not settings.IS_LOCAL and os.path.exists(zip_file_path):
# Clean up a zip file that might exist from a prior attempt at this download
os.remove(zip_file_path)
working_dir = os.path.splitext(zip_file_path)[0]
if not os.path.exists(working_dir):
os.mkdir(working_dir)
write_to_log(message=f"Generating {file_name}",
download_job=download_job)
# Generate sources from the JSON request object
sources = get_download_sources(json_request, origination)
for source in sources:
# Parse and write data to the file; if there are no matching columns for a source then add an empty file
source_column_count = len(source.columns(columns))
if source_column_count == 0:
create_empty_data_file(source, download_job, working_dir, piid,
assistance_id, zip_file_path,
file_format)
else:
download_job.number_of_columns += source_column_count
parse_source(source, columns, download_job, working_dir, piid,
assistance_id, zip_file_path, limit, file_format)
include_data_dictionary = json_request.get("include_data_dictionary")
if include_data_dictionary:
add_data_dictionary_to_zip(working_dir, zip_file_path)
include_file_description = json_request.get("include_file_description")
if include_file_description:
write_to_log(message="Adding file description to zip file")
file_description = build_file_description(
include_file_description["source"], sources)
file_description = file_description.replace(
"[AWARD_ID]", str(award_id))
file_description_path = save_file_description(
working_dir, include_file_description["destination"],
file_description)
append_files_to_zip_file([file_description_path], zip_file_path)
download_job.file_size = os.stat(zip_file_path).st_size
except InvalidParameterException as e:
exc_msg = "InvalidParameterException was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise InvalidParameterException(e)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to process the DownloadJob"
fail_download(download_job, e, exc_msg)
raise Exception(download_job.error_message) from e
finally:
# Remove working directory
if working_dir and os.path.exists(working_dir):
shutil.rmtree(working_dir)
_kill_spawned_processes(download_job)
# push file to S3 bucket, if not local
if not settings.IS_LOCAL:
with tracer.trace(
name=f"job.{JOB_TYPE}.download.s3",
service="bulk-download",
resource=f"s3://{settings.BULK_DOWNLOAD_S3_BUCKET_NAME}",
span_type=SpanTypes.WORKER,
) as span, tracer.trace(
name="s3.command",
service="aws.s3",
resource=".".join([
multipart_upload.__module__,
(multipart_upload.__qualname__
or multipart_upload.__name__)
]),
span_type=SpanTypes.WEB,
) as s3_span:
# NOTE: Traces still not auto-picking-up aws.s3 service upload activity
# Could be that the patches for boto and botocore don't cover the newer boto3 S3Transfer upload approach
span.set_tag("file_name", file_name)
try:
bucket = settings.BULK_DOWNLOAD_S3_BUCKET_NAME
region = settings.USASPENDING_AWS_REGION
s3_span.set_tags({
"bucket": bucket,
"region": region,
"file": zip_file_path
})
start_uploading = time.perf_counter()
multipart_upload(bucket, region, zip_file_path,
os.path.basename(zip_file_path))
write_to_log(
message=
f"Uploading took {time.perf_counter() - start_uploading:.2f}s",
download_job=download_job)
except Exception as e:
# Set error message; job_status_id will be set in download_sqs_worker.handle()
exc_msg = "An exception was raised while attempting to upload the file"
fail_download(download_job, e, exc_msg)
if isinstance(e, InvalidParameterException):
raise InvalidParameterException(e)
else:
raise Exception(download_job.error_message) from e
finally:
# Remove generated file
if os.path.exists(zip_file_path):
os.remove(zip_file_path)
_kill_spawned_processes(download_job)
return finish_download(download_job)