def get_local_file_for_report(self, report):
"""Get full path for local report file."""
return utils.get_local_file_name(report)
def test_get_local_file_name_no_assembly(self):
"""Test get_local_file_name is successful with no assembly ID."""
input_key = "/koku/20180701-20180801/koku-Manifest.json"
expected_local_file = "koku-Manifest.json"
local_file = utils.get_local_file_name(input_key)
self.assertEqual(expected_local_file, local_file)
def download_file(self,
key,
stored_etag=None,
manifest_id=None,
start_date=None):
"""
Download an S3 object to file.
Args:
key (str): The S3 object key identified.
Returns:
(String): The path and file name of the saved file
"""
s3_filename = key.split("/")[-1]
directory_path = f"{DATA_DIR}/{self.customer_name}/aws/{self.bucket}"
local_s3_filename = utils.get_local_file_name(key)
msg = f"Local S3 filename: {local_s3_filename}"
LOG.info(log_json(self.request_id, msg, self.context))
full_file_path = f"{directory_path}/{local_s3_filename}"
# Make sure the data directory exists
os.makedirs(directory_path, exist_ok=True)
s3_etag = None
file_creation_date = None
try:
s3_file = self.s3_client.get_object(
Bucket=self.report.get("S3Bucket"), Key=key)
s3_etag = s3_file.get("ETag")
file_creation_date = s3_file.get("LastModified")
except ClientError as ex:
if ex.response["Error"]["Code"] == "NoSuchKey":
msg = "Unable to find {} in S3 Bucket: {}".format(
s3_filename, self.report.get("S3Bucket"))
LOG.info(log_json(self.request_id, msg, self.context))
raise AWSReportDownloaderNoFileError(msg)
msg = f"Error downloading file: Error: {str(ex)}"
LOG.error(log_json(self.request_id, msg, self.context))
raise AWSReportDownloaderError(str(ex))
if not self._check_size(key, check_inflate=True):
raise AWSReportDownloaderError(
f"Insufficient disk space to download file: {s3_file}")
if s3_etag != stored_etag or not os.path.isfile(full_file_path):
LOG.debug("Downloading key: %s to file path: %s", key,
full_file_path)
self.s3_client.download_file(self.report.get("S3Bucket"), key,
full_file_path)
# Push to S3
s3_csv_path = get_path_prefix(self.account, Provider.PROVIDER_AWS,
self._provider_uuid, start_date,
Config.CSV_DATA_TYPE)
utils.copy_local_report_file_to_s3_bucket(
self.request_id, s3_csv_path, full_file_path,
local_s3_filename, manifest_id, start_date, self.context)
utils.remove_files_not_in_set_from_s3_bucket(
self.request_id, s3_csv_path, manifest_id)
return full_file_path, s3_etag, file_creation_date
