def test_create_daily_archives(self, mock_s3):
"""Test that we load daily files to S3."""
data = {
"usage_start_time": ["2021-02-01T00:00:00Z", "2021-02-02T00:00:00Z", "2021-02-03T00:00:00Z"],
"usage": [1, 2, 3],
"cost": [4, 5, 6],
}
expected_daily_files = ["/tmp/2021-02-01.csv", "/tmp/2021-02-02.csv", "/tmp/2021-02-03.csv"]
file_path = "/tmp/test.csv"
df = pd.DataFrame(data)
df.to_csv(file_path, index=False, header=True)
start_date = DateHelper().this_month_start
daily_file_names = create_daily_archives(
"request_id", "account", self.gcp_provider_uuid, "test.csv", file_path, None, start_date
)
mock_s3.assert_called()
self.assertEqual(sorted(daily_file_names), sorted(expected_daily_files))
for daily_file in expected_daily_files:
self.assertTrue(os.path.exists(daily_file))
os.remove(daily_file)
os.remove(file_path)
def test_create_daily_archives(self, mock_s3):
"""Test that we load daily files to S3."""
# Use the processor example for data:
file_path = "./koku/masu/test/data/gcp/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
file_name = "202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
temp_dir = tempfile.gettempdir()
temp_path = os.path.join(temp_dir, file_name)
shutil.copy2(file_path, temp_path)
expected_daily_files = [
f"{temp_dir}/202011_2020-11-08.csv",
f"{temp_dir}/202011_2020-11-09.csv",
f"{temp_dir}/202011_2020-11-10.csv",
f"{temp_dir}/202011_2020-11-11.csv",
]
start_date = DateHelper().this_month_start
daily_file_names = create_daily_archives("request_id", "account",
self.gcp_provider_uuid,
file_name, temp_path, None,
start_date, None)
mock_s3.assert_called()
self.assertEqual(sorted(daily_file_names),
sorted(expected_daily_files))
for daily_file in expected_daily_files:
self.assertTrue(os.path.exists(daily_file))
os.remove(daily_file)
os.remove(temp_path)
def download_file(self, key, stored_etag=None, manifest_id=None, start_date=None):
"""
Download a file from GCP storage bucket.
If we have a stored etag and it matches the current GCP blob, we can
safely skip download since the blob/file content must not have changed.
Args:
key (str): name of the blob in the GCP storage bucket
stored_etag (str): optional etag stored in our DB for comparison
Returns:
tuple(str, str) with the local filesystem path to file and GCP's etag.
"""
report_name = os.path.splitext(key)[0]
etag = report_name.split("_")[1]
full_local_path = self._get_local_file_path(key, etag)
msg = f"Returning full_file_path: {full_local_path}"
LOG.info(log_json(self.request_id, msg, self.context))
dh = DateHelper()
file_names = create_daily_archives(
self.request_id,
self.account,
self._provider_uuid,
key,
full_local_path,
manifest_id,
start_date,
self.context,
)
return full_local_path, etag, dh.today, file_names