def create_manifest_entries(report_meta, request_id, context={}):
"""
Creates manifest database entries for report processing tracking.
Args:
report_meta (dict): Report context dictionary from extract_payload.
request_id (String): Identifier associated with the payload
context (Dict): Context for logging (account, etc)
Returns:
manifest_id (Integer): Manifest identifier of the created db entry.
"""
class Request:
id = uuid.uuid4()
class Task:
request = Request()
downloader = OCPReportDownloader(
Task(),
report_meta.get("schema_name"),
report_meta.get("cluster_id"),
None,
provider_uuid=report_meta.get("provider_uuid"),
request_id=request_id,
account=context.get("account", "no_account"),
)
return downloader._prepare_db_manifest_record(report_meta)
def create_manifest_entries(report_meta):
"""
Creates manifest database entries for report processing tracking.
Args:
report_meta (dict): Report context dictionary from extract_payload.
Returns:
manifest_id (Integer): Manifest identifier of the created db entry.
"""
class Request:
id = uuid.uuid4()
class Task:
request = Request()
downloader = OCPReportDownloader(
Task(),
report_meta.get("schema_name"),
report_meta.get("cluster_id"),
None,
provider_uuid=report_meta.get("provider_uuid"),
)
return downloader._prepare_db_manifest_record(report_meta)
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def setUp(self):
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(self.fake_customer_name,
self.cluster_id, None, 'OCP',
self.ocp_provider_id)
self.ocp_report_downloader = OCPReportDownloader(
**{
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_id': 1,
})
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91' \
'-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type='OCP',
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_uuid': self.ocp_provider_uuid,
})
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == AMAZON_WEB_SERVICES:
return AWSReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AWS_LOCAL_SERVICE_PROVIDER:
return AWSLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE:
return AzureReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == AZURE_LOCAL_SERVICE_PROVIDER:
return AzureLocalReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == OPENSHIFT_CONTAINER_PLATFORM:
return OCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
if self.provider_type == GCP:
return GCPReportDownloader(customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_id=self.provider_id)
return None
def _set_downloader(self):
"""
Create the report downloader object.
Downloader is specific to the provider's cloud service.
Args:
None
Returns:
(Object) : Some object that is a child of CURAccountsInterface
"""
if self.provider_type == Provider.PROVIDER_AWS:
return AWSReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AWS_LOCAL:
return AWSLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE:
return AzureReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_AZURE_LOCAL:
return AzureLocalReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_OCP:
return OCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
bucket=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
if self.provider_type == Provider.PROVIDER_GCP:
return GCPReportDownloader(
customer_name=self.customer_name,
auth_credential=self.credential,
billing_source=self.cur_source,
report_name=self.report_name,
provider_uuid=self.provider_uuid,
request_id=self.request_id,
account=self.account,
)
return None
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
def test_divide_csv_daily_failure(self):
"""Test the divide_csv_daily method throw error on reading CSV."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
errorMsg = "CParserError: Error tokenizing data. C error: Expected 53 fields in line 1605634, saw 54"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_pd.read_csv.side_effect = Exception(errorMsg)
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.LOG.error"
) as mock_debug:
with self.assertRaises(Exception):
divide_csv_daily(file_path, filename)
mock_debug.assert_called_once_with(
f"File {file_path} could not be parsed. Reason: {errorMsg}"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._get_manifest"
)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
}
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
@override_settings(ENABLE_S3_ARCHIVING=True)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch("masu.external.downloader.ocp.ocp_report_downloader.os")
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.copy_local_report_file_to_s3_bucket"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.divide_csv_daily")
def test_create_daily_archives(self, mock_divide, mock_s3_copy, mock_os):
"""Test that this method returns a file list."""
start_date = DateHelper().this_month_start
daily_files = [
{
"filename": "file_one",
"filepath": "path/to/file_one"
},
{
"filename": "file_two",
"filepath": "path/to/file_two"
},
]
expected_filenames = ["path/to/file_one", "path/to/file_two"]
mock_divide.return_value = daily_files
result = create_daily_archives(1, "10001", self.ocp_provider_uuid,
"file", "path", 1, start_date)
self.assertEqual(result, expected_filenames)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = 'ocp-report'
self.cluster_id = 'my-ocp-cluster-1'
report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id,
'20180901-20181001')
os.makedirs(report_path, exist_ok=True)
test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv'
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = './koku/masu/test/data/ocp/manifest.json'
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type='OCP',
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
'task': self.mock_task,
'customer_name': self.fake_customer_name,
'auth_credential': self.cluster_id,
'bucket': None,
'provider_uuid': self.ocp_provider_uuid,
})
def tearDown(self):
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = '{}/{}/{}'.format(Config.TMP_DIR,
self.fake_customer_name, 'ocp')
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
reports = []
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
os.remove(self.test_file_path)
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
reports = []
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
# Create .txt file
txt_file_path = '{}/{}'.format(
os.path.dirname(self.test_file_path), 'report.txt')
open(txt_file_path, 'a').close()
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, 'today',
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
def test_download_bucket(self):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.mock_task = Mock(
request=Mock(id=str(self.fake.uuid4()), return_value={}))
self.report_downloader = ReportDownloader(
task=self.mock_task,
customer_name=self.fake_customer_name,
access_credential=self.cluster_id,
report_source=None,
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
cache_key=self.fake.word(),
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"task": self.mock_task,
"customer_name": self.fake_customer_name,
"auth_credential": self.cluster_id,
"bucket": None,
"provider_uuid": self.ocp_provider_uuid,
"cache_key": self.fake.word(),
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
self.report_downloader.download_report(test_report_date)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
self.report_downloader.download_report(test_report_date)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
self.report_downloader.download_report(test_report_date)
def test_download_bucket_source_directory_missing(self):
"""Test to verify that basic report downloading when source directory doesn't exist."""
reports = []
# Set current date to a day that is outside of the test file's date range.
test_report_date = datetime(year=2018, month=10, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
reports = self.report_downloader.download_report(test_report_date)
self.assertEqual(reports, [])
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
class OCPReportDownloaderTest(MasuTestCase):
"""Test Cases for the OCP Report Downloader."""
fake = Faker()
def setUp(self):
"""Set up each test."""
super().setUp()
self.fake_customer_name = CUSTOMER_NAME
self.fake_report_name = "ocp-report"
self.cluster_id = "my-ocp-cluster-1"
self.credentials = {"cluster_id": self.cluster_id}
report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id,
"20180901-20181001")
os.makedirs(report_path, exist_ok=True)
test_file_path = (
"./koku/masu/test/data/ocp/e6b3701e-1e91"
"-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv")
self.test_file_path = os.path.join(report_path,
os.path.basename(test_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_file_path))
test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv"
self.test_storage_file_path = os.path.join(
report_path, os.path.basename(test_storage_file_path))
shutil.copyfile(test_file_path,
os.path.join(report_path, self.test_storage_file_path))
test_manifest_path = "./koku/masu/test/data/ocp/manifest.json"
self.test_manifest_path = os.path.join(
report_path, os.path.basename(test_manifest_path))
shutil.copyfile(test_manifest_path,
os.path.join(report_path, self.test_manifest_path))
self.report_downloader = ReportDownloader(
customer_name=self.fake_customer_name,
credentials=self.credentials,
data_source={},
provider_type=Provider.PROVIDER_OCP,
provider_uuid=self.ocp_provider_uuid,
)
self.ocp_report_downloader = OCPReportDownloader(
**{
"customer_name": self.fake_customer_name,
"credentials": self.credentials,
"data_source": {},
"provider_uuid": self.ocp_provider_uuid,
})
def tearDown(self):
"""Remove created test data."""
super().tearDown()
shutil.rmtree(REPORTS_DIR, ignore_errors=True)
@patch("masu.util.aws.common.copy_data_to_s3_bucket", return_value=None)
def test_download_bucket(self, mock_copys3):
"""Test to verify that basic report downloading works."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
expected_path = "{}/{}/{}".format(Config.TMP_DIR,
self.fake_customer_name, "ocp")
self.assertTrue(os.path.isdir(expected_path))
def test_download_bucket_no_csv_found(self):
"""Test to verify that basic report downloading with no .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_download_bucket_non_csv_found(self):
"""Test to verify that basic report downloading with non .csv file in source directory."""
test_report_date = datetime(year=2018, month=9, day=7)
with patch.object(DateAccessor, "today",
return_value=test_report_date):
# Remove .csv
os.remove(self.test_file_path)
os.remove(self.test_storage_file_path)
# Create .txt file
txt_file_path = "{}/{}".format(
os.path.dirname(self.test_file_path), "report.txt")
open(txt_file_path, "a").close()
with self.assertRaises(FileNotFoundError):
report_context = {
"date": test_report_date.date(),
"manifest_id": 1,
"comporession": "GZIP",
"current_file": self.test_file_path,
}
self.report_downloader.download_report(report_context)
def test_remove_manifest_file(self):
"""Test that a manifest file is deleted after use."""
test_report_date = datetime(year=2018, month=9, day=7)
self.assertTrue(os.path.isfile(self.test_manifest_path))
self.ocp_report_downloader._remove_manifest_file(test_report_date)
self.assertFalse(os.path.isfile(self.test_manifest_path))
def test_delete_manifest_file_warning(self):
"""Test that an INFO is logged when removing a manifest file that does not exist."""
with self.assertLogs(
logger="masu.external.downloader.ocp.ocp_report_downloader",
level="INFO") as captured_logs:
# Disable log suppression
logging.disable(logging.NOTSET)
self.ocp_report_downloader._remove_manifest_file(datetime.now())
self.assertTrue(
captured_logs.output[0].startswith("INFO:"),
msg=
"The log is expected to start with 'INFO:' but instead was: " +
captured_logs.output[0],
)
self.assertTrue(
"Could not delete manifest file at" in captured_logs.output[0],
msg="""The log message is expected to contain
'Could not delete manifest file at' but instead was: """
+ captured_logs.output[0],
)
# Re-enable log suppression
logging.disable(logging.CRITICAL)
def test_divide_csv_daily(self):
"""Test the divide_csv_daily method."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_report = {
"interval_start": [
"2020-01-01 00:00:00 +UTC",
"2020-01-02 00:00:00 +UTC"
],
"persistentvolumeclaim_labels": ["label1", "label2"],
}
df = pd.DataFrame(data=mock_report)
mock_pd.read_csv.return_value = df
daily_files = divide_csv_daily(file_path, filename)
self.assertNotEqual([], daily_files)
self.assertEqual(len(daily_files), 2)
gen_files = [
"storage_usage.2020-01-01.csv",
"storage_usage.2020-01-02.csv"
]
expected = [{
"filename": gen_file,
"filepath": f"{td}/{gen_file}"
} for gen_file in gen_files]
for expected_item in expected:
self.assertIn(expected_item, daily_files)
def test_divide_csv_daily_failure(self):
"""Test the divide_csv_daily method throw error on reading CSV."""
with tempfile.TemporaryDirectory() as td:
filename = "storage_data.csv"
file_path = f"{td}/{filename}"
errorMsg = "CParserError: Error tokenizing data. C error: Expected 53 fields in line 1605634, saw 54"
with patch("masu.external.downloader.ocp.ocp_report_downloader.pd"
) as mock_pd:
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.detect_type",
return_value=("storage_usage", None),
):
mock_pd.read_csv.side_effect = Exception(errorMsg)
with patch(
"masu.external.downloader.ocp.ocp_report_downloader.LOG.error"
) as mock_debug:
with self.assertRaises(Exception):
divide_csv_daily(file_path, filename)
mock_debug.assert_called_once_with(
f"File {file_path} could not be parsed. Reason: {errorMsg}"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.get_report_details"
)
def test_get_manifest_context_for_date(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
version = "5678"
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
"version": version,
}
self.assertIsNone(self.ocp_report_downloader.context.get("version"))
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
manifest_id = result.get("manifest_id")
manifest = ReportManifestDBAccessor().get_manifest_by_id(manifest_id)
self.assertEqual(manifest.operator_version, version)
self.assertEqual(self.ocp_report_downloader.context.get("version"),
version)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.OCPReportDownloader._remove_manifest_file"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.utils.get_report_details"
)
def test_get_manifest_context_new_info(self, mock_manifest, mock_delete):
"""Test that the manifest is read."""
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
assembly_id = "1234"
compression = "PLAIN"
report_keys = ["file1", "file2"]
version = "5678"
mock_manifest.return_value = {
"uuid": assembly_id,
"Compression": compression,
"reportKeys": report_keys,
"date": current_month,
"files": report_keys,
"version": version,
"certified": False,
"cluster_id": "4e009161-4f40-42c8-877c-3e59f6baea3d",
"cr_status": {
"clusterID": "4e009161-4f40-42c8-877c-3e59f6baea3d",
"clusterVersion": "stable-4.6",
"api_url": "https://cloud.redhat.com",
"authentication": {
"type": "token"
},
"packaging": {
"max_reports_to_store": 30,
"max_size_MB": 100
},
"upload": {
"ingress_path": "/api/ingress/v1/upload",
"upload": False
},
"operator_commit": "a09a5b21e55ce4a07fe31aa560650b538ec6de7c",
"prometheus": {
"error": "fake error"
},
"reports": {
"report_month":
"07",
"last_hour_queried":
"2021-07-28 11:00:00 - 2021-07-28 11:59:59"
},
"source": {
"sources_path": "/api/sources/v1.0/",
"name": "INSERT-SOURCE-NAME"
},
},
}
self.assertIsNone(self.ocp_report_downloader.context.get("version"))
result = self.ocp_report_downloader.get_manifest_context_for_date(
current_month)
self.assertEqual(result.get("assembly_id"), assembly_id)
self.assertEqual(result.get("compression"), compression)
self.assertIsNotNone(result.get("files"))
manifest_id = result.get("manifest_id")
manifest = ReportManifestDBAccessor().get_manifest_by_id(manifest_id)
expected_errors = {"prometheus_error": "fake error"}
self.assertEqual(manifest.operator_version, version)
self.assertEqual(manifest.operator_certified, False)
self.assertEqual(manifest.operator_airgapped, True)
self.assertEqual(manifest.cluster_channel, "stable-4.6")
self.assertEqual(manifest.cluster_id,
"4e009161-4f40-42c8-877c-3e59f6baea3d")
self.assertEqual(manifest.operator_errors, expected_errors)
self.assertEqual(self.ocp_report_downloader.context.get("version"),
version)
@override_settings(ENABLE_S3_ARCHIVING=True)
@override_settings(ENABLE_PARQUET_PROCESSING=True)
@patch("masu.external.downloader.ocp.ocp_report_downloader.os")
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.copy_local_report_file_to_s3_bucket"
)
@patch(
"masu.external.downloader.ocp.ocp_report_downloader.divide_csv_daily")
def test_create_daily_archives(self, mock_divide, mock_s3_copy, mock_os):
"""Test that this method returns a file list."""
start_date = DateHelper().this_month_start
daily_files = [
{
"filename": "file_one",
"filepath": "path/to/file_one"
},
{
"filename": "file_two",
"filepath": "path/to/file_two"
},
]
expected_filenames = ["path/to/file_one", "path/to/file_two"]
mock_divide.return_value = daily_files
file_name = "file"
file_path = "path"
result = create_daily_archives(1, "10001", self.ocp_provider_uuid,
file_name, file_path, 1, start_date)
self.assertEqual(result, expected_filenames)
context = {"version": "1"}
expected = [file_path]
result = create_daily_archives(1,
"10001",
self.ocp_provider_uuid,
"file",
"path",
1,
start_date,
context=context)
self.assertEqual(result, expected)
@patch("masu.external.downloader.ocp.ocp_report_downloader.LOG")
def test_get_report_for_verify_tracing_id(self, log_mock):
self.ocp_report_downloader.tracing_id = "1111-2222-4444-5555"
current_month = DateAccessor().today().replace(day=1,
second=1,
microsecond=1)
self.ocp_report_downloader.get_report_for(current_month)
call_args = log_mock.debug.call_args[0][0]
self.assertTrue("Looking for cluster" in call_args.get("message"))
self.assertEqual(call_args.get("tracing_id"),
self.ocp_report_downloader.tracing_id)
call_args = log_mock.info.call_args[0][0]
self.assertTrue("manifest found:" in call_args.get("message"))
self.assertEqual(call_args.get("tracing_id"),
self.ocp_report_downloader.tracing_id)
