def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None):
"""Load Azure data into the database."""
provider_type = Provider.PROVIDER_AZURE_LOCAL
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
if credentials is None:
credentials = {
"subscription_id": "11111111-1111-1111-1111-11111111",
"tenant_id": "22222222-2222-2222-2222-22222222",
"client_id": "33333333-3333-3333-3333-33333333",
"client_secret": "MyPassW0rd!",
}
if data_source is None:
data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
template, static_data_path = self.prepare_template(provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"azure_report_name": report_name,
"azure_container_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date, base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
elif "manifest" in report.name.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider, manifest)
with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file):
"""Load GCP data into the database."""
provider_type = Provider.PROVIDER_GCP_LOCAL
nise_provider_type = provider_type.replace("-local", "")
credentials = {"project_id": "test_project_id"}
data_source = {"table_id": "test_table_id", "dataset": "test_dataset"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
etag = uuid4()
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"gcp_bucket_name": self.nise_data_path,
"gcp_etag": etag
}
base_path = f"{self.nise_data_path}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = f"{base_path}/{etag}"
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"), patch.object(
settings, "AUTO_DATA_INGEST", False):
update_summary_tables(self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None):
"""Load AWS data into the database."""
provider_type = Provider.PROVIDER_AWS_LOCAL
if account_id is None:
account_id = "9999999999999"
if role_arn is None:
role_arn = "arn:aws:iam::999999999999:role/CostManagement"
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
credentials = {"role_arn": role_arn}
data_source = {"bucket": "test-bucket"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
template, static_data_path = self.prepare_template(provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"aws_report_name": report_name,
"aws_bucket_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
with schema_context(self.schema):
baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account")
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date, base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "GZIP", provider_type, provider, manifest)
with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_openshift_data(self, customer, static_data_file, cluster_id):
"""Load OpenShift data into the database."""
provider_type = Provider.PROVIDER_OCP
with override_settings(AUTO_DATA_INGEST=False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__provider_resource_name=cluster_id,
billing_source__bucket="",
customer=customer,
)
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"insights_upload": self.nise_data_path,
"ocp_cluster_id": cluster_id,
}
base_path = f"{self.nise_data_path}/{cluster_id}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
num_processed_files=0,
num_total_files=3,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date,
base_path)
for report in os.scandir(report_path):
shutil.move(report.path, f"{base_path}/{report.name}")
for report in [f.path for f in os.scandir(base_path)]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"):
update_summary_tables(self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id)
update_cost_model_costs(self.schema, provider.uuid,
self.dh.last_month_start, self.dh.today)
refresh_materialized_views(self.schema, provider_type)
shutil.rmtree(report_path, ignore_errors=True)
def test_run_for_azure_enddates(self):
"""That that fix_dates corrects the azure end_date."""
start = date.today().replace(day=1)
end = date.today().replace(day=1)
args = ["report", "azure", "-s", str(start), "-e", str(end)]
parsed_args = self.parser.parse_args(args)
options = vars(parsed_args)
_, provider_type = _validate_provider_inputs(self.parser, options)
self.assertEqual(provider_type, "azure")
with patch("nise.__main__.azure_create_report"):
run(provider_type, options)
self.assertEqual(
options.get("end_date").date(), end + timedelta(days=1))
