def load_gcp_data(self, customer):
"""Load GCP data into the database."""
provider_type = Provider.PROVIDER_GCP_LOCAL
credentials = {"project_id": "test_project_id"}
data_source = {"table_id": "test_table_id", "dataset": "test_dataset"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None):
"""Load Azure data into the database."""
provider_type = Provider.PROVIDER_AZURE_LOCAL
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
if credentials is None:
credentials = {
"subscription_id": "11111111-1111-1111-1111-11111111",
"tenant_id": "22222222-2222-2222-2222-22222222",
"client_id": "33333333-3333-3333-3333-33333333",
"client_secret": "MyPassW0rd!",
}
if data_source is None:
data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
template, static_data_path = self.prepare_template(provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"azure_report_name": report_name,
"azure_container_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date, base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
elif "manifest" in report.name.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider, manifest)
with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None):
"""Load AWS data into the database."""
provider_type = Provider.PROVIDER_AWS_LOCAL
if account_id is None:
account_id = "9999999999999"
if role_arn is None:
role_arn = "arn:aws:iam::999999999999:role/CostManagement"
nise_provider_type = provider_type.replace("-local", "")
report_name = "Test"
credentials = {"role_arn": role_arn}
data_source = {"bucket": "test-bucket"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
template, static_data_path = self.prepare_template(provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"aws_report_name": report_name,
"aws_bucket_name": self.nise_data_path,
}
base_path = f"{self.nise_data_path}/{report_name}"
with schema_context(self.schema):
baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account")
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id))
run(nise_provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date, base_path)
for report in os.scandir(report_path):
if os.path.isdir(report):
for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "GZIP", provider_type, provider, manifest)
with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file):
"""Load GCP data into the database."""
provider_type = Provider.PROVIDER_GCP_LOCAL
nise_provider_type = provider_type.replace("-local", "")
credentials = {"project_id": "test_project_id"}
data_source = {"table_id": "test_table_id", "dataset": "test_dataset"}
with patch.object(settings, "AUTO_DATA_INGEST", False):
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source__data_source=data_source,
customer=customer,
)
etag = uuid4()
template, static_data_path = self.prepare_template(
provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"gcp_bucket_name": self.nise_data_path,
"gcp_etag": etag
}
base_path = f"{self.nise_data_path}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
)
with open(static_data_path, "w") as f:
f.write(
template.render(start_date=start_date, end_date=end_date))
run(nise_provider_type.lower(), options)
report_path = f"{base_path}/{etag}"
for report in os.scandir(report_path):
if os.path.isdir(report):
continue
self.process_report(report, "PLAIN", provider_type, provider,
manifest)
with patch("masu.processor.tasks.chain"), patch.object(
settings, "AUTO_DATA_INGEST", False):
update_summary_tables(self.schema,
provider_type,
provider.uuid,
start_date,
end_date,
manifest_id=manifest.id)
update_cost_model_costs.s(self.schema,
provider.uuid,
self.dh.last_month_start,
self.dh.today,
synchronous=True).apply()
refresh_materialized_views.s(self.schema,
provider_type,
provider_uuid=provider.uuid,
synchronous=True).apply()
shutil.rmtree(base_path, ignore_errors=True)
def load_openshift_data(self, customer, static_data_file, cluster_id):
"""Load OpenShift data into the database."""
provider_type = Provider.PROVIDER_OCP
credentials = {"cluster_id": cluster_id}
with override_settings(AUTO_DATA_INGEST=False):
ocp_billing_source, _ = ProviderBillingSource.objects.get_or_create(data_source={})
provider = baker.make(
"Provider",
type=provider_type,
authentication__credentials=credentials,
billing_source=ocp_billing_source,
customer=customer,
)
template, static_data_path = self.prepare_template(provider_type, static_data_file)
options = {
"static_report_file": static_data_path,
"insights_upload": self.nise_data_path,
"ocp_cluster_id": cluster_id,
}
base_path = f"{self.nise_data_path}/{cluster_id}"
for start_date, end_date, bill_date in self.dates:
manifest = baker.make(
"CostUsageReportManifest",
_fill_optional=True,
provider=provider,
billing_period_start_datetime=bill_date,
num_total_files=3,
)
with open(static_data_path, "w") as f:
f.write(template.render(start_date=start_date, end_date=end_date))
run(provider_type.lower(), options)
report_path = self.build_report_path(provider_type, bill_date, base_path)
for report in os.scandir(report_path):
shutil.move(report.path, f"{base_path}/{report.name}")
for report in [f.path for f in os.scandir(base_path)]:
if os.path.isdir(report):
continue
elif "manifest" in report.lower():
continue
self.process_report(report, "PLAIN", provider_type, provider, manifest)
with patch("masu.processor.tasks.chain"):
update_summary_tables(
self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id
)
update_cost_model_costs.s(
self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True
).apply()
refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply()
shutil.rmtree(report_path, ignore_errors=True)
def update_cost_model_costs(request):
"""Update report summary tables in the database."""
params = request.query_params
provider_uuid = params.get("provider_uuid")
schema_name = params.get("schema")
default_start_date = DateHelper().this_month_start.strftime("%Y-%m-%d")
default_end_date = DateHelper().today.strftime("%Y-%m-%d")
start_date = params.get("start_date", default=default_start_date)
end_date = params.get("end_date", default=default_end_date)
queue_name = params.get("queue") or PRIORITY_QUEUE
if provider_uuid is None or schema_name is None:
errmsg = "provider_uuid and schema_name are required parameters."
return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)
if queue_name not in QUEUE_LIST:
errmsg = f"'queue' must be one of {QUEUE_LIST}."
return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)
try:
provider = Provider.objects.get(uuid=provider_uuid)
except Provider.DoesNotExist:
return Response({"Error": "Provider does not exist."}, status=status.HTTP_400_BAD_REQUEST)
LOG.info("Calling update_cost_model_costs async task.")
async_result = chain(
cost_task.s(schema_name, provider_uuid, start_date, end_date, queue_name=queue_name).set(queue=queue_name),
refresh_materialized_views.si(
schema_name, provider.type, provider_uuid=provider_uuid, queue_name=queue_name
).set(queue=queue_name),
).apply_async()
return Response({"Update Cost Model Cost Task ID": str(async_result)})
def update_provider_uuids(self, provider_uuids):
"""Update rate with new provider uuids."""
current_providers_for_instance = []
for rate_map_instance in CostModelMap.objects.filter(
cost_model=self._model):
current_providers_for_instance.append(
str(rate_map_instance.provider_uuid))
providers_to_delete = set(current_providers_for_instance).difference(
provider_uuids)
providers_to_create = set(provider_uuids).difference(
current_providers_for_instance)
all_providers = set(current_providers_for_instance).union(
provider_uuids)
for provider_uuid in providers_to_delete:
CostModelMap.objects.filter(provider_uuid=provider_uuid,
cost_model=self._model).delete()
for provider_uuid in providers_to_create:
# Raise exception if source is already associated with another cost model.
existing_cost_model = CostModelMap.objects.filter(
provider_uuid=provider_uuid)
if existing_cost_model.exists():
cost_model_uuid = existing_cost_model.first().cost_model.uuid
log_msg = f"Source {provider_uuid} is already associated with cost model: {cost_model_uuid}."
LOG.warning(log_msg)
raise CostModelException(log_msg)
CostModelMap.objects.create(cost_model=self._model,
provider_uuid=provider_uuid)
start_date = DateHelper().this_month_start.strftime("%Y-%m-%d")
end_date = DateHelper().today.strftime("%Y-%m-%d")
for provider_uuid in all_providers:
# Update cost-model costs for each provider, on every PUT/DELETE
try:
provider = Provider.objects.get(uuid=provider_uuid)
except Provider.DoesNotExist:
LOG.info(
f"Provider {provider_uuid} does not exist. Skipping cost-model update."
)
else:
schema_name = provider.customer.schema_name
# Because this is triggered from the UI, we use the priority queue
chain(
update_cost_model_costs.s(
schema_name,
provider.uuid,
start_date,
end_date,
queue_name=PRIORITY_QUEUE).set(queue=PRIORITY_QUEUE),
refresh_materialized_views.si(
schema_name,
provider.type,
provider_uuid=provider.uuid,
queue_name=PRIORITY_QUEUE).set(queue=PRIORITY_QUEUE),
).apply_async()
