def load_gcp_data(self, customer): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply()
def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None): """Load Azure data into the database.""" provider_type = Provider.PROVIDER_AZURE_LOCAL nise_provider_type = provider_type.replace("-local", "") report_name = "Test" if credentials is None: credentials = { "subscription_id": "11111111-1111-1111-1111-11111111", "tenant_id": "22222222-2222-2222-2222-22222222", "client_id": "33333333-3333-3333-3333-33333333", "client_secret": "MyPassW0rd!", } if data_source is None: data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "azure_report_name": report_name, "azure_container_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): continue elif "manifest" in report.name.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None): """Load AWS data into the database.""" provider_type = Provider.PROVIDER_AWS_LOCAL if account_id is None: account_id = "9999999999999" if role_arn is None: role_arn = "arn:aws:iam::999999999999:role/CostManagement" nise_provider_type = provider_type.replace("-local", "") report_name = "Test" credentials = {"role_arn": role_arn} data_source = {"bucket": "test-bucket"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "aws_report_name": report_name, "aws_bucket_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" with schema_context(self.schema): baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account") for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "GZIP", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL nise_provider_type = provider_type.replace("-local", "") credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) etag = uuid4() template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "gcp_bucket_name": self.nise_data_path, "gcp_etag": etag } base_path = f"{self.nise_data_path}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = f"{base_path}/{etag}" for report in os.scandir(report_path): if os.path.isdir(report): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object( settings, "AUTO_DATA_INGEST", False): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs.s(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_openshift_data(self, customer, static_data_file, cluster_id): """Load OpenShift data into the database.""" provider_type = Provider.PROVIDER_OCP credentials = {"cluster_id": cluster_id} with override_settings(AUTO_DATA_INGEST=False): ocp_billing_source, _ = ProviderBillingSource.objects.get_or_create(data_source={}) provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source=ocp_billing_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "insights_upload": self.nise_data_path, "ocp_cluster_id": cluster_id, } base_path = f"{self.nise_data_path}/{cluster_id}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, num_total_files=3, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): shutil.move(report.path, f"{base_path}/{report.name}") for report in [f.path for f in os.scandir(base_path)]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(report_path, ignore_errors=True)
def update_cost_model_costs(request): """Update report summary tables in the database.""" params = request.query_params provider_uuid = params.get("provider_uuid") schema_name = params.get("schema") default_start_date = DateHelper().this_month_start.strftime("%Y-%m-%d") default_end_date = DateHelper().today.strftime("%Y-%m-%d") start_date = params.get("start_date", default=default_start_date) end_date = params.get("end_date", default=default_end_date) queue_name = params.get("queue") or PRIORITY_QUEUE if provider_uuid is None or schema_name is None: errmsg = "provider_uuid and schema_name are required parameters." return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST) if queue_name not in QUEUE_LIST: errmsg = f"'queue' must be one of {QUEUE_LIST}." return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST) try: provider = Provider.objects.get(uuid=provider_uuid) except Provider.DoesNotExist: return Response({"Error": "Provider does not exist."}, status=status.HTTP_400_BAD_REQUEST) LOG.info("Calling update_cost_model_costs async task.") async_result = chain( cost_task.s(schema_name, provider_uuid, start_date, end_date, queue_name=queue_name).set(queue=queue_name), refresh_materialized_views.si( schema_name, provider.type, provider_uuid=provider_uuid, queue_name=queue_name ).set(queue=queue_name), ).apply_async() return Response({"Update Cost Model Cost Task ID": str(async_result)})
def update_provider_uuids(self, provider_uuids): """Update rate with new provider uuids.""" current_providers_for_instance = [] for rate_map_instance in CostModelMap.objects.filter( cost_model=self._model): current_providers_for_instance.append( str(rate_map_instance.provider_uuid)) providers_to_delete = set(current_providers_for_instance).difference( provider_uuids) providers_to_create = set(provider_uuids).difference( current_providers_for_instance) all_providers = set(current_providers_for_instance).union( provider_uuids) for provider_uuid in providers_to_delete: CostModelMap.objects.filter(provider_uuid=provider_uuid, cost_model=self._model).delete() for provider_uuid in providers_to_create: # Raise exception if source is already associated with another cost model. existing_cost_model = CostModelMap.objects.filter( provider_uuid=provider_uuid) if existing_cost_model.exists(): cost_model_uuid = existing_cost_model.first().cost_model.uuid log_msg = f"Source {provider_uuid} is already associated with cost model: {cost_model_uuid}." LOG.warning(log_msg) raise CostModelException(log_msg) CostModelMap.objects.create(cost_model=self._model, provider_uuid=provider_uuid) start_date = DateHelper().this_month_start.strftime("%Y-%m-%d") end_date = DateHelper().today.strftime("%Y-%m-%d") for provider_uuid in all_providers: # Update cost-model costs for each provider, on every PUT/DELETE try: provider = Provider.objects.get(uuid=provider_uuid) except Provider.DoesNotExist: LOG.info( f"Provider {provider_uuid} does not exist. Skipping cost-model update." ) else: schema_name = provider.customer.schema_name # Because this is triggered from the UI, we use the priority queue chain( update_cost_model_costs.s( schema_name, provider.uuid, start_date, end_date, queue_name=PRIORITY_QUEUE).set(queue=PRIORITY_QUEUE), refresh_materialized_views.si( schema_name, provider.type, provider_uuid=provider.uuid, queue_name=PRIORITY_QUEUE).set(queue=PRIORITY_QUEUE), ).apply_async()