def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None): """Load Azure data into the database.""" provider_type = Provider.PROVIDER_AZURE_LOCAL nise_provider_type = provider_type.replace("-local", "") report_name = "Test" if credentials is None: credentials = { "subscription_id": "11111111-1111-1111-1111-11111111", "tenant_id": "22222222-2222-2222-2222-22222222", "client_id": "33333333-3333-3333-3333-33333333", "client_secret": "MyPassW0rd!", } if data_source is None: data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "azure_report_name": report_name, "azure_container_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): continue elif "manifest" in report.name.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None): """Load AWS data into the database.""" provider_type = Provider.PROVIDER_AWS_LOCAL if account_id is None: account_id = "9999999999999" if role_arn is None: role_arn = "arn:aws:iam::999999999999:role/CostManagement" nise_provider_type = provider_type.replace("-local", "") report_name = "Test" credentials = {"role_arn": role_arn} data_source = {"bucket": "test-bucket"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "aws_report_name": report_name, "aws_bucket_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" with schema_context(self.schema): baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account") for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "GZIP", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL nise_provider_type = provider_type.replace("-local", "") credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) etag = uuid4() template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "gcp_bucket_name": self.nise_data_path, "gcp_etag": etag } base_path = f"{self.nise_data_path}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = f"{base_path}/{etag}" for report in os.scandir(report_path): if os.path.isdir(report): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object( settings, "AUTO_DATA_INGEST", False): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs.s(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_openshift_data(self, customer, static_data_file, cluster_id): """Load OpenShift data into the database.""" provider_type = Provider.PROVIDER_OCP credentials = {"cluster_id": cluster_id} with override_settings(AUTO_DATA_INGEST=False): ocp_billing_source, _ = ProviderBillingSource.objects.get_or_create(data_source={}) provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source=ocp_billing_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "insights_upload": self.nise_data_path, "ocp_cluster_id": cluster_id, } base_path = f"{self.nise_data_path}/{cluster_id}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, num_total_files=3, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): shutil.move(report.path, f"{base_path}/{report.name}") for report in [f.path for f in os.scandir(base_path)]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(report_path, ignore_errors=True)
def provider_post_delete_callback(*args, **kwargs): """ Asynchronously delete this Provider's archived data. Note: Signal receivers must accept keyword arguments (**kwargs). """ provider = kwargs["instance"] if provider.authentication: auth_count = (Provider.objects.exclude(uuid=provider.uuid).filter( authentication=provider.authentication).count()) if auth_count == 0: provider.authentication.delete() if provider.billing_source: billing_count = (Provider.objects.exclude(uuid=provider.uuid).filter( billing_source=provider.billing_source).count()) if billing_count == 0: provider.billing_source.delete() provider_rate_objs = CostModelMap.objects.filter( provider_uuid=provider.uuid) if provider_rate_objs: provider_rate_objs.delete() if not provider.customer: LOG.warning( "Provider %s has no Customer; we cannot call delete_archived_data.", provider.uuid) return customer = provider.customer customer.date_updated = DateHelper().now_utc customer.save() if settings.ENABLE_S3_ARCHIVING: # Local import of task function to avoid potential import cycle. from masu.celery.tasks import delete_archived_data delete_func = partial(delete_archived_data.delay, provider.customer.schema_name, provider.type, provider.uuid) transaction.on_commit(delete_func) refresh_materialized_views.s(provider.customer.schema_name, provider.type, provider_uuid=provider.uuid, synchronous=True).apply()