def load_azure_data(self, customer, static_data_file, credentials=None, data_source=None): """Load Azure data into the database.""" provider_type = Provider.PROVIDER_AZURE_LOCAL nise_provider_type = provider_type.replace("-local", "") report_name = "Test" if credentials is None: credentials = { "subscription_id": "11111111-1111-1111-1111-11111111", "tenant_id": "22222222-2222-2222-2222-22222222", "client_id": "33333333-3333-3333-3333-33333333", "client_secret": "MyPassW0rd!", } if data_source is None: data_source = {"resource_group": "resourcegroup1", "storage_account": "storageaccount1"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "azure_report_name": report_name, "azure_container_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): continue elif "manifest" in report.name.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_gcp_data(self, customer, static_data_file): """Load GCP data into the database.""" provider_type = Provider.PROVIDER_GCP_LOCAL nise_provider_type = provider_type.replace("-local", "") credentials = {"project_id": "test_project_id"} data_source = {"table_id": "test_table_id", "dataset": "test_dataset"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) etag = uuid4() template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "gcp_bucket_name": self.nise_data_path, "gcp_etag": etag } base_path = f"{self.nise_data_path}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(nise_provider_type.lower(), options) report_path = f"{base_path}/{etag}" for report in os.scandir(report_path): if os.path.isdir(report): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object( settings, "AUTO_DATA_INGEST", False): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs.s(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_aws_data(self, customer, static_data_file, account_id=None, role_arn=None): """Load AWS data into the database.""" provider_type = Provider.PROVIDER_AWS_LOCAL if account_id is None: account_id = "9999999999999" if role_arn is None: role_arn = "arn:aws:iam::999999999999:role/CostManagement" nise_provider_type = provider_type.replace("-local", "") report_name = "Test" credentials = {"role_arn": role_arn} data_source = {"bucket": "test-bucket"} with patch.object(settings, "AUTO_DATA_INGEST", False): provider = baker.make( "Provider", type=provider_type, authentication__credentials=credentials, billing_source__data_source=data_source, customer=customer, ) template, static_data_path = self.prepare_template(provider_type, static_data_file) options = { "static_report_file": static_data_path, "aws_report_name": report_name, "aws_bucket_name": self.nise_data_path, } base_path = f"{self.nise_data_path}/{report_name}" with schema_context(self.schema): baker.make("AWSAccountAlias", account_id=account_id, account_alias="Test Account") for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, ) with open(static_data_path, "w") as f: f.write(template.render(start_date=start_date, end_date=end_date, account_id=account_id)) run(nise_provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): if os.path.isdir(report): for report in [f.path for f in os.scandir(f"{report_path}/{report.name}")]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "GZIP", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"), patch.object(settings, "AUTO_DATA_INGEST", False): update_summary_tables( self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id ) update_cost_model_costs.s( self.schema, provider.uuid, self.dh.last_month_start, self.dh.today, synchronous=True ).apply() refresh_materialized_views.s(self.schema, provider_type, provider_uuid=provider.uuid, synchronous=True).apply() shutil.rmtree(base_path, ignore_errors=True)
def load_openshift_data(self, customer, static_data_file, cluster_id): """Load OpenShift data into the database.""" provider_type = Provider.PROVIDER_OCP with override_settings(AUTO_DATA_INGEST=False): provider = baker.make( "Provider", type=provider_type, authentication__provider_resource_name=cluster_id, billing_source__bucket="", customer=customer, ) template, static_data_path = self.prepare_template( provider_type, static_data_file) options = { "static_report_file": static_data_path, "insights_upload": self.nise_data_path, "ocp_cluster_id": cluster_id, } base_path = f"{self.nise_data_path}/{cluster_id}" for start_date, end_date, bill_date in self.dates: manifest = baker.make( "CostUsageReportManifest", _fill_optional=True, provider=provider, billing_period_start_datetime=bill_date, num_processed_files=0, num_total_files=3, ) with open(static_data_path, "w") as f: f.write( template.render(start_date=start_date, end_date=end_date)) run(provider_type.lower(), options) report_path = self.build_report_path(provider_type, bill_date, base_path) for report in os.scandir(report_path): shutil.move(report.path, f"{base_path}/{report.name}") for report in [f.path for f in os.scandir(base_path)]: if os.path.isdir(report): continue elif "manifest" in report.lower(): continue self.process_report(report, "PLAIN", provider_type, provider, manifest) with patch("masu.processor.tasks.chain"): update_summary_tables(self.schema, provider_type, provider.uuid, start_date, end_date, manifest_id=manifest.id) update_cost_model_costs(self.schema, provider.uuid, self.dh.last_month_start, self.dh.today) refresh_materialized_views(self.schema, provider_type) shutil.rmtree(report_path, ignore_errors=True)
def test_run_for_azure_enddates(self): """That that fix_dates corrects the azure end_date.""" start = date.today().replace(day=1) end = date.today().replace(day=1) args = ["report", "azure", "-s", str(start), "-e", str(end)] parsed_args = self.parser.parse_args(args) options = vars(parsed_args) _, provider_type = _validate_provider_inputs(self.parser, options) self.assertEqual(provider_type, "azure") with patch("nise.__main__.azure_create_report"): run(provider_type, options) self.assertEqual( options.get("end_date").date(), end + timedelta(days=1))