def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.test_report_path = './koku/masu/test/data/gcp/evidence-2019-06-03.csv' cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map
def setUp(self): """Setup up shared variables.""" super().setUp() self.dh = DateHelper() manifest_id = 1 with ReportManifestDBAccessor() as manifest_accessor: self.manifest = manifest_accessor.get_manifest_by_id(manifest_id) self.updater = AWSReportParquetSummaryUpdater(self.schema_name, self.aws_provider, self.manifest)
def setUpClass(cls): """Set up class variables.""" super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = "koku-local" cls.fake_bucket_prefix = PREFIX cls.selected_region = REGION cls.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.accessor = AzureReportDBAccessor("acct10001") cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema) cls.date_accessor = DateHelper() cls.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls): super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = REPORT cls.fake_bucket_prefix = PREFIX cls.fake_bucket_name = BUCKET cls.selected_region = REGION cls.auth_credential = fake_arn(service='iam', generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor()
def _process_manifest_db_record(self, assembly_id, billing_start, num_of_files): """Insert or update the manifest DB record.""" LOG.info("Inserting/updating manifest in database for assembly_id: %s", assembly_id) with ReportManifestDBAccessor() as manifest_accessor: manifest_entry = manifest_accessor.get_manifest( assembly_id, self._provider_uuid) if not manifest_entry: msg = f"No manifest entry found in database. Adding for bill period start: {billing_start}" LOG.info(log_json(self.request_id, msg, self.context)) manifest_dict = { "assembly_id": assembly_id, "billing_period_start_datetime": billing_start, "num_total_files": num_of_files, "provider_uuid": self._provider_uuid, } try: manifest_entry = manifest_accessor.add(**manifest_dict) except IntegrityError as error: msg = ( f"Manifest entry uniqueness collision: Error {error}. " "Manifest already added, getting manifest_entry_id.") LOG.warning(log_json(self.request_id, msg, self.context)) with ReportManifestDBAccessor() as manifest_accessor: manifest_entry = manifest_accessor.get_manifest( assembly_id, self._provider_uuid) if not manifest_entry: msg = f"Manifest entry not found for given manifest {manifest_dict}." with ProviderDBAccessor( self._provider_uuid) as provider_accessor: provider = provider_accessor.get_provider() if not provider: msg = f"Provider entry not found for {self._provider_uuid}." LOG.warning(log_json(self.request_id, msg, self.context)) raise IntegrityError(msg) else: manifest_accessor.mark_manifest_as_updated(manifest_entry) manifest_id = manifest_entry.id return manifest_id
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.accessor = AWSReportDBAccessor(cls.schema) cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AWS_CUR_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema) cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor()
def test_check_if_manifest_should_be_downloaded_done_processing_manifest(self, _): """Test that a manifest that has finished processing is not reprocessed.""" with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(self.manifest_id) manifest.num_processed_files = 2 manifest.num_total_files = 2 manifest.save() result = self.downloader.check_if_manifest_should_be_downloaded(self.assembly_id) self.assertFalse(result)
def summarize_reports(reports_to_summarize, queue_name=None): """ Summarize reports returned from line summary task. Args: reports_to_summarize (list) list of reports to process Returns: None """ reports_to_summarize = [ report for report in reports_to_summarize if report ] reports_deduplicated = [ dict(t) for t in {tuple(d.items()) for d in reports_to_summarize} ] for report in reports_deduplicated: # For day-to-day summarization we choose a small window to # cover new data from a window of days. # This saves us from re-summarizing unchanged data and cuts down # on processing time. There are override mechanisms in the # Updater classes for when full-month summarization is # required. with ReportManifestDBAccessor() as manifest_accesor: if manifest_accesor.manifest_ready_for_summary( report.get("manifest_id")): if report.get("start") and report.get("end"): LOG.info("using start and end dates from the manifest") start_date = parser.parse( report.get("start")).strftime("%Y-%m-%d") end_date = parser.parse( report.get("end")).strftime("%Y-%m-%d") else: LOG.info("generating start and end dates for manifest") start_date = DateAccessor().today() - datetime.timedelta( days=2) start_date = start_date.strftime("%Y-%m-%d") end_date = DateAccessor().today().strftime("%Y-%m-%d") msg = f"report to summarize: {str(report)}" tracing_id = report.get( "tracing_id", report.get("manifest_uuid", "no-tracing-id")) LOG.info(log_json(tracing_id, msg)) update_summary_tables.s( report.get("schema_name"), report.get("provider_type"), report.get("provider_uuid"), start_date=start_date, end_date=end_date, manifest_id=report.get("manifest_id"), queue_name=queue_name, tracing_id=tracing_id, ).apply_async(queue=queue_name or UPDATE_SUMMARY_TABLES_QUEUE)
def setUpClass(cls): """Set up the test class.""" billing_start = DateAccessor().today_with_timezone('UTC').replace( day=1) cls.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': 1 } cls.manifest_accessor = ReportManifestDBAccessor()
def _should_process_full_month(self): """Determine if we should process the full month of data.""" if not self._manifest_id: log_statement = ( f"No manifest provided, processing as a new billing period.\n" f" Processing entire month.\n" f" schema_name: {self._schema},\n" f" provider_uuid: {self._provider_uuid},\n" f" manifest_id: {self._manifest_id}" ) LOG.info(log_statement) return True with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(self._manifest_id) bill_date = manifest.billing_period_start_datetime.date() provider_uuid = manifest.provider_id log_statement = ( f"Processing bill starting on {bill_date}.\n" f" Processing entire month.\n" f" schema_name: {self._schema},\n" f" provider_uuid: {self._provider_uuid},\n" f" manifest_id: {self._manifest_id}" ) if (bill_date.month != self.data_cutoff_date.month) or ( bill_date.year != self.data_cutoff_date.year and bill_date.month == self.data_cutoff_date.month ): LOG.info(log_statement) return True manifest_list = manifest_accessor.get_manifest_list_for_provider_and_bill_date(provider_uuid, bill_date) if len(manifest_list) == 1: # This is the first manifest for this bill and we are currently # processing it LOG.info(log_statement) return True for manifest in manifest_list: if manifest.num_processed_files >= manifest.num_total_files: log_statement = ( f"Processing bill starting on {bill_date}.\n" f" Processing data on or after {self.data_cutoff_date}.\n" f" schema_name: {self._schema},\n" f" provider_uuid: {self._provider_uuid},\n" f" manifest_id: {self._manifest_id}" ) LOG.info(log_statement) # We have fully processed a manifest for this provider return False return True
def _get_existing_manifest_db_id(self, assembly_id): """Return a manifest DB object if it exists.""" manifest_id = None with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest( assembly_id, self._provider_id ) if manifest: manifest_id = manifest.id return manifest_id
def download_file(self, key, stored_etag=None, manifest_id=None, start_date=None): """ Download an S3 object to file. Args: key (str): The S3 object key identified. Returns: (String): The path and file name of the saved file """ local_s3_filename = utils.get_local_file_name(key) directory_path = f"{DATA_DIR}/{self.customer_name}/aws-local/{self.bucket}" full_file_path = f"{directory_path}/{local_s3_filename}" if not os.path.isfile(key): log_msg = f"Unable to locate {key} in {self.bucket_path}" raise AWSReportDownloaderNoFileError(log_msg) # Make sure the data directory exists os.makedirs(directory_path, exist_ok=True) s3_etag_hasher = hashlib.new("ripemd160") s3_etag_hasher.update(bytes(local_s3_filename, "utf-8")) s3_etag = s3_etag_hasher.hexdigest() file_creation_date = None if s3_etag != stored_etag or not os.path.isfile(full_file_path): msg = f"Downloading {key} to {full_file_path}" LOG.info(log_json(self.tracing_id, msg, self.context)) shutil.copy2(key, full_file_path) file_creation_date = datetime.datetime.fromtimestamp( os.path.getmtime(full_file_path)) # Push to S3 s3_csv_path = get_path_prefix(self.account, Provider.PROVIDER_AWS, self._provider_uuid, start_date, Config.CSV_DATA_TYPE) utils.copy_local_report_file_to_s3_bucket( self.tracing_id, s3_csv_path, full_file_path, local_s3_filename, manifest_id, start_date, self.context) manifest_accessor = ReportManifestDBAccessor() manifest = manifest_accessor.get_manifest_by_id(manifest_id) if not manifest_accessor.get_s3_csv_cleared(manifest): utils.remove_files_not_in_set_from_s3_bucket( self.tracing_id, s3_csv_path, manifest_id) manifest_accessor.mark_s3_csv_cleared(manifest) return full_file_path, s3_etag, file_creation_date, []
def test_is_last_completed_datetime_null(self): """Test is last completed datetime is null.""" manifest_id = 123456789 self.assertTrue( ReportManifestDBAccessor().is_last_completed_datetime_null( manifest_id)) baker.make(CostUsageReportManifest, id=manifest_id) baker.make(CostUsageReportStatus, manifest_id=manifest_id, last_completed_datetime=None) self.assertTrue( ReportManifestDBAccessor().is_last_completed_datetime_null( manifest_id)) CostUsageReportStatus.objects.filter(manifest_id=manifest_id).update( last_completed_datetime=FAKE.date()) self.assertFalse( ReportManifestDBAccessor().is_last_completed_datetime_null( manifest_id))
def setUpClass(cls): """Set up class variables.""" super().setUpClass() cls.fake_customer_name = CUSTOMER_NAME cls.fake_report_name = "koku-local" cls.fake_bucket_prefix = PREFIX cls.selected_region = REGION cls.fake_auth_credential = fake_arn(service="iam", generate_account_id=True) cls.manifest_accessor = ReportManifestDBAccessor()
def test_clean_volume(self, mock_date, mock_config): """Test that the clean volume function is cleaning the appropriate files""" # create a manifest mock_date.return_value = ["2020-02-01"] manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": "2020-02-01", "num_total_files": 2, "provider_uuid": self.aws_provider_uuid, } manifest_accessor = ReportManifestDBAccessor() manifest = manifest_accessor.add(**manifest_dict) # create two files on the temporary volume one with a matching prefix id # as the assembly_id in the manifest above with tempfile.TemporaryDirectory() as tmpdirname: mock_config.PVC_DIR = tmpdirname mock_config.VOLUME_FILE_RETENTION = 60 * 60 * 24 old_matching_file = os.path.join(tmpdirname, "%s.csv" % manifest.assembly_id) new_no_match_file = os.path.join(tmpdirname, "newfile.csv") old_no_match_file = os.path.join(tmpdirname, "oldfile.csv") filepaths = [old_matching_file, new_no_match_file, old_no_match_file] for path in filepaths: open(path, "a").close() self.assertEqual(os.path.exists(path), True) # Update timestame for oldfile.csv datehelper = DateHelper() now = datehelper.now old_datetime = now - timedelta(seconds=mock_config.VOLUME_FILE_RETENTION * 2) oldtime = old_datetime.timestamp() os.utime(old_matching_file, (oldtime, oldtime)) os.utime(old_no_match_file, (oldtime, oldtime)) # now run the clean volume task tasks.clean_volume() # make sure that the file with the matching id still exists and that # the file with the other id is gone self.assertEqual(os.path.exists(old_matching_file), True) self.assertEqual(os.path.exists(new_no_match_file), True) self.assertEqual(os.path.exists(old_no_match_file), False) # now edit the manifest to say that all the files have been processed # and rerun the clean_volumes task manifest.num_processed_files = manifest_dict.get("num_total_files") manifest.save() tasks.clean_volume() # ensure that the original file is deleted from the volume self.assertEqual(os.path.exists(old_matching_file), False) self.assertEqual(os.path.exists(new_no_match_file), True) # assert the tempdir is cleaned up self.assertEqual(os.path.exists(tmpdirname), False) # test no files found for codecov tasks.clean_volume()
def setUp(self): """Set up the test class.""" super().setUp() self.schema = self.schema_name billing_start = DateAccessor().today_with_timezone('UTC').replace(day=1) self.manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_uuid': self.provider_uuid, } self.manifest_accessor = ReportManifestDBAccessor()
def download_file(self, key, stored_etag=None, manifest_id=None, start_date=None): """ Download a file from Azure bucket. Args: key (str): The object key identified. Returns: (String): The path and file name of the saved file """ local_filename = utils.get_local_file_name(key) full_file_path = f"{self._get_exports_data_directory()}/{local_filename}" file_creation_date = None try: blob = self._azure_client.get_cost_export_for_key( key, self.container_name) etag = blob.etag file_creation_date = blob.last_modified except AzureCostReportNotFound as ex: msg = f"Error when downloading Azure report for key: {key}. Error {ex}" LOG.error(log_json(self.tracing_id, msg, self.context)) raise AzureReportDownloaderError(msg) msg = f"Downloading {key} to {full_file_path}" LOG.info(log_json(self.tracing_id, msg, self.context)) blob = self._azure_client.download_cost_export( key, self.container_name, destination=full_file_path) # Push to S3 s3_csv_path = get_path_prefix(self.account, Provider.PROVIDER_AZURE, self._provider_uuid, start_date, Config.CSV_DATA_TYPE) copy_local_report_file_to_s3_bucket(self.tracing_id, s3_csv_path, full_file_path, local_filename, manifest_id, start_date, self.context) manifest_accessor = ReportManifestDBAccessor() manifest = manifest_accessor.get_manifest_by_id(manifest_id) if not manifest_accessor.get_s3_csv_cleared(manifest): remove_files_not_in_set_from_s3_bucket(self.tracing_id, s3_csv_path, manifest_id) manifest_accessor.mark_s3_csv_cleared(manifest) msg = f"Returning full_file_path: {full_file_path}, etag: {etag}" LOG.info(log_json(self.tracing_id, msg, self.context)) return full_file_path, etag, file_creation_date, []
def test_check_if_manifest_should_be_downloaded_error_no_complete_date(self, _): """Test that a manifest that did not succeessfully process should be reprocessed.""" with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(self.manifest_id) manifest.num_processed_files = 1 manifest.num_total_files = 2 manifest.save() with ReportStatsDBAccessor(self.report_name, self.manifest_id) as file_accessor: file_accessor.log_last_started_datetime() result = self.downloader.check_if_manifest_should_be_downloaded(self.assembly_id) self.assertTrue(result)
def tearDown(self): """Tear down each test case.""" super().tearDown() with ReportStatsDBAccessor(self.report_name, self.manifest_id) as file_accessor: files = file_accessor._get_db_obj_query().all() for file in files: file_accessor.delete(file) with ReportManifestDBAccessor() as manifest_accessor: manifests = manifest_accessor._get_db_obj_query().all() for manifest in manifests: manifest_accessor.delete(manifest)
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() with ReportingCommonDBAccessor() as report_common_db: cls.column_map = report_common_db.column_map cls.accessor = AzureReportDBAccessor("acct10001", cls.column_map) cls.report_schema = cls.accessor.report_schema cls.all_tables = list(AZURE_REPORT_TABLE_MAP.values()) cls.creator = ReportObjectCreator(cls.schema, cls.column_map) cls.date_accessor = DateAccessor() cls.manifest_accessor = ReportManifestDBAccessor()
def setUp(self): """Set up GCP tests.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f"{self.temp_dir}/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv" shutil.copy2(self.test_report_path, self.test_report) gcp_auth = ProviderAuthentication.objects.create( credentials={"project-id": fake.word()}) gcp_billing_source = ProviderBillingSource.objects.create( data_source={"bucket": fake.word()}) with patch("masu.celery.tasks.check_report_updates"): self.gcp_provider = Provider.objects.create( uuid=uuid.uuid4(), name="Test Provider", type=Provider.PROVIDER_GCP, authentication=gcp_auth, billing_source=gcp_billing_source, customer=self.customer, setup_complete=True, ) start_time = "2020-11-08 23:00:00+00:00" report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split("-") self.start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.assembly_id = "1234" self.manifest_dict = { "assembly_id": self.assembly_id, "billing_period_start_datetime": self.start_date_utc, "num_total_files": 1, "provider_uuid": self.gcp_provider.uuid, } manifest_accessor = ReportManifestDBAccessor() self.manifest = manifest_accessor.add(**self.manifest_dict) self.processor = GCPReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.gcp_provider.uuid, manifest_id=self.manifest.id, ) self.accessor = GCPReportDBAccessor(self.schema)
def setUp(self): """Set up the test class.""" super().setUp() self.schema = self.schema_name billing_start = DateAccessor().today_with_timezone("UTC").replace( day=1) self.manifest_dict = { "assembly_id": "1234", "billing_period_start_datetime": billing_start, "num_total_files": 2, "provider_uuid": self.provider_uuid, } self.manifest_accessor = ReportManifestDBAccessor()
def setUpClass(cls): """Set up the test class.""" billing_start = datetime.utcnow().replace(day=1) manifest_dict = { 'assembly_id': '1234', 'billing_period_start_datetime': billing_start, 'num_total_files': 2, 'provider_id': 1 } cls.manifest_accessor = ReportManifestDBAccessor() manifest = cls.manifest_accessor.add(**manifest_dict) cls.manifest_accessor.commit() cls.manifest_id = manifest.id
def download_file(self, key, stored_etag=None, manifest_id=None, start_date=None): """ Download a file from Azure bucket. Args: key (str): The object key identified. Returns: (String): The path and file name of the saved file """ local_filename = utils.get_local_file_name(key) full_file_path = f"{self._get_exports_data_directory()}/{local_filename}" etag_hasher = hashlib.new("ripemd160") etag_hasher.update(bytes(local_filename, "utf-8")) etag = etag_hasher.hexdigest() file_creation_date = None if etag != stored_etag: msg = f"Downloading {key} to {full_file_path}" LOG.info(log_json(self.request_id, msg, self.context)) shutil.copy2(key, full_file_path) file_creation_date = datetime.datetime.fromtimestamp( os.path.getmtime(full_file_path)) # Push to S3 s3_csv_path = get_path_prefix(self.account, Provider.PROVIDER_AZURE, self._provider_uuid, start_date, Config.CSV_DATA_TYPE) copy_local_report_file_to_s3_bucket(self.request_id, s3_csv_path, full_file_path, local_filename, manifest_id, start_date, self.context) manifest_accessor = ReportManifestDBAccessor() manifest = manifest_accessor.get_manifest_by_id(manifest_id) if not manifest_accessor.get_s3_csv_cleared(manifest): remove_files_not_in_set_from_s3_bucket(self.request_id, s3_csv_path, manifest_id) manifest_accessor.mark_s3_csv_cleared(manifest) msg = f"Returning full_file_path: {full_file_path}, etag: {etag}" LOG.info(log_json(self.request_id, msg, self.context)) return full_file_path, etag, file_creation_date, []
def refresh_materialized_views(schema_name, provider_type, manifest_id=None, provider_uuid=None, synchronous=False): """Refresh the database's materialized views for reporting.""" task_name = "masu.processor.tasks.refresh_materialized_views" cache_args = [schema_name] if not synchronous: worker_cache = WorkerCache() while worker_cache.single_task_is_running(task_name, cache_args): time.sleep(5) worker_cache.lock_single_task(task_name, cache_args) materialized_views = () if provider_type in (Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL): materialized_views = (AWS_MATERIALIZED_VIEWS + OCP_ON_AWS_MATERIALIZED_VIEWS + OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) elif provider_type in (Provider.PROVIDER_OCP): materialized_views = (OCP_MATERIALIZED_VIEWS + OCP_ON_AWS_MATERIALIZED_VIEWS + OCP_ON_AZURE_MATERIALIZED_VIEWS + OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) elif provider_type in (Provider.PROVIDER_AZURE, Provider.PROVIDER_AZURE_LOCAL): materialized_views = (AZURE_MATERIALIZED_VIEWS + OCP_ON_AZURE_MATERIALIZED_VIEWS + OCP_ON_INFRASTRUCTURE_MATERIALIZED_VIEWS) with schema_context(schema_name): for view in materialized_views: table_name = view._meta.db_table with connection.cursor() as cursor: cursor.execute( f"REFRESH MATERIALIZED VIEW CONCURRENTLY {table_name}") LOG.info(f"Refreshed {table_name}.") invalidate_view_cache_for_tenant_and_source_type(schema_name, provider_type) if provider_uuid: ProviderDBAccessor(provider_uuid).set_data_updated_timestamp() if manifest_id: # Processing for this monifest should be complete after this step with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(manifest_id) manifest_accessor.mark_manifest_as_completed(manifest) if not synchronous: worker_cache.release_single_task(task_name, cache_args)
def setUp(self): """Set up GCP tests.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f'{self.temp_dir}/evidence-2019-06-03.csv' shutil.copy2(self.test_report_path, self.test_report) gcp_auth = ProviderAuthentication.objects.create( credentials={'project-id': fake.word()}) gcp_billing_source = ProviderBillingSource.objects.create( data_source={'bucket': fake.word()}) self.gcp_provider = Provider.objects.create( uuid=uuid.uuid4(), name='Test Provider', type=Provider.PROVIDER_GCP, authentication=gcp_auth, billing_source=gcp_billing_source, customer=self.customer, setup_complete=True, ) start_time = '2019-09-17T00:00:00-07:00' report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split('-') self.start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': self.start_date_utc, 'num_total_files': 1, 'provider_uuid': self.gcp_provider.uuid, } manifest_accessor = ReportManifestDBAccessor() self.manifest = manifest_accessor.add(**self.manifest_dict) self.processor = GCPReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.gcp_provider.uuid, manifest_id=self.manifest.id, ) self.accessor = GCPReportDBAccessor(self.schema, self.column_map)
def test_refresh_materialized_views(self): """Test that materialized views are refreshed.""" manifest_dict = { 'assembly_id': '12345', 'billing_period_start_datetime': DateAccessor().today_with_timezone('UTC'), 'num_total_files': 2, 'provider_uuid': self.aws_provider_uuid, 'task': '170653c0-3e66-4b7e-a764-336496d7ca5a', } fake_aws = FakeAWSCostData(self.aws_provider) generator = AWSReportDataGenerator(self.tenant) generator.add_data_to_tenant(fake_aws) with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.add(**manifest_dict) manifest.save() refresh_materialized_views(self.schema, Provider.PROVIDER_AWS, manifest_id=manifest.id) views_to_check = [ view for view in AWS_MATERIALIZED_VIEWS if 'Cost' in view._meta.db_table ] with schema_context(self.schema): for view in views_to_check: self.assertNotEqual(view.objects.count(), 0) with ReportManifestDBAccessor() as manifest_accessor: manifest = manifest_accessor.get_manifest_by_id(manifest.id) self.assertIsNotNone(manifest.manifest_completed_datetime)
def setUpClass(cls): """Set up the test class with required objects.""" super().setUpClass() cls.accessor = GCPReportDBAccessor(schema=cls.schema) cls.report_schema = cls.accessor.report_schema cls.all_tables = list(GCP_REPORT_TABLE_MAP.values()) cls.foreign_key_tables = [ GCP_REPORT_TABLE_MAP["bill"], GCP_REPORT_TABLE_MAP["product"], GCP_REPORT_TABLE_MAP["project"], ] cls.manifest_accessor = ReportManifestDBAccessor()
def download_file(self, key, stored_etag=None, manifest_id=None, start_date=None): """ Download an OCP usage file. Args: key (str): The OCP file name. Returns: (String): The path and file name of the saved file """ if not self.manifest: self.manifest = ReportManifestDBAccessor().get_manifest_by_id( manifest_id) self.context["version"] = self.manifest.operator_version local_filename = utils.get_local_file_name(key) directory_path = f"{DATA_DIR}/{self.customer_name}/ocp/{self.cluster_id}" full_file_path = f"{directory_path}/{local_filename}" # Make sure the data directory exists os.makedirs(directory_path, exist_ok=True) etag_hasher = hashlib.new("ripemd160") etag_hasher.update(bytes(local_filename, "utf-8")) ocp_etag = etag_hasher.hexdigest() file_creation_date = None if ocp_etag != stored_etag or not os.path.isfile(full_file_path): msg = f"Downloading {key} to {full_file_path}" LOG.info(log_json(self.request_id, msg, self.context)) shutil.move(key, full_file_path) file_creation_date = datetime.datetime.fromtimestamp( os.path.getmtime(full_file_path)) file_names = create_daily_archives( self.request_id, self.account, self._provider_uuid, local_filename, full_file_path, manifest_id, start_date, self.context, ) return full_file_path, ocp_etag, file_creation_date, file_names