def test_download_report_missing_bucket(self, mock_stats, fake_session): mock_stats.return_value.__enter__ = Mock() fake_report_date = self.fake.date_time().replace(day=1) fake_report_date_str = fake_report_date.strftime('%Y%m%dT000000.000Z') expected_assembly_id = '882083b7-ea62-4aab-aa6a-f0d08d65ee2b' input_key = f'/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz' mock_manifest = { 'assemblyId': expected_assembly_id, 'billingPeriod': { 'start': fake_report_date_str }, 'reportKeys': [input_key] } with patch.object(AWSReportDownloader, '_get_manifest', return_value=mock_manifest): with self.assertRaises(AWSReportDownloaderError): report_downloader = ReportDownloader(self.fake_customer_name, self.auth_credential, self.fake_bucket_name, 'AWS', 2) AWSReportDownloader( **{ 'customer_name': self.fake_customer_name, 'auth_credential': self.auth_credential, 'bucket': self.fake_bucket_name, 'report_name': self.fake_report_name, 'provider_id': 2 }) report_downloader.download_report(fake_report_date)
def setUp(self): super().setUp() self.fake_customer_name = CUSTOMER_NAME self.fake_report_name = 'ocp-report' self.cluster_id = 'my-ocp-cluster-1' report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id, '20180901-20181001') os.makedirs(report_path, exist_ok=True) test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' self.test_file_path = os.path.join(report_path, os.path.basename(test_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_file_path)) test_manifest_path = './koku/masu/test/data/ocp/manifest.json' self.test_manifest_path = os.path.join( report_path, os.path.basename(test_manifest_path)) shutil.copyfile(test_manifest_path, os.path.join(report_path, self.test_manifest_path)) self.report_downloader = ReportDownloader(self.fake_customer_name, self.cluster_id, None, 'OCP', self.ocp_provider_id) self.ocp_report_downloader = OCPReportDownloader( **{ 'customer_name': self.fake_customer_name, 'auth_credential': self.cluster_id, 'bucket': None, 'provider_id': 1, })
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.fake_auth_credential, "bucket": self.fake_bucket_name, "provider_uuid": self.aws_provider_uuid, } )
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket.tar.gz") mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.credentials = {"role_arn": self.fake_auth_credential} self.data_source = {"bucket": self.fake_bucket_name} self.report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source=self.data_source, provider_type=Provider.PROVIDER_AWS_LOCAL, provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, "provider_uuid": self.aws_provider_uuid, })
def test_download_bucket_with_prefix(self): """Test to verify that basic report downloading works.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open( "./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=8, day=7) fake_data_source = {"bucket": fake_bucket} with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.fake_customer_name, self.credentials, fake_data_source, Provider.PROVIDER_AWS_LOCAL, self.aws_provider_uuid, ) # Names from test report .gz file report_context = { "date": test_report_date.date(), "manifest_id": 1, "comporession": "GZIP", "current_file": "./koku/masu/test/data/test_local_bucket.tar.gz", } report_downloader.download_report(report_context) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_bucket)
def setUp(self, fake_session): """Set up shared variables.""" super().setUp() os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock( request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.auth_credential, report_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, cache_key=self.fake.word(), ) self.aws_report_downloader = AWSReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.auth_credential, "bucket": self.fake_bucket_name, "report_name": self.fake_report_name, "provider_uuid": self.aws_provider_uuid, })
def setUp(self): """Set up each test.""" super().setUp() self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open('./koku/masu/test/data/test_local_bucket.tar.gz') mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type='AWS-local', provider_uuid=self.aws_provider_uuid, ) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ 'task': self.mock_task, 'customer_name': self.fake_customer_name, 'auth_credential': self.fake_auth_credential, 'bucket': self.fake_bucket_name, 'provider_uuid': self.aws_provider_uuid, } )
def test_download_report_missing_bucket(self, mock_stats, fake_session): """Test download fails when bucket is missing.""" mock_stats.return_value.__enter__ = Mock() fake_report_date = self.fake.date_time().replace(day=1) fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z") expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b" input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz" mock_manifest = { "assemblyId": expected_assembly_id, "billingPeriod": { "start": fake_report_date_str }, "reportKeys": [input_key], } with patch.object(AWSReportDownloader, "_get_manifest", return_value=("", mock_manifest)): with self.assertRaises(AWSReportDownloaderError): report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source=self.data_source, provider_type=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, ) report_context = { "date": fake_report_date.date(), "manifest_id": 1, "comporession": "GZIP", "current_file": "/my/file", } report_downloader.download_report(report_context)
def test_download_report_missing_bucket(self, mock_stats, fake_session): """Test download fails when bucket is missing.""" mock_stats.return_value.__enter__ = Mock() fake_report_date = self.fake.date_time().replace(day=1) fake_report_date_str = fake_report_date.strftime("%Y%m%dT000000.000Z") expected_assembly_id = "882083b7-ea62-4aab-aa6a-f0d08d65ee2b" input_key = f"/koku/20180701-20180801/{expected_assembly_id}/koku-1.csv.gz" mock_manifest = { "assemblyId": expected_assembly_id, "billingPeriod": {"start": fake_report_date_str}, "reportKeys": [input_key], } with patch.object(AWSReportDownloader, "_get_manifest", return_value=("", mock_manifest)): with self.assertRaises(AWSReportDownloaderError): report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.auth_credential, report_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AWS, provider_uuid=self.aws_provider_uuid, ) AWSReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.auth_credential, "bucket": self.fake_bucket_name, "report_name": self.fake_report_name, "provider_uuid": self.aws_provider_uuid, } ) report_downloader.download_report(fake_report_date)
def setUp(self): """Set up each test.""" super().setUp() self.customer_name = 'testcustomer' self.local_storage = tempfile.mkdtemp() self.container_name = 'my_container' self.directory = 'dir' self.export_name = 'myexport' self.date_range = '20190801-20190831' self.fake_auth_credential = {"credentials": { "subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc", "tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7", "client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa", "client_secret": "ahhhhh" } } self.fake_bucket_name = { "resource_group": { "export_name": self.export_name, "directory": self.directory }, "storage_account": { "local_dir": self.local_storage, "container": self.container_name } } test_report = './koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv' local_dir = '{}/{}/{}/{}/{}'.format(self.local_storage, self.container_name, self.directory, self.export_name, self.date_range) os.makedirs(local_dir) self.csv_file_name = test_report.split('/')[-1] self.csv_key = f'{local_dir}/{self.csv_file_name}' shutil.copy2(test_report, self.csv_key) os.makedirs(DATA_DIR, exist_ok=True) self.mock_task = Mock(request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.customer_name, access_credential=self.fake_auth_credential, report_source=self.fake_bucket_name, provider_type='AZURE-local', provider_uuid=self.azure_provider_uuid, ) self.azure_local_report_downloader = AzureLocalReportDownloader( **{ 'task': self.mock_task, 'customer_name': self.customer_name, 'auth_credential': self.fake_auth_credential, 'billing_source': self.fake_bucket_name, 'bucket': self.fake_bucket_name, 'provider_uuid': self.azure_provider_uuid, } )
def _get_report_files(task, customer_name, authentication, billing_source, provider_type, provider_uuid, report_month): """ Task to download a Report. Args: task (Object): Bound celery task. customer_name (String): Name of the customer owning the cost usage report. access_credential (String): Credential needed to access cost usage report in the backend provider. report_source (String): Location of the cost usage report in the backend provider. provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS' provider_uuid (String): Provider uuid. report_month (DateTime): Month for report to download. Returns: files (List) List of filenames with full local path. Example: ['/var/tmp/masu/region/aws/catch-clearly.csv', '/var/tmp/masu/base/aws/professor-hour-industry-television.csv'] """ month_string = report_month.strftime("%B %Y") log_statement = (f"Downloading report for:\n" f" schema_name: {customer_name}\n" f" provider: {provider_type}\n" f" account (provider uuid): {provider_uuid}\n" f" report_month: {month_string}") LOG.info(log_statement) try: disk = psutil.disk_usage(Config.PVC_DIR) disk_msg = f"Available disk space: {disk.free} bytes ({100 - disk.percent}%)" except OSError: disk_msg = f"Unable to find available disk space. {Config.PVC_DIR} does not exist" LOG.info(disk_msg) reports = None try: downloader = ReportDownloader( task=task, customer_name=customer_name, access_credential=authentication, report_source=billing_source, provider_type=provider_type, provider_uuid=provider_uuid, report_name=None, ) reports = downloader.download_report(report_month) except (MasuProcessingError, MasuProviderError, ReportDownloaderError) as err: worker_stats.REPORT_FILE_DOWNLOAD_ERROR_COUNTER.labels( provider_type=provider_type).inc() LOG.error(str(err)) with ProviderStatus(provider_uuid) as status: status.set_error(error=err) raise err with ProviderStatus(provider_uuid) as status: status.set_status(ProviderStatusCode.READY) return reports
def setUp(self): """Set up each test.""" super().setUp() self.customer_name = "testcustomer" self.local_storage = tempfile.mkdtemp() self.container_name = "my_container" self.directory = "dir" self.export_name = "myexport" self.date_range = "20190801-20190831" self.fake_auth_credential = { "credentials": { "subscription_id": "2639de71-ca37-4a17-a104-17665a51e7fc", "tenant_id": "ae4f8f55-f1a8-4080-9aa8-10779e4113f7", "client_id": "d6b607d7-d07a-4ca0-b81d-39631f7323aa", "client_secret": "ahhhhh", } } self.fake_bucket_name = { "resource_group": { "export_name": self.export_name, "directory": self.directory }, "storage_account": { "local_dir": self.local_storage, "container": self.container_name }, } test_report = "./koku/masu/test/data/azure/costreport_a243c6f2-199f-4074-9a2c-40e671cf1584.csv" local_dir = "{}/{}/{}/{}/{}".format(self.local_storage, self.container_name, self.directory, self.export_name, self.date_range) os.makedirs(local_dir) self.csv_file_name = test_report.split("/")[-1] self.csv_key = f"{local_dir}/{self.csv_file_name}" shutil.copy2(test_report, self.csv_key) os.makedirs(DATA_DIR, exist_ok=True) self.report_downloader = ReportDownloader( customer_name=self.customer_name, credentials=self.fake_auth_credential, data_source=self.fake_bucket_name, provider_type=Provider.PROVIDER_AZURE_LOCAL, provider_uuid=self.azure_provider_uuid, ) self.azure_local_report_downloader = AzureLocalReportDownloader( **{ "customer_name": self.customer_name, "credentials": self.fake_auth_credential, "data_source": self.fake_bucket_name, "bucket": self.fake_bucket_name, # TODO: bucket? "provider_uuid": self.azure_provider_uuid, })
def test_get_reports_error(self, fake_downloader): """Test get_reports function with error.""" downloader = ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=AMAZON_WEB_SERVICES, provider_id=1) with patch.object(AWSReportDownloader, 'get_report_context_for_date', side_effect=Exception('some error')): with self.assertRaises(ReportDownloaderError): downloader.get_reports()
def _get_report_files(customer_name, authentication, billing_source, provider_type, report_name=None): """ Task to download a Report. Note that report_name will be not optional once Koku can specify what report we should download. Args: customer_name (String): Name of the customer owning the cost usage report. access_credential (String): Credential needed to access cost usage report in the backend provider. report_source (String): Location of the cost usage report in the backend provider. provider_type (String): Koku defined provider type string. Example: Amazon = 'AWS' report_name (String): Name of the cost usage report to download. Returns: files (List) List of filenames with full local path. Example: ['/var/tmp/masu/region/aws/catch-clearly.csv', '/var/tmp/masu/base/aws/professor-hour-industry-television.csv'] """ stmt = ('Downloading report for' ' credential: {},' ' source: {},' ' customer_name: {},' ' provider: {}') log_statement = stmt.format(authentication, billing_source, customer_name, provider_type) LOG.info(log_statement) try: disk = psutil.disk_usage(Config.TMP_DIR) disk_msg = 'Avaiable disk space: {} bytes ({}%)'.format( disk.free, 100 - disk.percent) except OSError: disk_msg = 'Unable to find avaiable disk space. {} does not exist'.format( Config.TMP_DIR) LOG.info(disk_msg) try: downloader = ReportDownloader(customer_name=customer_name, access_credential=authentication, report_source=billing_source, provider_type=provider_type, report_name=report_name) return downloader.get_current_report() except (MasuProcessingError, MasuProviderError, ReportDownloaderError) as err: LOG.error(str(err)) raise err
def test_get_current_report(self, fake_downloader): """Test get_current_report function.""" downloader = ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=AMAZON_WEB_SERVICES) with patch.object(AWSReportDownloader, 'download_current_report', return_value=self.file_list): files = downloader.get_current_report() self.assertEqual(len(files), len(self.file_list))
def setUp(self): """Set up each test.""" super().setUp() self.fake_customer_name = CUSTOMER_NAME self.fake_report_name = "ocp-report" self.cluster_id = "my-ocp-cluster-1" report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id, "20180901-20181001") os.makedirs(report_path, exist_ok=True) test_file_path = ( "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv") self.test_file_path = os.path.join(report_path, os.path.basename(test_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_file_path)) test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv" self.test_storage_file_path = os.path.join( report_path, os.path.basename(test_storage_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_storage_file_path)) test_manifest_path = "./koku/masu/test/data/ocp/manifest.json" self.test_manifest_path = os.path.join( report_path, os.path.basename(test_manifest_path)) shutil.copyfile(test_manifest_path, os.path.join(report_path, self.test_manifest_path)) self.mock_task = Mock( request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.cluster_id, report_source=None, provider_type=Provider.PROVIDER_OCP, provider_uuid=self.ocp_provider_uuid, cache_key=self.fake.word(), ) self.ocp_report_downloader = OCPReportDownloader( **{ "task": self.mock_task, "customer_name": self.fake_customer_name, "auth_credential": self.cluster_id, "bucket": None, "provider_uuid": self.ocp_provider_uuid, "cache_key": self.fake.word(), })
def setUp(self, fake_session): os.makedirs(DATA_DIR, exist_ok=True) self.report_downloader = ReportDownloader(self.fake_customer_name, self.auth_credential, self.fake_bucket_name, 'AWS', 1) self.aws_report_downloader = AWSReportDownloader( **{ 'customer_name': self.fake_customer_name, 'auth_credential': self.auth_credential, 'bucket': self.fake_bucket_name, 'report_name': self.fake_report_name, 'provider_id': 1 })
def setUp(self): """Set up each test.""" super().setUp() self.start_date = datetime(year=2020, month=11, day=8).date() self.etag = "30c31bca571d9b7f3b2c8459dd8bc34a" self.invoice = "202011" test_report = f"./koku/masu/test/data/gcp/{self.invoice}_{self.etag}_2020-11-08:2020-11-11.csv" self.local_storage = tempfile.mkdtemp() local_dir = f"{self.local_storage}/{self.etag}" os.makedirs(local_dir) self.csv_file_name = test_report.split("/")[-1] self.csv_file_path = f"{local_dir}/{self.csv_file_name}" shutil.copy2(test_report, self.csv_file_path) self.credentials = {"project_id": "test-project"} self.data_source = {"table_id": "test-id", "dataset": "test-database", "local_dir": local_dir} self.report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source=self.data_source, provider_type=Provider.PROVIDER_GCP_LOCAL, provider_uuid=self.gcp_provider_uuid, ) self.gcp_local_report_downloader = GCPLocalReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": self.data_source, "provider_uuid": self.gcp_provider_uuid, } )
def test_download_missing_month(self): """Test to verify that downloading a non-existant month throws proper exception.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz') mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=7, day=7) with patch.object(DateAccessor, 'today', return_value=test_report_date): report_downloader = ReportDownloader(self.fake_customer_name, self.fake_auth_credential, fake_bucket, 'AWS-local', 1) # Names from test report .gz file report_downloader.download_report(test_report_date) expected_path = '{}/{}/{}'.format(DATA_DIR, self.fake_customer_name, 'aws-local') self.assertFalse(os.path.isdir(expected_path))
def setUp(self): """Set up each test.""" super().setUp() self.fake_customer_name = CUSTOMER_NAME self.fake_report_name = "ocp-report" self.cluster_id = "my-ocp-cluster-1" self.credentials = {"cluster_id": self.cluster_id} report_path = "{}/{}/{}".format(REPORTS_DIR, self.cluster_id, "20180901-20181001") os.makedirs(report_path, exist_ok=True) test_file_path = ( "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv") self.test_file_path = os.path.join(report_path, os.path.basename(test_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_file_path)) test_storage_file_path = "./koku/masu/test/data/ocp/e6b3701e-1e91" "-433b-b238-a31e49937558_storage.csv" self.test_storage_file_path = os.path.join( report_path, os.path.basename(test_storage_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_storage_file_path)) test_manifest_path = "./koku/masu/test/data/ocp/manifest.json" self.test_manifest_path = os.path.join( report_path, os.path.basename(test_manifest_path)) shutil.copyfile(test_manifest_path, os.path.join(report_path, self.test_manifest_path)) self.report_downloader = ReportDownloader( customer_name=self.fake_customer_name, credentials=self.credentials, data_source={}, provider_type=Provider.PROVIDER_OCP, provider_uuid=self.ocp_provider_uuid, ) self.ocp_report_downloader = OCPReportDownloader( **{ "customer_name": self.fake_customer_name, "credentials": self.credentials, "data_source": {}, "provider_uuid": self.ocp_provider_uuid, })
def test_initializer_downloader_exception(self, fake_downloader): """Test to initializer where _set_downloader throws exception""" with self.assertRaises(ReportDownloaderError): ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=AMAZON_WEB_SERVICES)
def test_initializer(self, fake_downloader): """Test to initializer""" downloader = ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=AMAZON_WEB_SERVICES) self.assertIsNotNone(downloader._downloader)
def test_initializer_ocp(self, fake_downloader): """Test to initializer for OCP downloader""" downloader = ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=OCP_LOCAL_SERVICE_PROVIDER, provider_id=1) self.assertIsNotNone(downloader._downloader)
def test_download_bucket_with_prefix(self): """Test to verify that basic report downloading works.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open('./tests/data/test_local_bucket_prefix.tar.gz') mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=8, day=7) with patch.object(DateAccessor, 'today', return_value=test_report_date): report_downloader = ReportDownloader(self.fake_customer_name, self.fake_auth_credential, fake_bucket, 'AWS-local', 1) # Names from test report .gz file report_downloader.download_report(test_report_date) expected_path = '{}/{}/{}'.format(DATA_DIR, self.fake_customer_name, 'aws-local') self.assertTrue(os.path.isdir(expected_path)) shutil.rmtree(fake_bucket)
def setUp(self): """Set up each test.""" self.fake_bucket_name = tempfile.mkdtemp() mytar = TarFile.open('./tests/data/test_local_bucket.tar.gz') mytar.extractall(path=self.fake_bucket_name) os.makedirs(DATA_DIR, exist_ok=True) self.report_downloader = ReportDownloader(self.fake_customer_name, self.fake_auth_credential, self.fake_bucket_name, 'AWS-local', 1) self.aws_local_report_downloader = AWSLocalReportDownloader( **{ 'customer_name': self.fake_customer_name, 'auth_credential': self.fake_auth_credential, 'bucket': self.fake_bucket_name, 'provider_id': 1 })
def test_invalid_provider_type(self): """Test that error is thrown with invalid account source.""" with self.assertRaises(ReportDownloaderError): ReportDownloader(customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type='unknown')
def setUp(self): """Set up each test.""" super().setUp() self.fake_customer_name = CUSTOMER_NAME self.fake_report_name = 'ocp-report' self.cluster_id = 'my-ocp-cluster-1' report_path = '{}/{}/{}'.format(REPORTS_DIR, self.cluster_id, '20180901-20181001') os.makedirs(report_path, exist_ok=True) test_file_path = './koku/masu/test/data/ocp/e6b3701e-1e91' \ '-433b-b238-a31e49937558_February-2019-my-ocp-cluster-1.csv' self.test_file_path = os.path.join(report_path, os.path.basename(test_file_path)) shutil.copyfile(test_file_path, os.path.join(report_path, self.test_file_path)) test_manifest_path = './koku/masu/test/data/ocp/manifest.json' self.test_manifest_path = os.path.join( report_path, os.path.basename(test_manifest_path)) shutil.copyfile(test_manifest_path, os.path.join(report_path, self.test_manifest_path)) self.mock_task = Mock( request=Mock(id=str(self.fake.uuid4()), return_value={})) self.report_downloader = ReportDownloader( task=self.mock_task, customer_name=self.fake_customer_name, access_credential=self.cluster_id, report_source=None, provider_type='OCP', provider_uuid=self.ocp_provider_uuid, ) self.ocp_report_downloader = OCPReportDownloader( **{ 'task': self.mock_task, 'customer_name': self.fake_customer_name, 'auth_credential': self.cluster_id, 'bucket': None, 'provider_uuid': self.ocp_provider_uuid, })
def test_download_missing_month(self): """Test to verify that downloading a non-existant month throws proper exception.""" fake_bucket = tempfile.mkdtemp() mytar = TarFile.open("./koku/masu/test/data/test_local_bucket_prefix.tar.gz") mytar.extractall(fake_bucket) test_report_date = datetime(year=2018, month=7, day=7) with patch.object(DateAccessor, "today", return_value=test_report_date): report_downloader = ReportDownloader( self.mock_task, self.fake_customer_name, self.fake_auth_credential, fake_bucket, Provider.PROVIDER_AWS_LOCAL, 1, ) # Names from test report .gz file report_downloader.download_report(test_report_date) expected_path = "{}/{}/{}".format(DATA_DIR, self.fake_customer_name, "aws-local") self.assertFalse(os.path.isdir(expected_path))
def test_initializer_azure(self, fake_downloader): """Test to initializer for Azure downloader""" downloader = ReportDownloader( customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=AZURE, provider_id=self.azure_provider_id, ) self.assertIsNotNone(downloader._downloader)
def test_initializer_ocp(self, fake_downloader): """Test to initializer""" downloader = ReportDownloader( customer_name='customer name', access_credential=self.fake_creds, report_source='hereiam', report_name='bestreport', provider_type=OPENSHIFT_CONTAINER_PLATFORM, provider_id=self.ocp_provider_id, ) self.assertIsNotNone(downloader._downloader)