def __init__(self): self.export_name = "costreport" self.container = "test_container" self.directory = "cost" self.test_date = datetime(2019, 8, 15) self.month_range = utils.month_date_range(self.test_date) self.report_path = '{}/{}/{}'.format(self.directory, self.export_name, self.month_range) self.export_uuid = '9c308505-61d3-487c-a1bb-017956c9170a' self.export_file = '{}_{}.csv'.format(self.export_name, self.export_uuid) self.export_etag = 'absdfwef' self.export_key = '{}/{}'.format(self.report_path, self.export_file) self.bad_test_date = datetime(2019, 7, 15) self.bad_month_range = utils.month_date_range(self.bad_test_date) self.bad_report_path = '{}/{}/{}'.format(self.directory, self.export_name, self.bad_month_range)
def __init__(self): """Initialize a mocked azure service.""" self.export_name = "costreport" self.container = "test_container" self.directory = "cost" self.test_date = datetime(2019, 8, 15) self.month_range = utils.month_date_range(self.test_date) self.report_path = f"{self.directory}/{self.export_name}/{self.month_range}" self.export_uuid = "9c308505-61d3-487c-a1bb-017956c9170a" self.export_file = f"{self.export_name}_{self.export_uuid}.csv" self.export_etag = "absdfwef" self.export_key = f"{self.report_path}/{self.export_file}" self.bad_test_date = datetime(2019, 7, 15) self.bad_month_range = utils.month_date_range(self.bad_test_date) self.bad_report_path = f"{self.directory}/{self.export_name}/{self.bad_month_range}"
def _get_or_create_cost_entry_bill(self, row, report_db_accessor): """Get or Create a GCP cost entry bill object. Args: row (OrderedDict): A dictionary representation of a CSV file row. Returns: (string) An id of a GCP Bill. """ table_name = GCPCostEntryBill start_time = row["Start Time"] report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split("-") start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) data = { "billing_period_start": datetime.strftime(start_date_utc, "%Y-%m-%d %H:%M%z"), "billing_period_end": datetime.strftime(end_date_utc, "%Y-%m-%d %H:%M%z"), "provider_id": self._provider_uuid, } key = (start_date_utc, self._provider_uuid) if key in self.processed_report.bills: return self.processed_report.bills[key] bill_id = report_db_accessor.insert_on_conflict_do_nothing( table_name, data, conflict_columns=["billing_period_start", "provider_id"] ) self.processed_report.bills[key] = bill_id return bill_id
def create_bill(self, bill_date): """Create bill postgres entry.""" if isinstance(bill_date, str): bill_date = ciso8601.parse_datetime(bill_date) report_date_range = month_date_range(bill_date) start_date, end_date = report_date_range.split("-") report_period_start = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) report_period_end = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) # Make end date first of next month report_period_end = report_period_end + datetime.timedelta(days=1) provider = self._get_provider() cluster_id = utils.get_cluster_id_from_provider(provider.uuid) cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id) with schema_context(self._schema_name): OCPUsageReportPeriod.objects.get_or_create( cluster_id=cluster_id, cluster_alias=cluster_alias, report_period_start=report_period_start, report_period_end=report_period_end, provider=provider, )
def create_bill(self, bill_date): """Create bill postgres entry.""" if isinstance(bill_date, str): bill_date = ciso8601.parse_datetime(bill_date) report_date_range = utils.month_date_range(bill_date) start_date, end_date = report_date_range.split("-") start_date_utc = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) sql = f""" SELECT DISTINCT bill_payeraccountid FROM {self._table_name} WHERE source = '{self._provider_uuid}' AND year = '{bill_date.strftime("%Y")}' AND month = '{bill_date.strftime("%m")}' """ rows = self._execute_sql(sql, self._schema_name) payer_account_id = None if rows: payer_account_id = rows[0][0] provider = self._get_provider() with schema_context(self._schema_name): AWSCostEntryBill.objects.get_or_create( billing_period_start=start_date_utc, billing_period_end=end_date_utc, payer_account_id=payer_account_id, provider=provider, )
def test_get_gcp_cost_entry_bill(self): """Test calling _get_or_create_cost_entry_bill on an entry bill that exists fetches its id.""" start_time = '2019-09-17T00:00:00-07:00' report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split('-') start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) with schema_context(self.schema): entry_bill = GCPCostEntryBill.objects.create( provider=self.gcp_provider, billing_period_start=start_date_utc, billing_period_end=end_date_utc, ) entry_bill_id = self.processor._get_or_create_cost_entry_bill( { 'Start Time': datetime.strftime(start_date_utc, '%Y-%m-%d %H:%M%z') }, self.accessor) self.assertEquals(entry_bill.id, entry_bill_id)
def _get_or_create_cost_entry_bill(self, row, report_db_accessor): """Get or Create a GCP cost entry bill object. Args: row (OrderedDict): A dictionary representation of a CSV file row. Returns: (string) An id of a GCP Bill. """ table_name = GCPCostEntryBill dh = DateHelper() invoice_month = row["invoice.month"] start_time = dh.gcp_invoice_month_start(invoice_month) report_date_range = utils.month_date_range(start_time) start_date, end_date = report_date_range.split("-") start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) billing_period_start = datetime.strftime(start_date_utc, "%Y-%m-%d %H:%M%z") key = (billing_period_start, self._provider_uuid) if key in self.existing_bill_map: return self.existing_bill_map[key] data = { "billing_period_start": billing_period_start, "billing_period_end": datetime.strftime(end_date_utc, "%Y-%m-%d %H:%M%z"), "provider_id": self._provider_uuid, } key = (start_date_utc, self._provider_uuid) if key in self.processed_report.bills: return self.processed_report.bills[key] if key in self.existing_bill_map: return self.existing_bill_map[key] with transaction.atomic(): bill_id = report_db_accessor.insert_on_conflict_do_nothing( table_name, data, conflict_columns=["billing_period_start", "provider_id"]) self.processed_report.bills[key] = bill_id return bill_id
def setUp(self): """Set up GCP tests.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f"{self.temp_dir}/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv" shutil.copy2(self.test_report_path, self.test_report) gcp_auth = ProviderAuthentication.objects.create( credentials={"project-id": fake.word()}) gcp_billing_source = ProviderBillingSource.objects.create( data_source={"bucket": fake.word()}) with patch("masu.celery.tasks.check_report_updates"): self.gcp_provider = Provider.objects.create( uuid=uuid.uuid4(), name="Test Provider", type=Provider.PROVIDER_GCP, authentication=gcp_auth, billing_source=gcp_billing_source, customer=self.customer, setup_complete=True, ) start_time = "2020-11-08 23:00:00+00:00" report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split("-") self.start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.assembly_id = "1234" self.manifest_dict = { "assembly_id": self.assembly_id, "billing_period_start_datetime": self.start_date_utc, "num_total_files": 1, "provider_uuid": self.gcp_provider.uuid, } manifest_accessor = ReportManifestDBAccessor() self.manifest = manifest_accessor.add(**self.manifest_dict) self.processor = GCPReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.gcp_provider.uuid, manifest_id=self.manifest.id, ) self.accessor = GCPReportDBAccessor(self.schema)
def setUp(self): """Set up GCP tests.""" super().setUp() self.temp_dir = tempfile.mkdtemp() self.test_report = f'{self.temp_dir}/evidence-2019-06-03.csv' shutil.copy2(self.test_report_path, self.test_report) gcp_auth = ProviderAuthentication.objects.create( credentials={'project-id': fake.word()}) gcp_billing_source = ProviderBillingSource.objects.create( data_source={'bucket': fake.word()}) self.gcp_provider = Provider.objects.create( uuid=uuid.uuid4(), name='Test Provider', type=Provider.PROVIDER_GCP, authentication=gcp_auth, billing_source=gcp_billing_source, customer=self.customer, setup_complete=True, ) start_time = '2019-09-17T00:00:00-07:00' report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split('-') self.start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) self.assembly_id = '1234' self.manifest_dict = { 'assembly_id': self.assembly_id, 'billing_period_start_datetime': self.start_date_utc, 'num_total_files': 1, 'provider_uuid': self.gcp_provider.uuid, } manifest_accessor = ReportManifestDBAccessor() self.manifest = manifest_accessor.add(**self.manifest_dict) self.processor = GCPReportProcessor( schema_name=self.schema, report_path=self.test_report, compression=UNCOMPRESSED, provider_uuid=self.gcp_provider.uuid, manifest_id=self.manifest.id, ) self.accessor = GCPReportDBAccessor(self.schema, self.column_map)
def _get_report_path(self, date_time): """ Return path of report files. Args: date_time (DateTime): The starting datetime object Returns: (String): "/blob_dir/export_name/YYYYMMDD-YYYYMMDD", example: "/cost/costreport/20190801-20190831" """ report_date_range = month_date_range(date_time) return f"{self.directory}/{self.export_name}/{report_date_range}"
def test_get_gcp_cost_entry_bill(self): """Test calling _get_or_create_cost_entry_bill on an entry bill that exists fetches its id.""" start_time = "2020-11-01 00:00:00+00" report_date_range = utils.month_date_range(parser.parse(start_time)) start_date, end_date = report_date_range.split("-") start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) with schema_context(self.schema): entry_bill = GCPCostEntryBill.objects.create( provider=self.gcp_provider, billing_period_start=start_date_utc, billing_period_end=end_date_utc ) entry_bill_id = self.processor._get_or_create_cost_entry_bill({"invoice.month": "202011"}, self.accessor) self.assertEquals(entry_bill.id, entry_bill_id)
def _create_cost_entry_bill(self, row, report_db_accessor): """Create a cost entry bill object. Args: row (dict): A dictionary representation of a CSV file row Returns: (str): A cost entry bill object id """ table_name = AzureCostEntryBill row_date = row.get('UsageDateTime') report_date_range = utils.month_date_range(parser.parse(row_date)) start_date, end_date = report_date_range.split('-') start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC) key = (start_date_utc, self._provider_uuid) if key in self.processed_report.bills: return self.processed_report.bills[key] if key in self.existing_bill_map: return self.existing_bill_map[key] data = self._get_data_for_table(row, table_name._meta.db_table) data['provider_id'] = self._provider_uuid data['billing_period_start'] = datetime.strftime( start_date_utc, '%Y-%m-%d %H:%M%z') data['billing_period_end'] = datetime.strftime(end_date_utc, '%Y-%m-%d %H:%M%z') bill_id = report_db_accessor.insert_on_conflict_do_nothing( table_name, data, conflict_columns=['billing_period_start', 'provider_id']) self.processed_report.bills[key] = bill_id return bill_id
def create_azure_cost_entry_bill(self, provider_uuid, bill_date=None): """Create an Azure cost entry bill database object for test.""" table_name = AZURE_REPORT_TABLE_MAP['bill'] data = self.create_columns_for_table(table_name) data['provider_id'] = provider_uuid fake_bill_date = self.make_datetime_aware(self.fake.past_datetime()) data['billing_period_start'] = fake_bill_date data['billing_period_end'] = fake_bill_date if bill_date: report_date_range = azure_utils.month_date_range(bill_date) bill_start, bill_end = report_date_range.split('-') data['billing_period_start'] = self.make_datetime_aware(parser.parse(bill_start)) data['billing_period_end'] = self.make_datetime_aware(parser.parse(bill_end)) with AzureReportDBAccessor(self.schema, self.column_map) as accessor: return accessor.create_db_object(table_name, data)
def _create_cost_entry_bill(self, row, report_db_accessor): """Create a cost entry bill object. Args: row (dict): A dictionary representation of a CSV file row Returns: (str): A cost entry bill object id """ row_date = row.get("UsageDateTime") report_date_range = utils.month_date_range( ciso8601.parse_datetime(row_date)) start_date, end_date = report_date_range.split("-") start_date_utc = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) key = (start_date_utc, self._provider_uuid) if key in self.processed_report.bills: return self.processed_report.bills[key] if key in self.existing_bill_map: return self.existing_bill_map[key] data = self._get_data_for_table(row, AzureCostEntryBill._meta.db_table) data["provider_id"] = self._provider_uuid data["billing_period_start"] = datetime.strftime( start_date_utc, "%Y-%m-%d %H:%M%z") data["billing_period_end"] = datetime.strftime(end_date_utc, "%Y-%m-%d %H:%M%z") with transaction.atomic(): bill_id = report_db_accessor.insert_on_conflict_do_nothing( AzureCostEntryBill, data, conflict_columns=["billing_period_start", "provider_id"]) self.processed_report.bills[key] = bill_id return bill_id
def create_bill(self, bill_date): """Create bill postgres entry.""" if isinstance(bill_date, str): bill_date = ciso8601.parse_datetime(bill_date) report_date_range = utils.month_date_range(bill_date) start_date, end_date = report_date_range.split("-") start_date_utc = ciso8601.parse_datetime(start_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) end_date_utc = ciso8601.parse_datetime(end_date).replace( hour=0, minute=0, tzinfo=pytz.UTC) provider = self._get_provider() with schema_context(self._schema_name): AzureCostEntryBill.objects.get_or_create( billing_period_start=start_date_utc, billing_period_end=end_date_utc, provider=provider)