def __init__(self):
self.export_name = "costreport"
self.container = "test_container"
self.directory = "cost"
self.test_date = datetime(2019, 8, 15)
self.month_range = utils.month_date_range(self.test_date)
self.report_path = '{}/{}/{}'.format(self.directory, self.export_name, self.month_range)
self.export_uuid = '9c308505-61d3-487c-a1bb-017956c9170a'
self.export_file = '{}_{}.csv'.format(self.export_name, self.export_uuid)
self.export_etag = 'absdfwef'
self.export_key = '{}/{}'.format(self.report_path, self.export_file)
self.bad_test_date = datetime(2019, 7, 15)
self.bad_month_range = utils.month_date_range(self.bad_test_date)
self.bad_report_path = '{}/{}/{}'.format(self.directory, self.export_name, self.bad_month_range)
def __init__(self):
"""Initialize a mocked azure service."""
self.export_name = "costreport"
self.container = "test_container"
self.directory = "cost"
self.test_date = datetime(2019, 8, 15)
self.month_range = utils.month_date_range(self.test_date)
self.report_path = f"{self.directory}/{self.export_name}/{self.month_range}"
self.export_uuid = "9c308505-61d3-487c-a1bb-017956c9170a"
self.export_file = f"{self.export_name}_{self.export_uuid}.csv"
self.export_etag = "absdfwef"
self.export_key = f"{self.report_path}/{self.export_file}"
self.bad_test_date = datetime(2019, 7, 15)
self.bad_month_range = utils.month_date_range(self.bad_test_date)
self.bad_report_path = f"{self.directory}/{self.export_name}/{self.bad_month_range}"
def _get_or_create_cost_entry_bill(self, row, report_db_accessor):
"""Get or Create a GCP cost entry bill object.
Args:
row (OrderedDict): A dictionary representation of a CSV file row.
Returns:
(string) An id of a GCP Bill.
"""
table_name = GCPCostEntryBill
start_time = row["Start Time"]
report_date_range = utils.month_date_range(parser.parse(start_time))
start_date, end_date = report_date_range.split("-")
start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC)
end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC)
data = {
"billing_period_start": datetime.strftime(start_date_utc, "%Y-%m-%d %H:%M%z"),
"billing_period_end": datetime.strftime(end_date_utc, "%Y-%m-%d %H:%M%z"),
"provider_id": self._provider_uuid,
}
key = (start_date_utc, self._provider_uuid)
if key in self.processed_report.bills:
return self.processed_report.bills[key]
bill_id = report_db_accessor.insert_on_conflict_do_nothing(
table_name, data, conflict_columns=["billing_period_start", "provider_id"]
)
self.processed_report.bills[key] = bill_id
return bill_id
def create_bill(self, bill_date):
"""Create bill postgres entry."""
if isinstance(bill_date, str):
bill_date = ciso8601.parse_datetime(bill_date)
report_date_range = month_date_range(bill_date)
start_date, end_date = report_date_range.split("-")
report_period_start = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
report_period_end = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
# Make end date first of next month
report_period_end = report_period_end + datetime.timedelta(days=1)
provider = self._get_provider()
cluster_id = utils.get_cluster_id_from_provider(provider.uuid)
cluster_alias = utils.get_cluster_alias_from_cluster_id(cluster_id)
with schema_context(self._schema_name):
OCPUsageReportPeriod.objects.get_or_create(
cluster_id=cluster_id,
cluster_alias=cluster_alias,
report_period_start=report_period_start,
report_period_end=report_period_end,
provider=provider,
)
def create_bill(self, bill_date):
"""Create bill postgres entry."""
if isinstance(bill_date, str):
bill_date = ciso8601.parse_datetime(bill_date)
report_date_range = utils.month_date_range(bill_date)
start_date, end_date = report_date_range.split("-")
start_date_utc = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
end_date_utc = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
sql = f"""
SELECT DISTINCT bill_payeraccountid
FROM {self._table_name}
WHERE source = '{self._provider_uuid}'
AND year = '{bill_date.strftime("%Y")}'
AND month = '{bill_date.strftime("%m")}'
"""
rows = self._execute_sql(sql, self._schema_name)
payer_account_id = None
if rows:
payer_account_id = rows[0][0]
provider = self._get_provider()
with schema_context(self._schema_name):
AWSCostEntryBill.objects.get_or_create(
billing_period_start=start_date_utc,
billing_period_end=end_date_utc,
payer_account_id=payer_account_id,
provider=provider,
)
def test_get_gcp_cost_entry_bill(self):
"""Test calling _get_or_create_cost_entry_bill on an entry bill that exists fetches its id."""
start_time = '2019-09-17T00:00:00-07:00'
report_date_range = utils.month_date_range(parser.parse(start_time))
start_date, end_date = report_date_range.split('-')
start_date_utc = parser.parse(start_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
end_date_utc = parser.parse(end_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
with schema_context(self.schema):
entry_bill = GCPCostEntryBill.objects.create(
provider=self.gcp_provider,
billing_period_start=start_date_utc,
billing_period_end=end_date_utc,
)
entry_bill_id = self.processor._get_or_create_cost_entry_bill(
{
'Start Time': datetime.strftime(start_date_utc,
'%Y-%m-%d %H:%M%z')
}, self.accessor)
self.assertEquals(entry_bill.id, entry_bill_id)
def _get_or_create_cost_entry_bill(self, row, report_db_accessor):
"""Get or Create a GCP cost entry bill object.
Args:
row (OrderedDict): A dictionary representation of a CSV file row.
Returns:
(string) An id of a GCP Bill.
"""
table_name = GCPCostEntryBill
dh = DateHelper()
invoice_month = row["invoice.month"]
start_time = dh.gcp_invoice_month_start(invoice_month)
report_date_range = utils.month_date_range(start_time)
start_date, end_date = report_date_range.split("-")
start_date_utc = parser.parse(start_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
end_date_utc = parser.parse(end_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
billing_period_start = datetime.strftime(start_date_utc,
"%Y-%m-%d %H:%M%z")
key = (billing_period_start, self._provider_uuid)
if key in self.existing_bill_map:
return self.existing_bill_map[key]
data = {
"billing_period_start":
billing_period_start,
"billing_period_end":
datetime.strftime(end_date_utc, "%Y-%m-%d %H:%M%z"),
"provider_id":
self._provider_uuid,
}
key = (start_date_utc, self._provider_uuid)
if key in self.processed_report.bills:
return self.processed_report.bills[key]
if key in self.existing_bill_map:
return self.existing_bill_map[key]
with transaction.atomic():
bill_id = report_db_accessor.insert_on_conflict_do_nothing(
table_name,
data,
conflict_columns=["billing_period_start", "provider_id"])
self.processed_report.bills[key] = bill_id
return bill_id
def setUp(self):
"""Set up GCP tests."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f"{self.temp_dir}/202011_30c31bca571d9b7f3b2c8459dd8bc34a_2020-11-08:2020-11-11.csv"
shutil.copy2(self.test_report_path, self.test_report)
gcp_auth = ProviderAuthentication.objects.create(
credentials={"project-id": fake.word()})
gcp_billing_source = ProviderBillingSource.objects.create(
data_source={"bucket": fake.word()})
with patch("masu.celery.tasks.check_report_updates"):
self.gcp_provider = Provider.objects.create(
uuid=uuid.uuid4(),
name="Test Provider",
type=Provider.PROVIDER_GCP,
authentication=gcp_auth,
billing_source=gcp_billing_source,
customer=self.customer,
setup_complete=True,
)
start_time = "2020-11-08 23:00:00+00:00"
report_date_range = utils.month_date_range(parser.parse(start_time))
start_date, end_date = report_date_range.split("-")
self.start_date_utc = parser.parse(start_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
self.end_date_utc = parser.parse(end_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
self.assembly_id = "1234"
self.manifest_dict = {
"assembly_id": self.assembly_id,
"billing_period_start_datetime": self.start_date_utc,
"num_total_files": 1,
"provider_uuid": self.gcp_provider.uuid,
}
manifest_accessor = ReportManifestDBAccessor()
self.manifest = manifest_accessor.add(**self.manifest_dict)
self.processor = GCPReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.gcp_provider.uuid,
manifest_id=self.manifest.id,
)
self.accessor = GCPReportDBAccessor(self.schema)
def setUp(self):
"""Set up GCP tests."""
super().setUp()
self.temp_dir = tempfile.mkdtemp()
self.test_report = f'{self.temp_dir}/evidence-2019-06-03.csv'
shutil.copy2(self.test_report_path, self.test_report)
gcp_auth = ProviderAuthentication.objects.create(
credentials={'project-id': fake.word()})
gcp_billing_source = ProviderBillingSource.objects.create(
data_source={'bucket': fake.word()})
self.gcp_provider = Provider.objects.create(
uuid=uuid.uuid4(),
name='Test Provider',
type=Provider.PROVIDER_GCP,
authentication=gcp_auth,
billing_source=gcp_billing_source,
customer=self.customer,
setup_complete=True,
)
start_time = '2019-09-17T00:00:00-07:00'
report_date_range = utils.month_date_range(parser.parse(start_time))
start_date, end_date = report_date_range.split('-')
self.start_date_utc = parser.parse(start_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
self.end_date_utc = parser.parse(end_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
self.assembly_id = '1234'
self.manifest_dict = {
'assembly_id': self.assembly_id,
'billing_period_start_datetime': self.start_date_utc,
'num_total_files': 1,
'provider_uuid': self.gcp_provider.uuid,
}
manifest_accessor = ReportManifestDBAccessor()
self.manifest = manifest_accessor.add(**self.manifest_dict)
self.processor = GCPReportProcessor(
schema_name=self.schema,
report_path=self.test_report,
compression=UNCOMPRESSED,
provider_uuid=self.gcp_provider.uuid,
manifest_id=self.manifest.id,
)
self.accessor = GCPReportDBAccessor(self.schema, self.column_map)
def _get_report_path(self, date_time):
"""
Return path of report files.
Args:
date_time (DateTime): The starting datetime object
Returns:
(String): "/blob_dir/export_name/YYYYMMDD-YYYYMMDD",
example: "/cost/costreport/20190801-20190831"
"""
report_date_range = month_date_range(date_time)
return f"{self.directory}/{self.export_name}/{report_date_range}"
def test_get_gcp_cost_entry_bill(self):
"""Test calling _get_or_create_cost_entry_bill on an entry bill that exists fetches its id."""
start_time = "2020-11-01 00:00:00+00"
report_date_range = utils.month_date_range(parser.parse(start_time))
start_date, end_date = report_date_range.split("-")
start_date_utc = parser.parse(start_date).replace(hour=0, minute=0, tzinfo=pytz.UTC)
end_date_utc = parser.parse(end_date).replace(hour=0, minute=0, tzinfo=pytz.UTC)
with schema_context(self.schema):
entry_bill = GCPCostEntryBill.objects.create(
provider=self.gcp_provider, billing_period_start=start_date_utc, billing_period_end=end_date_utc
)
entry_bill_id = self.processor._get_or_create_cost_entry_bill({"invoice.month": "202011"}, self.accessor)
self.assertEquals(entry_bill.id, entry_bill_id)
def _create_cost_entry_bill(self, row, report_db_accessor):
"""Create a cost entry bill object.
Args:
row (dict): A dictionary representation of a CSV file row
Returns:
(str): A cost entry bill object id
"""
table_name = AzureCostEntryBill
row_date = row.get('UsageDateTime')
report_date_range = utils.month_date_range(parser.parse(row_date))
start_date, end_date = report_date_range.split('-')
start_date_utc = parser.parse(start_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
end_date_utc = parser.parse(end_date).replace(hour=0,
minute=0,
tzinfo=pytz.UTC)
key = (start_date_utc, self._provider_uuid)
if key in self.processed_report.bills:
return self.processed_report.bills[key]
if key in self.existing_bill_map:
return self.existing_bill_map[key]
data = self._get_data_for_table(row, table_name._meta.db_table)
data['provider_id'] = self._provider_uuid
data['billing_period_start'] = datetime.strftime(
start_date_utc, '%Y-%m-%d %H:%M%z')
data['billing_period_end'] = datetime.strftime(end_date_utc,
'%Y-%m-%d %H:%M%z')
bill_id = report_db_accessor.insert_on_conflict_do_nothing(
table_name,
data,
conflict_columns=['billing_period_start', 'provider_id'])
self.processed_report.bills[key] = bill_id
return bill_id
def create_azure_cost_entry_bill(self, provider_uuid, bill_date=None):
"""Create an Azure cost entry bill database object for test."""
table_name = AZURE_REPORT_TABLE_MAP['bill']
data = self.create_columns_for_table(table_name)
data['provider_id'] = provider_uuid
fake_bill_date = self.make_datetime_aware(self.fake.past_datetime())
data['billing_period_start'] = fake_bill_date
data['billing_period_end'] = fake_bill_date
if bill_date:
report_date_range = azure_utils.month_date_range(bill_date)
bill_start, bill_end = report_date_range.split('-')
data['billing_period_start'] = self.make_datetime_aware(parser.parse(bill_start))
data['billing_period_end'] = self.make_datetime_aware(parser.parse(bill_end))
with AzureReportDBAccessor(self.schema, self.column_map) as accessor:
return accessor.create_db_object(table_name, data)
def _create_cost_entry_bill(self, row, report_db_accessor):
"""Create a cost entry bill object.
Args:
row (dict): A dictionary representation of a CSV file row
Returns:
(str): A cost entry bill object id
"""
row_date = row.get("UsageDateTime")
report_date_range = utils.month_date_range(
ciso8601.parse_datetime(row_date))
start_date, end_date = report_date_range.split("-")
start_date_utc = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
end_date_utc = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
key = (start_date_utc, self._provider_uuid)
if key in self.processed_report.bills:
return self.processed_report.bills[key]
if key in self.existing_bill_map:
return self.existing_bill_map[key]
data = self._get_data_for_table(row, AzureCostEntryBill._meta.db_table)
data["provider_id"] = self._provider_uuid
data["billing_period_start"] = datetime.strftime(
start_date_utc, "%Y-%m-%d %H:%M%z")
data["billing_period_end"] = datetime.strftime(end_date_utc,
"%Y-%m-%d %H:%M%z")
with transaction.atomic():
bill_id = report_db_accessor.insert_on_conflict_do_nothing(
AzureCostEntryBill,
data,
conflict_columns=["billing_period_start", "provider_id"])
self.processed_report.bills[key] = bill_id
return bill_id
def create_bill(self, bill_date):
"""Create bill postgres entry."""
if isinstance(bill_date, str):
bill_date = ciso8601.parse_datetime(bill_date)
report_date_range = utils.month_date_range(bill_date)
start_date, end_date = report_date_range.split("-")
start_date_utc = ciso8601.parse_datetime(start_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
end_date_utc = ciso8601.parse_datetime(end_date).replace(
hour=0, minute=0, tzinfo=pytz.UTC)
provider = self._get_provider()
with schema_context(self._schema_name):
AzureCostEntryBill.objects.get_or_create(
billing_period_start=start_date_utc,
billing_period_end=end_date_utc,
provider=provider)
