def calculate_delta(self, delta, query, query_sum, **kwargs):
"""Calculate cost deltas.
Args:
delta (String) 'day', 'month', or 'year'
query (Query) the original query being compared
Returns:
(dict) with keys "value" and "percent"
"""
delta_filter = copy.deepcopy(kwargs)
# get delta date range
_start = kwargs.get('usage_start__gte')
_end = kwargs.get('usage_end__lte')
if self.is_sum:
# Override as summary table is date based, not timestamp
_end = kwargs.get('usage_start__lte')
if delta == 'day':
previous = datetime.timedelta(days=1)
elif delta == 'month':
dh = DateHelper()
last_month = dh.days_in_month(_start.replace(day=1) - dh.one_day)
previous = datetime.timedelta(days=last_month)
elif delta == 'year':
previous = datetime.timedelta(days=365)
else:
# paranoia.
raise NotImplementedError(f'invalid parameter: {delta}')
delta_filter['usage_start__gte'] = _start - previous
if self.is_sum:
delta_filter['usage_start__lte'] = _end - previous
# construct new query
delta_query = AWSCostEntryLineItemDailySummary.objects.filter(
**delta_filter)
else:
delta_filter['usage_end__lte'] = _end - previous
delta_query = AWSCostEntryLineItem.objects.filter(**delta_filter)
# get aggregate sum from query
delta_sum = delta_query.aggregate(value=Sum(self.aggregate_key))
# calculate percentage difference: ((total - delta_total) / delta_total * 100)
q_sum = Decimal(query_sum.get('value') or 0)
d_sum = Decimal(delta_sum.get('value') or 0)
d_value = q_sum - d_sum
try:
d_percent = (q_sum - d_sum) / d_sum * Decimal(100)
except DivisionByZero:
d_percent = Decimal(0)
query_delta = {'value': d_value, 'percent': d_percent}
return query_delta
class AzureReportDataGenerator:
"""Populate the database with Azure fake report data."""
def __init__(self,
tenant,
provider,
current_month_only=False,
config=None):
"""Set up the class."""
self.tenant = tenant
self.config = config if config else FakeAzureConfig()
self.fake = Faker()
self.dh = DateHelper()
self.provider_uuid = provider.uuid
# generate a list of dicts with unique keys.
self.period_ranges, self.report_ranges = self.report_period_and_range(
current_month_only=current_month_only)
def _randomize_line_item(self, retained_fields=None):
"""Update our FakeAzureConfig to generate a new line item."""
DEFAULT_FIELDS = ['subscription_guid', 'resource_location', 'tags']
if not retained_fields:
retained_fields = DEFAULT_FIELDS
config_dict = {}
for field in retained_fields:
if field in self.config:
config_dict[field] = getattr(self.config, field)
self.config = FakeAzureConfig(**config_dict)
def report_period_and_range(self, current_month_only=False):
"""Return the report period and range."""
period = []
ranges = []
if current_month_only:
report_days = 10
diff_from_first = self.dh.today - self.dh.this_month_start
if diff_from_first.days < 10:
report_days = 1 + diff_from_first.days
period = [(self.dh.this_month_start, self.dh.this_month_end)]
ranges = [
list(self.dh.this_month_start + relativedelta(days=i)
for i in range(report_days))
]
else:
period = [(self.dh.this_month_start, self.dh.this_month_end)]
ranges = [
list(self.dh.today - relativedelta(days=i)
for i in range(report_days))
]
else:
period = [(self.dh.last_month_start, self.dh.last_month_end),
(self.dh.this_month_start, self.dh.this_month_end)]
one_month_ago = self.dh.today - relativedelta(months=1)
diff_from_first = self.dh.today - self.dh.this_month_start
if diff_from_first.days < 10:
report_days = 1 + diff_from_first.days
ranges = [
list(self.dh.last_month_start + relativedelta(days=i)
for i in range(report_days)),
list(self.dh.this_month_start + relativedelta(days=i)
for i in range(report_days)),
]
else:
ranges = [
list(one_month_ago - relativedelta(days=i)
for i in range(10)),
list(self.dh.today - relativedelta(days=i)
for i in range(10)),
]
return (period, ranges)
def add_data_to_tenant(self, fixed_fields=None):
"""Populate tenant with data.
Args:
provider_uuid (uuid) - provider uuid
fixed_fields (list) - a list of field names the data generator should NOT randomize.
"""
with tenant_context(self.tenant):
for i, period in enumerate(self.period_ranges):
manifest = self._manifest(period[0], self.provider_uuid)
self._report_status(manifest.billing_period_start_datetime,
manifest.id)
for report_date in self.report_ranges[i]:
self._randomize_line_item(retained_fields=fixed_fields)
self._cost_entry_line_item_daily_summary(report_date)
self._tag_summary()
def remove_data_from_tenant(self):
"""Remove the added data."""
with tenant_context(self.tenant):
for table in (AzureCostEntryLineItemDaily,
AzureCostEntryLineItemDailySummary,
AzureCostEntryBill, AzureCostEntryProductService,
AzureMeter, AzureTagsSummary):
table.objects.all().delete()
# pylint: disable=no-self-use
def remove_data_from_reporting_common(self):
"""Remove the public report statistics."""
for table in (CostUsageReportManifest, CostUsageReportStatus):
table.objects.all().delete()
# pylint: disable=no-self-use
def _manifest(self, start, provider_uuid):
"""Populate a report manifest entry."""
manifest_creation_datetime = start + relativedelta(
days=random.randint(1, 27))
manifest_updated_datetime = manifest_creation_datetime \
+ relativedelta(days=random.randint(1, 2))
data = {
'assembly_id': uuid4(),
'manifest_creation_datetime': manifest_creation_datetime,
'manifest_updated_datetime': manifest_updated_datetime,
'billing_period_start_datetime': start,
'num_processed_files': 1,
'num_total_files': 1,
'provider_id': provider_uuid
}
manifest_entry = CostUsageReportManifest(**data)
manifest_entry.save()
return manifest_entry
# pylint: disable=no-self-use
def _report_status(self, billing_period_start, manifest_id):
"""Populate a report status entry."""
etag_hash = hashlib.new('ripemd160')
etag_hash.update(bytes(str(billing_period_start), 'utf-8'))
last_started_datetime = billing_period_start + relativedelta(
days=random.randint(1, 3))
last_completed_datetime = last_started_datetime + relativedelta(days=1)
data = {
'report_name': uuid4(),
'last_completed_datetime': last_completed_datetime,
'last_started_datetime': last_started_datetime,
'etag': etag_hash.hexdigest(),
'manifest_id': manifest_id,
}
status_entry = CostUsageReportStatus(**data)
status_entry.save()
return status_entry
def _cost_entry_bill(self, report_date=None):
"""Populate AzureCostEntryBill."""
billing_period_start = self.config.billing_period_start
billing_period_end = self.config.billing_period_end
if report_date:
billing_period_start = report_date.replace(day=1)
last_day = self.dh.days_in_month(report_date)
billing_period_end = report_date.replace(day=last_day)
obj, _ = AzureCostEntryBill.objects.get_or_create(
billing_period_start=billing_period_start,
billing_period_end=billing_period_end,
summary_data_creation_datetime=self.dh.today,
summary_data_updated_datetime=self.dh.today,
finalized_datetime=None,
derived_cost_datetime=self.dh.today,
provider_id=self.provider_uuid)
# obj.save()
return obj
def _cost_entry_product(self):
"""Populate AzureCostEntryProduct."""
fake_dict = {self.fake.word(): self.fake.word()}
obj = AzureCostEntryProductService(
instance_id=self.config.instance_id,
resource_location=self.config.resource_location,
consumed_service=self.config.resource_service,
resource_type=self.config.resource_type,
resource_group=self.fake.word(),
additional_info=fake_dict,
service_name=self.config.service_name,
service_tier=self.config.service_tier,
service_info1=random.choice([None, self.fake.word()]),
service_info2=random.choice([None, self.fake.word()]))
obj.save()
return obj
def _meter(self):
"""Populate AzureMeter."""
obj = AzureMeter(
meter_id=uuid4(),
meter_name=self.config.meter_name,
meter_category=self.config.meter_category,
meter_subcategory=self.config.meter_subcategory,
meter_region=self.config.resource_location,
resource_rate=random.random(),
currency=random.choice(_currency_symbols()),
)
obj.save()
return obj
def _cost_entry_line_item_daily(self, report_date=None):
"""Populate AzureCostEntryLineItemDaily."""
if report_date:
usage_dt = report_date
else:
usage_dt = self.fake.date_time_between_dates(
self.dh.this_month_start, self.dh.today)
usage_qty = random.random() * random.randrange(0, 100)
obj = AzureCostEntryLineItemDaily(
cost_entry_bill=self._cost_entry_bill(report_date),
cost_entry_product=self._cost_entry_product(),
meter=self._meter(),
subscription_guid=self.config.subscription_guid,
tags=self.select_tags(),
usage_date_time=usage_dt,
usage_quantity=usage_qty,
pretax_cost=usage_qty * self.config.meter_rate,
offer_id=random.choice([None, self.fake.pyint()]))
obj.save()
return obj
def _cost_entry_line_item_daily_summary(self, report_date=None):
"""Populate AzureCostEntryLineItemDailySummary."""
line_item = self._cost_entry_line_item_daily(report_date)
obj = AzureCostEntryLineItemDailySummary(
cost_entry_bill=line_item.cost_entry_bill,
meter=line_item.meter,
subscription_guid=line_item.subscription_guid,
instance_type=self.config.instance_type,
service_name=line_item.cost_entry_product.service_name,
resource_location=line_item.cost_entry_product.resource_location,
tags=line_item.tags,
usage_start=line_item.usage_date_time,
usage_end=line_item.usage_date_time,
usage_quantity=line_item.usage_quantity,
pretax_cost=line_item.pretax_cost,
markup_cost=line_item.pretax_cost * 0.1,
offer_id=line_item.offer_id)
obj.save()
return obj
def _tag_summary(self):
"""Populate AzureTagsSummary."""
for key, val in self.config.tags.items():
try:
AzureTagsSummary.objects.get_or_create(key=key, values=[val])
except IntegrityError:
tags = AzureTagsSummary.objects.filter(key=key).first()
tags.values.append(val)
tags.save()
def select_tags(self):
"""Return a random selection of the defined tags."""
return {
key: self.config.tags[key]
for key in random.choices(list(self.config.tags.keys()),
k=random.randrange(
2, len(self.config.tags.keys())))
}