def calculate_delta(self, delta, query, query_sum, **kwargs): """Calculate cost deltas. Args: delta (String) 'day', 'month', or 'year' query (Query) the original query being compared Returns: (dict) with keys "value" and "percent" """ delta_filter = copy.deepcopy(kwargs) # get delta date range _start = kwargs.get('usage_start__gte') _end = kwargs.get('usage_end__lte') if self.is_sum: # Override as summary table is date based, not timestamp _end = kwargs.get('usage_start__lte') if delta == 'day': previous = datetime.timedelta(days=1) elif delta == 'month': dh = DateHelper() last_month = dh.days_in_month(_start.replace(day=1) - dh.one_day) previous = datetime.timedelta(days=last_month) elif delta == 'year': previous = datetime.timedelta(days=365) else: # paranoia. raise NotImplementedError(f'invalid parameter: {delta}') delta_filter['usage_start__gte'] = _start - previous if self.is_sum: delta_filter['usage_start__lte'] = _end - previous # construct new query delta_query = AWSCostEntryLineItemDailySummary.objects.filter( **delta_filter) else: delta_filter['usage_end__lte'] = _end - previous delta_query = AWSCostEntryLineItem.objects.filter(**delta_filter) # get aggregate sum from query delta_sum = delta_query.aggregate(value=Sum(self.aggregate_key)) # calculate percentage difference: ((total - delta_total) / delta_total * 100) q_sum = Decimal(query_sum.get('value') or 0) d_sum = Decimal(delta_sum.get('value') or 0) d_value = q_sum - d_sum try: d_percent = (q_sum - d_sum) / d_sum * Decimal(100) except DivisionByZero: d_percent = Decimal(0) query_delta = {'value': d_value, 'percent': d_percent} return query_delta
class AzureReportDataGenerator: """Populate the database with Azure fake report data.""" def __init__(self, tenant, provider, current_month_only=False, config=None): """Set up the class.""" self.tenant = tenant self.config = config if config else FakeAzureConfig() self.fake = Faker() self.dh = DateHelper() self.provider_uuid = provider.uuid # generate a list of dicts with unique keys. self.period_ranges, self.report_ranges = self.report_period_and_range( current_month_only=current_month_only) def _randomize_line_item(self, retained_fields=None): """Update our FakeAzureConfig to generate a new line item.""" DEFAULT_FIELDS = ['subscription_guid', 'resource_location', 'tags'] if not retained_fields: retained_fields = DEFAULT_FIELDS config_dict = {} for field in retained_fields: if field in self.config: config_dict[field] = getattr(self.config, field) self.config = FakeAzureConfig(**config_dict) def report_period_and_range(self, current_month_only=False): """Return the report period and range.""" period = [] ranges = [] if current_month_only: report_days = 10 diff_from_first = self.dh.today - self.dh.this_month_start if diff_from_first.days < 10: report_days = 1 + diff_from_first.days period = [(self.dh.this_month_start, self.dh.this_month_end)] ranges = [ list(self.dh.this_month_start + relativedelta(days=i) for i in range(report_days)) ] else: period = [(self.dh.this_month_start, self.dh.this_month_end)] ranges = [ list(self.dh.today - relativedelta(days=i) for i in range(report_days)) ] else: period = [(self.dh.last_month_start, self.dh.last_month_end), (self.dh.this_month_start, self.dh.this_month_end)] one_month_ago = self.dh.today - relativedelta(months=1) diff_from_first = self.dh.today - self.dh.this_month_start if diff_from_first.days < 10: report_days = 1 + diff_from_first.days ranges = [ list(self.dh.last_month_start + relativedelta(days=i) for i in range(report_days)), list(self.dh.this_month_start + relativedelta(days=i) for i in range(report_days)), ] else: ranges = [ list(one_month_ago - relativedelta(days=i) for i in range(10)), list(self.dh.today - relativedelta(days=i) for i in range(10)), ] return (period, ranges) def add_data_to_tenant(self, fixed_fields=None): """Populate tenant with data. Args: provider_uuid (uuid) - provider uuid fixed_fields (list) - a list of field names the data generator should NOT randomize. """ with tenant_context(self.tenant): for i, period in enumerate(self.period_ranges): manifest = self._manifest(period[0], self.provider_uuid) self._report_status(manifest.billing_period_start_datetime, manifest.id) for report_date in self.report_ranges[i]: self._randomize_line_item(retained_fields=fixed_fields) self._cost_entry_line_item_daily_summary(report_date) self._tag_summary() def remove_data_from_tenant(self): """Remove the added data.""" with tenant_context(self.tenant): for table in (AzureCostEntryLineItemDaily, AzureCostEntryLineItemDailySummary, AzureCostEntryBill, AzureCostEntryProductService, AzureMeter, AzureTagsSummary): table.objects.all().delete() # pylint: disable=no-self-use def remove_data_from_reporting_common(self): """Remove the public report statistics.""" for table in (CostUsageReportManifest, CostUsageReportStatus): table.objects.all().delete() # pylint: disable=no-self-use def _manifest(self, start, provider_uuid): """Populate a report manifest entry.""" manifest_creation_datetime = start + relativedelta( days=random.randint(1, 27)) manifest_updated_datetime = manifest_creation_datetime \ + relativedelta(days=random.randint(1, 2)) data = { 'assembly_id': uuid4(), 'manifest_creation_datetime': manifest_creation_datetime, 'manifest_updated_datetime': manifest_updated_datetime, 'billing_period_start_datetime': start, 'num_processed_files': 1, 'num_total_files': 1, 'provider_id': provider_uuid } manifest_entry = CostUsageReportManifest(**data) manifest_entry.save() return manifest_entry # pylint: disable=no-self-use def _report_status(self, billing_period_start, manifest_id): """Populate a report status entry.""" etag_hash = hashlib.new('ripemd160') etag_hash.update(bytes(str(billing_period_start), 'utf-8')) last_started_datetime = billing_period_start + relativedelta( days=random.randint(1, 3)) last_completed_datetime = last_started_datetime + relativedelta(days=1) data = { 'report_name': uuid4(), 'last_completed_datetime': last_completed_datetime, 'last_started_datetime': last_started_datetime, 'etag': etag_hash.hexdigest(), 'manifest_id': manifest_id, } status_entry = CostUsageReportStatus(**data) status_entry.save() return status_entry def _cost_entry_bill(self, report_date=None): """Populate AzureCostEntryBill.""" billing_period_start = self.config.billing_period_start billing_period_end = self.config.billing_period_end if report_date: billing_period_start = report_date.replace(day=1) last_day = self.dh.days_in_month(report_date) billing_period_end = report_date.replace(day=last_day) obj, _ = AzureCostEntryBill.objects.get_or_create( billing_period_start=billing_period_start, billing_period_end=billing_period_end, summary_data_creation_datetime=self.dh.today, summary_data_updated_datetime=self.dh.today, finalized_datetime=None, derived_cost_datetime=self.dh.today, provider_id=self.provider_uuid) # obj.save() return obj def _cost_entry_product(self): """Populate AzureCostEntryProduct.""" fake_dict = {self.fake.word(): self.fake.word()} obj = AzureCostEntryProductService( instance_id=self.config.instance_id, resource_location=self.config.resource_location, consumed_service=self.config.resource_service, resource_type=self.config.resource_type, resource_group=self.fake.word(), additional_info=fake_dict, service_name=self.config.service_name, service_tier=self.config.service_tier, service_info1=random.choice([None, self.fake.word()]), service_info2=random.choice([None, self.fake.word()])) obj.save() return obj def _meter(self): """Populate AzureMeter.""" obj = AzureMeter( meter_id=uuid4(), meter_name=self.config.meter_name, meter_category=self.config.meter_category, meter_subcategory=self.config.meter_subcategory, meter_region=self.config.resource_location, resource_rate=random.random(), currency=random.choice(_currency_symbols()), ) obj.save() return obj def _cost_entry_line_item_daily(self, report_date=None): """Populate AzureCostEntryLineItemDaily.""" if report_date: usage_dt = report_date else: usage_dt = self.fake.date_time_between_dates( self.dh.this_month_start, self.dh.today) usage_qty = random.random() * random.randrange(0, 100) obj = AzureCostEntryLineItemDaily( cost_entry_bill=self._cost_entry_bill(report_date), cost_entry_product=self._cost_entry_product(), meter=self._meter(), subscription_guid=self.config.subscription_guid, tags=self.select_tags(), usage_date_time=usage_dt, usage_quantity=usage_qty, pretax_cost=usage_qty * self.config.meter_rate, offer_id=random.choice([None, self.fake.pyint()])) obj.save() return obj def _cost_entry_line_item_daily_summary(self, report_date=None): """Populate AzureCostEntryLineItemDailySummary.""" line_item = self._cost_entry_line_item_daily(report_date) obj = AzureCostEntryLineItemDailySummary( cost_entry_bill=line_item.cost_entry_bill, meter=line_item.meter, subscription_guid=line_item.subscription_guid, instance_type=self.config.instance_type, service_name=line_item.cost_entry_product.service_name, resource_location=line_item.cost_entry_product.resource_location, tags=line_item.tags, usage_start=line_item.usage_date_time, usage_end=line_item.usage_date_time, usage_quantity=line_item.usage_quantity, pretax_cost=line_item.pretax_cost, markup_cost=line_item.pretax_cost * 0.1, offer_id=line_item.offer_id) obj.save() return obj def _tag_summary(self): """Populate AzureTagsSummary.""" for key, val in self.config.tags.items(): try: AzureTagsSummary.objects.get_or_create(key=key, values=[val]) except IntegrityError: tags = AzureTagsSummary.objects.filter(key=key).first() tags.values.append(val) tags.save() def select_tags(self): """Return a random selection of the defined tags.""" return { key: self.config.tags[key] for key in random.choices(list(self.config.tags.keys()), k=random.randrange( 2, len(self.config.tags.keys()))) }