def csv_reader(self, csv_file, envs, dtime_from=None, dtime_to=None):
envs_ids = [int(env['id']) for env in envs]
aws_account_id = envs[0]['ec2.account_id']
dtime_to = dtime_to or datetime.datetime.utcnow()
chunk_size = 500
with open(csv_file, 'r') as f:
i = 0
rows = []
reader = csv.DictReader(f)
for row in reader:
try:
if not row.get('user:scalr-meta'):
continue
row['scalr_meta'] = helper.get_scalr_meta(row['user:scalr-meta'])
if envs_ids and row['scalr_meta'].get('env_id'):
if row['scalr_meta']['env_id'] not in envs_ids:
continue
if aws_account_id and row['LinkedAccountId'] != aws_account_id:
continue
start_dtime = datetime.datetime.strptime(row['UsageStartDate'],
self.usage_start_dtime_format)
if dtime_from and start_dtime < dtime_from:
continue
if start_dtime > dtime_to:
break
i += 1
rows.append(row)
if i >= chunk_size:
yield rows
i = 0
rows = []
except:
helper.handle_error(message='CSV reader error')
if rows:
yield rows
def get_record(self, row):
record = {}
properties = row['properties']
cost_distr_type = 4
usage_type_name = 'Other'
usage_item_name = ''
instance_data = json.loads(properties['instanceData'])
tags = instance_data['Microsoft.Resources'].get('tags')
if not tags or 'scalr-meta' not in tags:
return
record.update(helper.get_scalr_meta(tags['scalr-meta']))
# Compute
if properties['meterCategory'] == 'Virtual Machines' and \
properties['meterName'] == 'Compute Hours':
cost_distr_type = 1
usage_type_name = 'BoxUsage'
usage_item_name = instance_data['Microsoft.Resources']['additionalInfo']['ServiceType']
record['instance_type'] = usage_item_name
record['num'] = 1.0
record['meter_id'] = properties['meterId']
record['resource_id'] = instance_data['Microsoft.Resources']['resourceUri'].split('/')[-1]
record['instance_id'] = record['resource_id']
if not record['server_id']:
record['server_id'] = record['resource_id']
else:
# Unsupported type
return
record['meter_id'] = properties['meterId']
record['dtime'] = datetime.datetime.strptime(properties['usageStartTime'].split('+')[0],
'%Y-%m-%dT%H:%M:%S')
record['platform'] = 'azure'
record['cloud_location'] = instance_data['Microsoft.Resources']['location']
record['cloud_location'] = record['cloud_location'].lower().replace(' ', '')
record['url'] = ''
record['cost_distr_type'] = cost_distr_type
record['usage_type_name'] = usage_type_name
record['usage_item_name'] = usage_item_name
assert record['usage_item_name'] or (cost_distr_type in (1, 4))
record['quantity'] = float(properties['quantity'])
if 'num' not in record:
record['num'] = record['quantity']
record['record_date'] = row['reported_date']
record['record_id'] = self.get_record_id(record)
return record