def document_field(field):
"""
The default ``field_factory`` method for converting Django field instances to ``elasticsearch_dsl.Field`` instances.
Auto-created fields (primary keys, for example) and one-to-many fields (reverse FK relationships) are skipped.
"""
if field.auto_created or field.one_to_many:
return None
if field.many_to_many:
return RawMultiString
defaults = {
models.DateField:
dsl.Date(),
models.DateTimeField:
dsl.Date(),
models.IntegerField:
dsl.Long(),
models.PositiveIntegerField:
dsl.Long(),
models.BooleanField:
dsl.Boolean(),
models.NullBooleanField:
dsl.Boolean(),
# models.SlugField: dsl.String(index='not_analyzed'),
models.SlugField:
dsl.Text(index='not_analyzed'),
models.DecimalField:
dsl.Double(),
models.FloatField:
dsl.Float(),
}
return defaults.get(field.__class__, RawString)
def doc_field(type):
defaults = {
'date': dsl.Date(),
'integer': dsl.Long(),
'boolean': dsl.Boolean(),
'double': dsl.Double(),
'float': dsl.Float(),
}
return defaults.get(type, RawString)
class AWSMetric(dsl.DocType):
class Meta:
index = 'awsmetric'
key = dsl.String(index='not_analyzed')
resource = dsl.String(index='not_analyzed')
metric = dsl.String(index='not_analyzed')
time = dsl.Date(format='date_optional_time||epoch_millis')
period = dsl.Integer()
value = dsl.Double()
@classmethod
def underutilized_resources(cls, keys, timespan=timedelta(days=30)):
keys = any_key_to_string_array(keys)
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:CPUUtilization:Maximum')
s = s.filter('terms', key=keys)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('percentiles',
'percentile_ranks',
field='value',
values=[20, 50])
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
resources = []
for resource in res['aggregations']['resources']['buckets']:
if resource['percentiles']['values']['20.0'] == 100:
res_region, res_id = resource['key'].split('/')
resources.append(
dict(type='EC2 Instance',
id=res_id,
region=res_region,
underutilized=['CPU usage under 20%']))
return dict(resources=resources)
@classmethod
def hourly_cpu_usage(cls, keys, resources=None):
s = cls.search()
if isinstance(keys, basestring):
keys = [keys]
elif not isinstance(keys, list):
keys = list(keys)
assert all(isinstance(key, basestring) for key in keys)
s = s.filter('terms', key=keys)
if resources:
s = s.filter('terms', resource=resources)
s = s.filter('term', metric='AWS/EC2:CPUUtilization:Maximum')
agg = s.aggs.bucket('intervals',
'date_histogram',
field='time',
interval='hour',
min_doc_count=1)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
tmp_hours = defaultdict(list)
for interval in res['aggregations']['intervals']['buckets']:
interval_hour = interval['key_as_string'].split('T')[1].split(
':')[0]
tmp_hours[interval_hour].append(interval['utilization']['value'])
hours = OrderedDict(
zip(["{:02d}".format(x) for x in range(0, 24)],
itertools.repeat(0)))
for hour, values in tmp_hours.iteritems():
hours[hour] = sum(values) / len(values)
if not tmp_hours:
return None
return [
dict(hour=hour, cpu=float(cpu)) for hour, cpu in hours.iteritems()
]
@classmethod
def days_of_the_week_cpu_usage(cls, keys, resources=None):
s = cls.search()
if isinstance(keys, basestring):
keys = [keys]
elif not isinstance(keys, list):
keys = list(keys)
assert all(isinstance(key, basestring) for key in keys)
s = s.filter('terms', key=keys)
if resources:
s = s.filter('terms', resource=resources)
s = s.filter('term', metric='AWS/EC2:CPUUtilization:Maximum')
agg = s.aggs.bucket('intervals',
'date_histogram',
field='time',
interval='day',
min_doc_count=1)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
tmp_days_of_the_week = defaultdict(list)
for interval in res['aggregations']['intervals']['buckets']:
weekday = datetime.strptime(
interval['key_as_string'].split('T')[0],
'%Y-%m-%d').date().weekday()
tmp_days_of_the_week[weekday].append(
interval['utilization']['value'])
days = OrderedDict(zip(range(0, 7), itertools.repeat(0)))
for weekday, values in tmp_days_of_the_week.iteritems():
days[weekday] = sum(values) / len(values)
if not tmp_days_of_the_week:
return None
return [
dict(day=calendar.day_name[weekday], cpu=float(cpu))
for weekday, cpu in days.iteritems()
]
@classmethod
def daily_cpu_utilization(cls, key):
s = cls.search()
s = s.filter('term', key=key)
s = s.filter('term', metric='AWS/EC2:CPUUtilization:Maximum')
agg = s.aggs.bucket('intervals',
'date_histogram',
field='time',
interval='day',
min_doc_count=1)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for interval in res['aggregations']['intervals']['buckets']:
yield interval['key_as_string'].split(
'T')[0], interval['utilization']['value']
@classmethod
def get_cpu_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:CPUUtilization:Maximum')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_instance_read_iops_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:DiskReadOps:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_instance_write_iops_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:DiskWriteOps:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_instance_read_bytes_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:DiskReadBytes:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_instance_write_bytes_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:DiskWriteBytes:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_volume_read_iops_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:VolumeReadOps:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_volume_write_iops_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:VolumeWriteOps:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_volume_read_bytes_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:VolumeReadBytes:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_volume_write_bytes_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EBS:VolumeWriteBytes:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_network_in_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:NetworkIn:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_network_out_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/EC2:NetworkOut:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_s3_space_usage(cls, key, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range',
time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='AWS/S3:BucketSizeBytes:Average')
s = s.filter('term', key=key)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='awsmetric',
body=s.to_dict(),
size=0,
request_timeout=60)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
class AWSDetailedLineitem(dsl.DocType):
class Meta:
index = 'awsdetailedlineitem'
availability_zone = dsl.String(index='not_analyzed')
cost = dsl.Double()
un_blended_cost = dsl.Double()
item_description = dsl.String(index='not_analyzed')
linked_account_id = dsl.String(index='not_analyzed')
operation = dsl.String()
payer_account_id = dsl.String(index='not_analyzed')
pricing_plan_id = dsl.Long()
product_name = dsl.String(index='not_analyzed')
rate = dsl.Double()
un_blended_rate = dsl.Double()
rate_id = dsl.Long()
record_id = dsl.String(index='not_analyzed')
reserved_instance = dsl.Boolean()
resource_id = dsl.String(index='not_analyzed')
subscription_id = dsl.Long()
tag = dsl.Object(
properties={
'key': dsl.String(index='not_analyzed'),
'value': dsl.String(index='not_analyzed')
})
usage_end_date = dsl.Date(format='strict_date_optional_time||epoch_millis')
usage_quantity = dsl.Double()
usage_start_date = dsl.Date(
format='strict_date_optional_time||epoch_millis')
usage_type = dsl.String(index='not_analyzed')
@classmethod
@with_cache(ttl=3600 * 3, worker_refresh=True)
def keys_has_data(cls, keys, date_from=None, date_to=None):
date_to = date_to or datetime.utcnow()
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
if date_from:
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
return res['hits']['total'] > 0
@classmethod
@with_cache(is_json=False, ret=lambda x: datetime.strptime(x, "%Y-%m-%d"))
def get_first_date(cls, keys):
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.sort('usage_start_date')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=1,
request_timeout=60)
if res['hits']['total'] == 0:
return
return res['hits']['hits'][0]['_source']['usage_start_date'].split(
'T')[0]
@classmethod
@with_cache(is_json=False, ret=lambda x: datetime.strptime(x, "%Y-%m-%d"))
def get_last_date(cls, keys, limit=None):
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
if limit:
s = s.filter('range', usage_start_date={'to': limit.isoformat()})
s = s.sort('-usage_start_date')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=1,
request_timeout=60)
if res['hits']['total'] == 0:
return
return res['hits']['hits'][0]['_source']['usage_start_date'].split(
'T')[0]
@classmethod
def get_first_to_now_date(cls, keys):
def from_date_to_today(d):
now = datetime.utcnow()
while d < now:
yield d
d += relativedelta(months=1)
return list(from_date_to_today(cls.get_first_date(keys)))
@classmethod
def get_first_to_last_date(cls, keys):
def from_date_to_last(d):
last = cls.get_last_date(keys)
while d < last:
yield d
d += relativedelta(months=1)
return list(from_date_to_last(cls.get_first_date(keys)))
@classmethod
@with_cache(6 * 3600)
def get_available_tags(cls, keys, only_with_data=None, product_name=None):
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
if product_name:
s = s.filter('term', product_name=product_name)
s.aggs.bucket('tag_key', 'terms', field='tag.key')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
tags = []
for tag in res['aggregations']['tag_key']['buckets']:
if tag['key'].startswith('user:'******'key'].split(':')[1]
if not only_with_data or name in AWSStat.latest_hourly_cpu_usage_by_tag(
only_with_data
)['tags'] or name in AWSStat.latest_daily_cpu_usage_by_tag(
only_with_data)['tags']:
tags.append(name)
tags.sort()
return dict(tags=tags)
@classmethod
@with_cache(ttl=6 * 3600)
def get_cost_by_tag(cls, keys, tag, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('term', **{'tag.key': 'user:{}'.format(tag)})
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s.aggs.bucket('total_cost', 'sum', field='cost')
agg = s.aggs.bucket('tag_value',
'terms',
field='tag.value',
size=0x7FFFFFFF)
agg.bucket('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
tags = [{
'tag_value': tag['key'],
'cost': tag['cost']['value'],
} for tag in res['aggregations']['tag_value']['buckets']]
return dict(tags=tags,
total_cost=res['aggregations']['total_cost']['value'])
@classmethod
@with_cache(ttl=6 * 3600)
def get_cost(cls, keys, date_from, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(
hour=23, minute=59, second=59, microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s.aggs.bucket('total_cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
return dict(total_cost=res['aggregations']['total_cost']['value'])
@classmethod
@with_cache()
def get_monthly_cost_by_tag(cls, keys, tag, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('term', **{'tag.key': 'user:{}'.format(tag)})
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='month',
min_doc_count=1)
agg.bucket('total_cost', 'sum', field='cost')
agg = agg.bucket('tag_value',
'terms',
field='tag.value',
size=0x7FFFFFFF)
agg.bucket('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
months = [{
'month':
interval['key_as_string'].split('T')[0][:-3],
'tags': [{
'tag_value': tag['key'],
'cost': tag['cost']['value'],
} for tag in interval['tag_value']['buckets']],
'total_cost':
interval['total_cost']['value'],
} for interval in res['aggregations']['intervals']['buckets']]
return dict(months=months)
@classmethod
@with_cache()
def get_cost_by_product(cls,
key,
date_from=None,
date_to=None,
without_discount=False,
only_discount=False,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter('term', linked_account_id=key)
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
if without_discount:
s = s.query(
'bool',
filter=[
~dsl.Q('term', item_description='PAR_APN_ProgramFee_2500')
])
if only_discount:
s = s.filter('term', item_description='PAR_APN_ProgramFee_2500')
agg = s.aggs.bucket('products',
'terms',
field='product_name',
order={'cost': 'desc'},
size=size)
agg.bucket('cost', 'sum', field='cost')
s = s.query('bool', filter=[~dsl.Q('term', cost=0)])
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
products = [{
'product': SHORT_NAMES.get(product['key'], product['key']),
'cost': product['cost']['value'],
} for product in res['aggregations']['products']['buckets']]
return dict(products=products)
@classmethod
@with_cache()
def get_cost_by_region(cls,
keys,
tagged=False,
byaccount=False,
date_from=None,
date_to=None,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs
if byaccount:
agg = agg.bucket('accounts', 'terms', field='linked_account_id')
agg = agg.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='month',
min_doc_count=1)
agg = agg.bucket('regions',
'terms',
field='availability_zone',
size=size)
agg.bucket('cost', 'sum', field='cost')
if tagged:
agg = agg.bucket('tags', 'terms', field='tag.value')
agg.bucket('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0)
return res['aggregations']
@classmethod
@with_cache()
def get_monthly_cost(cls,
keys,
date_from=None,
date_to=None,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='month',
min_doc_count=1)
agg.bucket('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
res = [{
'month': interval['key_as_string'].split('T')[0],
'total_cost': interval['cost']['value'],
} for interval in res['aggregations']['intervals']['buckets']]
return dict(months=res)
@classmethod
@with_cache()
def get_monthly_cost_by_product(cls,
keys,
tagged=False,
date_from=None,
date_to=None,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='month',
min_doc_count=1)
agg = agg.bucket('products', 'terms', field='product_name', size=size)
agg.bucket('cost', 'sum', field='cost')
if tagged:
agg = agg.bucket('tags', 'terms', field='tag.value')
agg.bucket('cost', 'sum', field='cost')
s = s.query('bool', filter=[~dsl.Q('term', cost=0)])
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
def tagged_cost(bucket, total):
total_tag = 0.0
for tag in bucket:
total_tag += tag['cost']['value']
yield (tag['key'], tag['cost']['value'])
if total != total_tag:
yield ('untagged', total - total_tag)
res = [{
'month':
interval['key_as_string'].split('T')[0],
'products': [{
'product':
SHORT_NAMES.get(product['key'], product['key']),
'cost':
product['cost']['value'],
'tags': [{
'name': tag[0],
'cost': tag[1],
} for tag in tagged_cost(product['tags']['buckets'],
product['cost']['value'])],
} for product in interval['products']['buckets']] if tagged else [{
'product':
SHORT_NAMES.get(product['key'], product['key']),
'cost':
product['cost']['value'],
} for product in interval['products']['buckets']]
} for interval in res['aggregations']['intervals']['buckets']]
return dict(months=res)
@classmethod
@with_cache(ttl=4 * 3600)
def get_daily_cost_by_product(cls,
keys,
date_from=None,
date_to=None,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(
hour=23, minute=59, second=59, microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='day',
min_doc_count=1)
agg = agg.bucket('products', 'terms', field='product_name', size=size)
agg.metric('cost', 'sum', field='cost')
s = s.query('bool', filter=[~dsl.Q('term', cost=0)])
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
res = [{
'day':
interval['key_as_string'].split('T')[0],
'products': [{
'product':
SHORT_NAMES.get(product['key'], product['key']),
'cost':
product['cost']['value'],
} for product in interval['products']['buckets']]
} for interval in res['aggregations']['intervals']['buckets']]
return dict(days=res)
@classmethod
@with_cache(ttl=24 * 3600)
def get_yearly_cost_by_product(cls,
keys,
date_from=None,
date_to=None,
size=0x7FFFFFFF):
date_from = date_from or datetime.utcnow().replace(
month=1, day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(month=12,
day=31,
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='year',
min_doc_count=1)
agg = agg.bucket('products', 'terms', field='product_name', size=size)
agg.metric('cost', 'sum', field='cost')
s = s.query('bool', filter=[~dsl.Q('term', cost=0)])
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
res = [{
'year':
interval['key_as_string'][:4],
'products': [{
'product':
SHORT_NAMES.get(product['key'], product['key']),
'cost':
product['cost']['value'],
} for product in interval['products']['buckets']]
} for interval in res['aggregations']['intervals']['buckets']]
return dict(years=res)
@classmethod
@with_cache()
def get_cost_by_resource(cls,
keys,
date_from=None,
date_to=None,
search=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
if search:
s = s.query('wildcard', resource_id='*{}*'.format(search))
agg = s.aggs.bucket('resources',
'terms',
field='resource_id',
order={'cost': 'desc'},
size=0x7FFFFFFF)
agg.bucket('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
resources = [{
'resource': resource['key'],
'cost': resource['cost']['value'],
} for resource in res['aggregations']['resources']['buckets']]
return resources
@classmethod
def get_monthly_cost_by_resource(cls,
resource_ids,
date_from=None,
date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
if resource_ids:
s = cls.search()
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s = s.filter('terms', resource_id=list(resource_ids))
agg = s.aggs.bucket('months',
'date_histogram',
field='usage_start_date',
interval='month',
min_doc_count=1)
agg.metric('cost', 'sum', field='cost')
r = client.search('awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
return {
e['key_as_string']: e['cost']['value']
for e in r['aggregations']['months']['buckets']
}
else:
return {}
@classmethod
@with_cache()
def get_lambda_usage(cls, keys, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('term', product_name='AWS Lambda')
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('resources',
'terms',
field='resource_id',
size=0x7FFFFFFF)
agg.metric('cost', 'avg', field='cost')
agg = agg.bucket('types', 'terms', field='usage_type', size=0x7FFFFFFF)
agg.metric('quantity', 'sum', field='usage_quantity')
agg = agg.bucket('descriptions',
'terms',
field='item_description',
size=0x7FFFFFFF)
agg.metric('quantity', 'sum', field='usage_quantity')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
#return res
def _lambda_usage_regb(buckets, endswith):
for b in buckets:
if b['key'].endswith(endswith):
return b['quantity']['value']
usages = [{
'rid':
usage['key'],
'name':
usage['key'].split(':')[-1],
'requests':
_lambda_usage_regb(usage['types']['buckets'], '-Request'),
'gb_seconds':
_lambda_usage_regb(usage['types']['buckets'], '-Lambda-GB-Second'),
'cost':
usage['cost']['value'],
'raw_cost':
lambdapricing.get_raw_cost([
x['descriptions']['buckets'] for x in usage['types']['buckets']
]),
} for usage in res['aggregations']['resources']['buckets']]
return usages
@classmethod
@with_cache()
def get_s3_bandwidth_costs(cls, key, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter('term', linked_account_id=key)
s = s.filter('term', product_name='Amazon Simple Storage Service')
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('types',
'terms',
field='usage_type',
size=0x7FFFFFFF)
agg.metric('cost', 'sum', field='cost')
agg.metric('gb', 'sum', field='usage_quantity')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
transfers = [{
'type': transfer['key'],
'quantity': transfer['gb']['value'],
'cost': transfer['cost']['value'],
} for transfer in res['aggregations']['types']['buckets']]
return transfers
@classmethod
@with_cache()
def get_ec2_bandwidth_costs(cls, key, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter('term', linked_account_id=key)
s = s.filter('term', product_name='Amazon Elastic Compute Cloud')
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
agg = s.aggs.bucket('types',
'terms',
field='usage_type',
size=0x7FFFFFFF)
agg.metric('cost', 'sum', field='cost')
agg.metric('gb', 'sum', field='usage_quantity')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
transfers = [{
'type': transfer['key'],
'quantity': transfer['gb']['value'],
'cost': transfer['cost']['value'],
} for transfer in res['aggregations']['types']['buckets']]
return transfers
@classmethod
def get_ec2_daily_cost(cls, key):
s = cls.search()
s = s.filter('term', linked_account_id=key)
s = s.filter('term', product_name='Amazon Elastic Compute Cloud')
agg = s.aggs.bucket('intervals',
'date_histogram',
field='usage_start_date',
interval='day',
min_doc_count=1)
agg.metric('cost', 'sum', field='cost')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
for interval in res['aggregations']['intervals']['buckets']:
yield interval['key_as_string'].split(
'T')[0], interval['cost']['value']
@classmethod
@with_cache()
def get_elb_usage_a_day(cls, keys, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
gib = Fraction(2**30)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s = s.filter("prefix", resource_id="arn:aws:elasticloadbalancing")
s = s.sort({"usage_start_date": {"order": "desc"}})
agg = s.aggs.bucket('rid',
'terms',
field='resource_id',
size=0x7FFFFFFF)
agg.metric('cost', 'sum', field='cost')
agg = agg.bucket('types', 'terms', field='usage_type', size=0x7FFFFFFF)
agg.metric('quantity', 'sum', field='usage_quantity')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
elbs = [{
'rid':
elb['key'],
'cost':
elb['cost']['value'] / (date_to - date_from).days,
'hours':
float(
sum([
x['quantity']['value'] for x in elb['types']['buckets']
if x['key'].endswith('LoadBalancerUsage')
]) / (date_to - date_from).days),
'bytes':
float((sum([
x['quantity']['value']
for x in elb['types']['buckets'] if x['key'].endswith('Bytes')
]) * gib) / (date_to - date_from).days),
} for elb in res['aggregations']['rid']['buckets']]
return elbs
@classmethod
@with_cache()
def get_instance_type(cls, keys, date_from=None, date_to=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.extra(_source=[
'usage_start_date', 'usage_type', 'availability_zone',
'resource_id'
])
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s = s.filter("term", product_name='Amazon Elastic Compute Cloud')
s = s.query('wildcard', usage_type='*BoxUsage:*')
s = s.filter('exists', field='resource_id')
s = s.sort({"usage_start_date": {"order": "desc"}})
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=10000,
request_timeout=60)
def cut_region_name(s):
return s[:-1] if s[-1].isalpha() else s
types = []
refs = {}
def add_in_types(type, rid):
ref_tuple = (type['hour'], type['instance'], type['region'])
if ref_tuple in refs:
refs[ref_tuple]['rids'].append(rid)
refs[ref_tuple]['ridCount'] += 1
return
type['rids'] = [rid]
types.append(type)
refs[ref_tuple] = types[-1]
for r in res['hits']['hits']:
elem = {
'hour':
r['_source']['usage_start_date'],
'instance':
r['_source']['usage_type'].split(':')[1],
'region':
cut_region_name(r['_source']['availability_zone'])
if 'availability_zone' in r['_source'] else 'unknown',
'ridCount':
1,
}
add_in_types(elem, r['_source']['resource_id'])
return types
@classmethod
@with_cache()
def get_instance_hour(cls,
keys,
date_from=None,
date_to=None,
min_hour=None):
date_from = date_from or datetime.utcnow().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s = s.filter("term", product_name='Amazon Elastic Compute Cloud')
s = s.filter('prefix', resource_id='i-')
s = s.query('wildcard', usage_type='*BoxUsage*')
agg = s.aggs.bucket('resource_id',
'terms',
field='resource_id',
size=0x7FFFFFFF)
agg.bucket('days',
'date_histogram',
field='usage_start_date',
interval='day',
min_doc_count=1)
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
instance_list = []
for instance in res['aggregations']['resource_id']['buckets']:
tmp_hours = []
for day in instance['days']['buckets']:
tmp_hours.append(day['doc_count'])
avg_hours = sum(tmp_hours) / float(len(tmp_hours))
if not min_hour or avg_hours >= min_hour:
instance_list.append(dict(id=instance['key'], hours=avg_hours))
return sorted(instance_list, key=lambda x: x['hours'], reverse=True)
@classmethod
@with_cache()
def get_s3_buckets_per_tag(cls, keys):
def _check_if_in_list(dict_list, value, key):
return next((item for item in dict_list if item[key] == value),
None)
def _parse_tag_keys_results(res):
bucket_tagged = []
for bucket_tag_key in res['aggregations']['tag_key']['buckets']:
buff_tag_key = _check_if_in_list(bucket_tagged,
bucket_tag_key['key'],
'tag_key')
if buff_tag_key is None:
buff_tag_key = {
"tag_key": bucket_tag_key['key'],
"tag_value": []
}
buff_tag_key = _parse_tag_values_results(
bucket_tag_key, buff_tag_key)
bucket_tagged.append(buff_tag_key)
return bucket_tagged
def _parse_tag_values_results(bucket_tag_key, buff_tag_key):
for bucket_tag_value in bucket_tag_key['tag_value']['buckets']:
buff_tag_value = _check_if_in_list(buff_tag_key['tag_value'],
bucket_tag_value['key'],
'tag_value')
if buff_tag_value is None:
buff_tag_value = {
"tag_value": bucket_tag_value['key'],
"s3_buckets": []
}
buff_tag_value = _parse_buckets_results(
buff_tag_value, bucket_tag_value)
buff_tag_key['tag_value'].append(buff_tag_value)
return buff_tag_key
def _parse_buckets_results(buff_tag_value, bucket_tag_value):
for bucket_resource_id in bucket_tag_value['ressource_id'][
'buckets']:
buff_bucket_resource_id = _check_if_in_list(
buff_tag_value['s3_buckets'], bucket_resource_id['key'],
'bucket_name')
if buff_bucket_resource_id is None:
buff_bucket_resource_id = {
"bucket_name":
bucket_resource_id['key'],
"account_id":
bucket_resource_id['account_id']['buckets'][0]['key']
}
buff_tag_value['s3_buckets'].append(buff_bucket_resource_id)
return buff_tag_value
s = cls.search()
s = s.filter(
'terms',
linked_account_id=keys if isinstance(keys, list) else [keys])
s = s.filter('term', product_name='Amazon Simple Storage Service')
s = s.query('exists', field="tag")
s = s.query('wildcard', item_description="*storage*")
agg = s.aggs.bucket('tag_key', 'terms', field="tag.key")
agg = agg.bucket('tag_value', 'terms', field='tag.value')
agg.bucket('ressource_id', 'terms',
field='resource_id').bucket('account_id',
'terms',
field='linked_account_id')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
'''
bucket_tagged structure
[{
"tag_key" : "KEY", # Unique in list
"tag_value": [{
"tag_value": "VALUE", # Unique in list
"s3_buckets": [{
"bucket_name": "BUCKET_NAME",
"account_id": "ACCOUND_ID"
}, {...}]
}, {...}]
}, {...}]
'''
bucket_tagged = _parse_tag_keys_results(res)
return bucket_tagged
@classmethod
@with_cache()
def get_s3_bandwidth_info_and_cost_per_name(cls,
key,
bucket_resource_ids,
date_from=None,
date_to=None):
date_from = date_from or (datetime.utcnow() - relativedelta(
month=1)).replace(day=1, hour=0, minute=0, second=0, microsecond=0)
date_to = date_to or date_from.replace(day=calendar.monthrange(
date_from.year, date_from.month)[1],
hour=23,
minute=59,
second=59,
microsecond=999999)
s = cls.search()
s = s.filter('term', linked_account_id=key)
s = s.filter('term', product_name='Amazon Simple Storage Service')
s = s.filter('terms',
resource_id=bucket_resource_ids if isinstance(
bucket_resource_ids, list) else [bucket_resource_ids])
s = s.filter('range',
usage_start_date={
'from': date_from.isoformat(),
'to': date_to.isoformat()
})
s = s.filter('wildcard', usage_type="*Bytes")
agg = s.aggs.bucket('bucket_name',
'terms',
field='resource_id',
size=0x7FFFFFFF)
agg.metric('cost', 'sum', field='cost')
agg = agg.bucket('transfer_type', 'terms', field='usage_type')
agg.metric('data', 'sum', field='usage_quantity')
res = client.search(index='awsdetailedlineitem',
body=s.to_dict(),
size=0,
request_timeout=60)
data = [{
"bucket_name":
bucket['key'],
"cost":
bucket['cost']['value'],
"transfer_stats": [{
"type": transfer_stat['key'],
"data": transfer_stat['data']['value']
} for transfer_stat in bucket['transfer_type']['buckets']]
} for bucket in res['aggregations']['bucket_name']['buckets']]
return data
class GoogleDailyResource(dsl.DocType):
class Meta:
index = 'googledailyresource'
identity = dsl.String(index='not_analyzed')
rid = dsl.String(index='not_analyzed')
product = dsl.String(index='not_analyzed')
project_name = dsl.String(index='not_analyzed')
date = dsl.Date(format='date_optional_time||epoch_millis')
cost = dsl.Double()
@classmethod
def daily_compute_cost(cls, identity_email):
s = cls.search()
s = s.filter('term', identity=identity_email)
s = s.filter('term',
product='com.google.cloud/services/compute-engine')
agg = s.aggs.bucket('intervals',
'date_histogram',
field='date',
interval='day',
min_doc_count=1)
agg.metric('cost', 'sum', field='cost')
res = client.search(index='googledailyresource',
body=s.to_dict(),
size=0)
for interval in res['aggregations']['intervals']['buckets']:
yield interval['key_as_string'].split(
'T')[0], interval['cost']['value']
@classmethod
def daily_cost_by_product(cls,
identity_email,
timespan=timedelta(days=7),
top=4):
now = datetime.utcnow()
rollup = cls.rollup_by_product(identity_email, now - timespan, now,
'day', top)
days = defaultdict(list)
for interval, product, cost in rollup:
days[interval.split('T')[0]].append(
dict(cost=cost, product=get_google_uri_name(product)))
res = dict(days=[dict(day=d, products=ps) for d, ps in days.items()])
res['days'] = sorted(res['days'], key=lambda x: x['day'])
return res
@classmethod
def month_cost_by_product(cls, identity_email, top=4):
now = datetime.utcnow()
rollup = cls.rollup_by_product(identity_email,
datetime(now.year, now.month, 1),
datetime.utcnow(), 'month', top)
month = {'products': []}
for interval, product, cost in rollup:
month['month'] = '-'.join(interval.split('-')[:2])
month['products'].append(
dict(cost=cost, product=get_google_uri_name(product)))
return month
@classmethod
def range_query(cls, identity_email, start, stop):
s = cls.search()
s = s.filter('term', identity=identity_email)
s = s.filter('range',
date={
'gt': start.isoformat(),
'lte': stop.isoformat()
})
return s
@classmethod
def rollup_by_product(cls, identity_email, start, stop, interval, top):
s = cls.range_query(identity_email, start, stop)
agg = s.aggs.bucket('intervals',
'date_histogram',
field='date',
interval=interval,
min_doc_count=1)
agg.bucket('product', 'terms', field='product').metric('cost',
'sum',
field='cost')
res = client.search(index='googledailyresource',
body=s.to_dict(),
size=0)
product_costs = defaultdict(float)
for interval in res['aggregations']['intervals']['buckets']:
for product in interval['product']['buckets']:
product_costs[product['key']] += product['cost']['value']
top_prods = set(
sorted(product_costs, key=lambda p: product_costs[p],
reverse=True)[:top])
interval_prods = defaultdict(set)
for interval in res['aggregations']['intervals']['buckets']:
for product in interval['product']['buckets']:
if product['key'] in top_prods:
yield interval['key_as_string'], product['key'], product[
'cost']['value']
interval_prods[interval['key_as_string']].add(
product['key'])
for interval, prods in interval_prods.items():
missing = top_prods - prods
for prod in missing:
yield interval, prod, 0.0
@classmethod
def monthly_aggregates_resource(cls, identity_email):
s = cls.search()
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('months',
'date_histogram',
field='date',
interval='month',
min_doc_count=1)
agg.bucket('rid', 'terms', field='rid',
size=0x7FFFFFFF).metric('cost', 'sum', field='cost')
res = client.search(index='googledailyresource',
body=s.to_dict(),
size=0)
months = []
for month in res['aggregations']['months']['buckets']:
resources = []
for resource in month['rid']['buckets']:
resources.append(
dict(cost=resource['cost']['value'],
resource=resource['key']))
if resources == []:
continue
months.append(
dict(month=month['key_as_string'].split('T')[0],
resources=resources))
return dict(months=months)
@classmethod
def monthly_aggregates_project(cls, identity_email):
s = cls.search()
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('months',
'date_histogram',
field='date',
interval='month',
min_doc_count=1)
agg.bucket('project_name',
'terms',
field='project_name',
size=0x7FFFFFFF).metric('cost', 'sum', field='cost')
res = client.search(index='googledailyresource',
body=s.to_dict(),
size=0)
months = []
for month in res['aggregations']['months']['buckets']:
projects = []
for project in month['project_name']['buckets']:
projects.append(
dict(cost=project['cost']['value'],
project=project['key']))
if projects == []:
continue
months.append(
dict(month=month['key_as_string'].split('T')[0],
projects=projects))
return dict(months=months)
class GoogleMetric(dsl.DocType):
class Meta:
index = 'googlemetric'
identity = dsl.String(index='not_analyzed')
resource = dsl.String(index='not_analyzed')
metric = dsl.String(index='not_analyzed')
time = dsl.Date(format='date_optional_time||epoch_millis')
value = dsl.Double()
@classmethod
def daily_cpu_utilization(cls, identity_email):
s = cls.search()
s = s.filter('term', identity=identity_email)
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/cpu/utilization')
agg = s.aggs.bucket('intervals', 'date_histogram', field='time', interval='day', min_doc_count=1)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for interval in res['aggregations']['intervals']['buckets']:
yield interval['key_as_string'].split('T')[0], interval['utilization']['value']
@classmethod
def get_cpu_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/cpu/utilization')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_disk_read_iops_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/disk/read_ops_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_disk_write_iops_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/disk/write_ops_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_disk_read_bytes_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/disk/read_bytes_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_disk_write_bytes_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/disk/write_bytes_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_network_in_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/network/received_bytes_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']
@classmethod
def get_network_out_usage(cls, identity_email, timespan=timedelta(days=30)):
s = cls.search()
s = s.filter('range', time={'gt': (datetime.utcnow() - timespan).isoformat()})
s = s.filter('term', metric='GCLOUD/COMPUTE:compute.googleapis.com/instance/network/sent_bytes_count')
s = s.filter('term', identity=identity_email)
agg = s.aggs.bucket('resources', 'terms', field='resource', size=300)
agg.metric('utilization', 'avg', field='value')
res = client.search(index='googlemetric', body=s.to_dict(), size=0)
for resource in res['aggregations']['resources']['buckets']:
yield resource['key'], resource['utilization']['value']