def describe_elb(accounts): now = datetime.utcnow() date_from = now.replace(hour=0, minute=0, second=0, microsecond=0) - relativedelta(days=30) date_to = now.replace(hour=23, minute=59, second=59, microsecond=999999) ares = [] for account in accounts: res = [] elb_usage = AWSDetailedLineitem.get_elb_usage_a_day( account.get_aws_user_id(), date_from=date_from, date_to=date_to) for elb in AWSELBInfo.get_elb_info(account.get_aws_user_id()): for usage in elb_usage: if usage['rid'].endswith(elb['name']) or usage['rid'].split( '/')[-2] == elb['name']: usage['region'] = elb['region'] usage['name'] = elb['name'] usage['instances'] = elb['instances'] res.append(usage) for d in res: if d not in ares: ares.append(d) if not len(ares): if AWSDetailedLineitem.keys_has_data( [account.get_aws_user_id() for account in accounts]): return jsonify( message="You do not have ELB set up in your environment") return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(elbs=ares)
def aws_cost_by_resource_months(accounts): raw_data = AWSDetailedLineitem.get_first_to_last_date( [account.get_aws_user_id() for account in accounts]) if not raw_data: return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(months=[data.strftime("%Y-%m-01") for data in raw_data])
def aws_cpu_days_of_the_week_usage_m(accounts): """--- get: tags: - aws produces: - application/json description: &desc AWS CPU usage by days of the week summary: *desc responses: 200: description: List of days schema: properties: hours: type: array items: properties: day: type: string cpu: type: number 403: description: Not logged in 404: description: AWS account not registered """ days = AWSMetric.days_of_the_week_cpu_usage(account.key for account in accounts) if not days: return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(dict(hours=days))
def aws_get_resource_tags(accounts): tags = AWSDetailedLineitem.get_available_tags( [account.get_aws_user_id() for account in accounts])['tags'] if not len(tags): return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(tags=sorted(tags, key=unicode.lower))
def aws_accounts_m_stats_usagecost(accounts): def get_account_data(account): for date, cpu_usage in dict(AWSMetric.daily_cpu_utilization(account.key)).iteritems(): yield (date, cpu_usage, None) for date, cost in dict(AWSDetailedLineitem.get_ec2_daily_cost(account.get_aws_user_id())).iteritems(): yield (date, None, cost) @with_cache() def get_all_account_data(): return list( itertools.chain.from_iterable( get_account_data(account) for account in accounts ) ) data = get_all_account_data() days = {} for day, cpu_usage, cost in data: day_data = days.setdefault(day, {'day': day, 'cpu': None, 'cost': None}) if cpu_usage is not None: day_data['cpu'] = (day_data['cpu'] or 0.0) + cpu_usage if cost is not None: day_data['cost'] = (day_data['cost'] or 0.0) + cost res = sorted([ value for value in days.itervalues() if value['cpu'] is not None and value['cost'] is not None # Comment/remove if None values are OK ], key=lambda e: e['day']) if not res: return jsonify(message=get_next_update_estimation_message_aws(accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(days=res)
def aws_get_resource_tags_with_data(accounts): tags = list(set(itertools.chain.from_iterable( AWSDetailedLineitem.get_available_tags(account.get_aws_user_id(), only_with_data=account.key)['tags'] for account in accounts ))) if not len(tags): return jsonify(message=get_next_update_estimation_message_aws(accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(tags=sorted(tags, key=unicode.lower))
def aws_accounts_m_stats_yearlycostbyproduct(accounts, nb_years): """--- get: tags: - aws produces: - application/json description: &desc Get yearly costs summed by product summary: *desc responses: 200: description: List of AWS accounts schema: properties: years: type: array items: properties: year: type: string products: type: array items: properties: cost: type: number product: type: string 403: description: Not logged in 404: description: AWS account not registered """ assert len(accounts) > 0 now = datetime.utcnow() date_from = now.replace( month=1, day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(years=nb_years - 1) date_to = now.replace(month=12, day=31, hour=23, minute=59, second=59, microsecond=999999) data = AWSDetailedLineitem.get_yearly_cost_by_product( keys=[account.get_aws_user_id() for account in accounts], date_from=date_from, date_to=date_to)['years'] for d in data: d['products'] = cut_cost_by_product( sorted(d['products'], key=lambda x: x['cost'], reverse=True), int(request.args['show']) - 1 if 'show' in request.args else 9) if not len(data): return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(years=data)
def aws_rds_estimation_m(accounts): """--- get: tags: - aws produces: - application/json description: &desc Get RDS cost estimation summary: *desc responses: 200: description: List of instances schema: properties: result: type: array items: properties: size: type: string name: type: string engine: type: string multi-az: type: boolean prices: type: array items: properties: name: type: string cost: type: number 403: description: Not logged in 404: description: AWS account not registered """ my_db_resources = (MyDBResourcesAWS.query.filter( MyDBResourcesAWS.key == a.key).order_by(desc( MyDBResourcesAWS.date)).first() for a in accounts) res = list( itertools.chain.from_iterable([] if not r else r.json() for r in my_db_resources)) last_updated = None # TODO: Handle this in a meaningful way. next_update_delta = timedelta(hours=AWS_RESOURCES_INTERVAL_HOURS) next_update, remainder = divmod(next_update_delta.seconds, 3600) if next_update < 1: next_update = 1 if not res: return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_RESOURCES_INTERVAL_HOURS), last_updated=last_updated, next_update=next_update) return jsonify( dict(result=res, last_updated=last_updated, next_update=next_update))
def aws_instance_stats(accounts): """--- get: tags: - aws produces: - application/json description: &desc Get instance statistics summary: *desc responses: 200: description: List of stats schema: properties: stats: type: array items: properties: reserved: type: number unreserved: type: number unused: type: number stopped: type: number 403: description: Not logged in 404: description: AWS account not registered """ datasets = [] reserved_report = [] for account in accounts: instance_stats = AWSStat.latest_instance_stats(account.key) if 'stats' in instance_stats and len(instance_stats['stats']): datasets.append(instance_stats['stats'][0]) if 'reserved_instances_report' in instance_stats['stats'][0]: reserved_report += instance_stats['stats'][0][ 'reserved_instances_report'] if not len(datasets): return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify({ 'stats': [{ 'reserved_report': reserved_report, 'reserved': sum(d['reserved'] for d in datasets), 'stopped': sum(d['stopped'] for d in datasets), 'unreserved': sum(d['unreserved'] for d in datasets), 'unused': sum(d['unused'] for d in datasets), 'time': datasets[0]['time'], }] })
def aws_accounts_m_stats_monthlycostbyproduct(accounts, nb_months): """--- get: tags: - aws produces: - application/json description: &desc Get monthly costs summed by product summary: *desc responses: 200: description: List of AWS accounts schema: properties: months: type: array items: properties: month: type: string products: type: array items: properties: cost: type: number product: type: string 403: description: Not logged in 404: description: AWS account not registered """ assert len(accounts) > 0 now = datetime.utcnow().replace(hour=23, minute=59, second=59, microsecond=999999) now = AWSDetailedLineitem.get_last_date([account.get_aws_user_id() for account in accounts], limit=now) date_from = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(months=nb_months - 1) date_to = now.replace(day=calendar.monthrange(now.year, now.month)[1], hour=23, minute=59, second=59, microsecond=999999) data = AWSDetailedLineitem.get_monthly_cost_by_product(keys=[account.get_aws_user_id() for account in accounts], date_from=date_from, date_to=date_to)['months'] for d in data: if 'csv' not in request.args: d['products'] = cut_cost_by_product(sorted(d['products'], key=lambda x: x['cost'], reverse=True), int(request.args['show']) - 1 if 'show' in request.args else 9) if not len(data): return jsonify(message=get_next_update_estimation_message_aws(accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) if 'csv' in request.args: return Response(generate_csv(data, 'products', 'product'), mimetype='text/csv') return jsonify(months=data)
def compare_providers_aws(accounts): my_resources = [ MyResourcesAWS.query.filter(MyResourcesAWS.key == a.key).order_by( desc(MyResourcesAWS.date)).first() for a in accounts ] res = list( itertools.chain.from_iterable([] if not r else r.json() for r in my_resources)) last_updated = None # TODO: Handle this in a meaningful way with several keys. next_update_delta = timedelta(hours=AWS_RESOURCES_INTERVAL_HOURS) next_update, remainder = divmod(next_update_delta.seconds, 3600) if next_update < 1: next_update = 1 if not res: return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_RESOURCES_INTERVAL_HOURS), last_updated=last_updated, next_update=next_update) return jsonify( dict(result=res, last_updated=last_updated, next_update=next_update))
def aws_cost_by_resource_month_chart_m(accounts, month): # TODO: Use ES agg to categorize try: date_from = datetime.strptime(month, "%Y-%m-%d") except: return jsonify(error='Not found.'), 404 raw_data = [ AWSDetailedLineitem.get_cost_by_resource(account.get_aws_user_id(), date_from=date_from) for account in accounts ] data = [] def get_cat_with_cost(cost): x = 1 while cost > x: x *= 10 return x def add_resource_in_data(new): new_cat = get_cat_with_cost(new['cost']) for cat in data: if cat['category'] == '<{}'.format(new_cat): cat['total'] += new['cost'] return data.append(dict(category='<{}'.format(new_cat), total=new['cost'])) for one in raw_data: for new in one: add_resource_in_data(new) if not len(data): return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) max_cat = 0 for i in range(len(data)): if len(data[i]['category']) > len(data[max_cat]['category']): max_cat = i data[max_cat]['category'] = data[max_cat]['category'][:-1] data[max_cat]['category'] = data[max_cat]['category'].replace('<', '>', 1) return jsonify(categories=data)
def get_lambda_usage(accounts): """--- get: tags: - aws produces: - application/json description: &desc Get average Lambda usage summary: *desc responses: 200: description: List of lambda resources schema: properties: lambdas: type: array items: properties: rid: type: string name: type: string gb_seconds: type: number requests: type: number cost: 403: description: Not logged in 404: description: AWS account not registered """ res = AWSDetailedLineitem.get_lambda_usage( [account.get_aws_user_id() for account in accounts]) if not len(res): return jsonify(message=get_next_update_estimation_message_aws( account, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(lambdas=res)
def aws_reservation_forecast(accounts): forecast = get_reservation_forecast(account.key for account in accounts) if not forecast or ('instances' in forecast and not forecast['instances']): return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS)) return jsonify(forecast)
def get_s3_space_usage(accounts): """--- get: tags: - aws produces: - application/json description: &desc Get S3 space usage summary: *desc responses: 200: description: List of buckets schema: properties: buckets: type: array items: properties: location: type: string name: type: string type: type: string provider: type: string used_space: type: number prices: type: array items: properties: name: type: string cost: type: number 403: description: Not logged in 404: description: AWS account not registered """ res = [AWSStat.latest_s3_space_usage(account) for account in accounts] buckets = sum((r['buckets'] for r in res if r and 'buckets' in r), []) last_updated = None #TODO: Decide how to handle this with several accounts. next_update_delta = timedelta( hours=AWS_KEY_PROCESSING_INTERVAL_HOURS ) #TODO: Same here, I put a value for the demos. next_update, remainder = divmod(next_update_delta.seconds, 3600) if next_update < 1: next_update = 1 for bucket in buckets: bucket[ 'metadata'] = None #TODO: See how to get the account the bucket comes from. if not buckets: return jsonify(message=get_next_update_estimation_message_aws( accounts, AWS_KEY_PROCESSING_INTERVAL_HOURS), last_updated=last_updated, next_update=next_update) return jsonify({ 'buckets': buckets, 'last_updated': last_updated, 'next_update': next_update, })