def __init__(self, base_usage, proposedDurationMs, proposedMemSizeMb): self.memSizeMb = proposedMemSizeMb self.memUsedPct = 0 self.memUsedPct = float(100 * base_usage.avgMemUsedMb / proposedMemSizeMb) self.durationMs = proposedDurationMs args = { 'region': base_usage.region, 'requestCount': base_usage.projectedRequestCount, 'avgDurationMs': self.durationMs, 'memoryMb': proposedMemSizeMb } self.cost = lambdapricing.calculate( data.LambdaPriceDimension(**args))['totalCost'] self.savingsAmt = round((base_usage.projectedCost - self.cost), 2) self.savingsPct = 0.00 if base_usage.projectedCost: self.savingsPct = round( (100 * self.savingsAmt / base_usage.projectedCost), 2) self.timePeriod = MONTHLY
def __init__(self, region, requestCount, avgDurationMs, avgMemUsedMb, memSizeMb, startTs, endTs, projectedPeriod): self.region = region self.requestCount = 0 if requestCount: self.requestCount = requestCount self.avgDurationMs = 0 if avgDurationMs: self.avgDurationMs = int(avgDurationMs) self.avgMemUsedMb = int(avgMemUsedMb) self.memSizeMb = memSizeMb self.memUsedPct = 0.00 if memSizeMb: self.memUsedPct = round(float(100 * avgMemUsedMb / memSizeMb), 2) self.startTs = startTs self.endTs = endTs self.elapsedMs = endTs - startTs self.avgTps = 0 if self.elapsedMs: self.avgTps = round( (1000 * float(self.requestCount) / float(self.elapsedMs)), 4) self.projectedPeriod = projectedPeriod self.projectedRequestCount = self.get_projected_request_count( requestCount) args = { 'region': region, 'requestCount': self.projectedRequestCount, 'avgDurationMs': math.ceil(avgDurationMs / 100) * 100, 'memoryMb': memSizeMb } print "args: {}".format(args) self.projectedCost = lambdapricing.calculate( data.LambdaPriceDimension(**args))['totalCost']
def handler(event, context): log.info("Received event {}".format(json.dumps(event))) init_clients(context) result = {} pricing_records = [] ec2Cost = 0 rdsCost = 0 lambdaCost = 0 ddbCost = 0 kinesisCost = 0 totalCost = 0 #First, get the tags we'll be searching for, from the CloudWatch scheduled event tagkey = "" tagvalue = "" if 'tag' in event: tagkey = event['tag']['key'] tagvalue = event['tag']['value'] if tagkey == "" or tagvalue == "": log.error("No tags specified, aborting function!") return {} log.info("Will search resources with the following tag:[" + tagkey + "] - value[" + tagvalue + "]") resource_manager = ResourceManager(tagkey, tagvalue) start, end = calculate_time_range() elb_hours = 0 elb_data_processed_gb = 0 elb_instances = {} #Get tagged ELB(s) and their registered instances #taggedelbs = find_elbs(tagkey, tagvalue) taggedelbs = resource_manager.get_resource_ids(SERVICE_ELB, RESOURCE_ELB) if taggedelbs: log.info("Found tagged ELBs:{}".format(taggedelbs)) elb_hours = len(taggedelbs) * HOURS_DICT[DEFAULT_FORECAST_PERIOD] elb_instances = get_elb_instances(taggedelbs) #Get all EC2 instances registered with each tagged ELB, so we can calculate ELB data processed #Registered instances will be used for data processed calculation, and not for instance hours, unless they're tagged. if elb_instances: try: log.info( "Found registered EC2 instances to tagged ELBs [{}]:{}". format(taggedelbs, elb_instances.keys())) elb_data_processed_gb = calculate_elb_data_processed( start, end, elb_instances) * calculate_forecast_factor() / (10**9) except Exception as failure: log.error('Error calculating costs for tagged ELBs: %s', failure.message) else: log.info( "Didn't find any EC2 instances registered to tagged ELBs [{}]". format(taggedelbs)) else: log.info("No tagged ELBs found") #Get tagged EC2 instances ec2_instances = get_ec2_instances_by_tag(tagkey, tagvalue) if ec2_instances: log.info("Tagged EC2 instances:{}".format(ec2_instances.keys())) else: log.info("Didn't find any tagged, running EC2 instances") #Calculate ELB cost if elb_hours: elb_cost = ec2pricing.calculate( data.Ec2PriceDimension(region=region, elbHours=elb_hours, elbDataProcessedGb=elb_data_processed_gb)) if 'pricingRecords' in elb_cost: pricing_records.extend(elb_cost['pricingRecords']) ec2Cost = ec2Cost + elb_cost['totalCost'] #Calculate EC2 compute time for ALL instance types found (subscribed to ELB or not) - group by instance types all_instance_dict = {} all_instance_dict.update(ec2_instances) all_instance_types = get_instance_type_count(all_instance_dict) log.info("All instance types:{}".format(all_instance_types)) #Calculate EC2 compute time cost for instance_type in all_instance_types: try: ec2_compute_cost = ec2pricing.calculate( data.Ec2PriceDimension( region=region, instanceType=instance_type, instanceHours=all_instance_types[instance_type] * HOURS_DICT[DEFAULT_FORECAST_PERIOD])) if 'pricingRecords' in ec2_compute_cost: pricing_records.extend(ec2_compute_cost['pricingRecords']) ec2Cost = ec2Cost + ec2_compute_cost['totalCost'] except Exception as failure: log.error('Error processing %s: %s', instance_type, failure.message) #Get provisioned storage by volume type, and provisioned IOPS (if applicable) ebs_storage_dict, piops = get_storage_by_ebs_type(all_instance_dict) #Calculate EBS storagecost for k in ebs_storage_dict.keys(): if k == 'io1': pricing_piops = piops else: pricing_piops = 0 try: ebs_storage_cost = ec2pricing.calculate( data.Ec2PriceDimension(region=region, ebsVolumeType=k, ebsStorageGbMonth=ebs_storage_dict[k], pIops=pricing_piops)) if 'pricingRecords' in ebs_storage_cost: pricing_records.extend(ebs_storage_cost['pricingRecords']) ec2Cost = ec2Cost + ebs_storage_cost['totalCost'] except Exception as failure: log.error('Error processing ebs storage costs: %s', failure.message) #Get tagged RDS DB instances #db_instances = get_db_instances_by_tag(tagkey, tagvalue) db_instances = get_db_instances_by_tag( resource_manager.get_resource_ids(SERVICE_RDS, RESOURCE_RDS_DB_INSTANCE)) if db_instances: log.info("Found the following tagged DB instances:{}".format( db_instances.keys())) else: log.info("Didn't find any tagged RDS DB instances") #Calculate RDS instance time for ALL instance types found - group by DB instance types all_db_instance_dict = {} all_db_instance_dict.update(db_instances) all_db_instance_types = get_db_instance_type_count(all_db_instance_dict) all_db_storage_types = get_db_storage_type_count(all_db_instance_dict) #Calculate RDS instance time cost rds_instance_cost = {} for db_instance_type in all_db_instance_types: try: dbInstanceClass = db_instance_type.split("|")[0] engine = db_instance_type.split("|")[1] licenseModel = db_instance_type.split("|")[2] multiAz = bool(int(db_instance_type.split("|")[3])) log.info("Calculating RDS DB Instance compute time") rds_instance_cost = rdspricing.calculate( data.RdsPriceDimension( region=region, dbInstanceClass=dbInstanceClass, multiAz=multiAz, engine=engine, licenseModel=licenseModel, instanceHours=all_db_instance_types[db_instance_type] * HOURS_DICT[DEFAULT_FORECAST_PERIOD])) if 'pricingRecords' in rds_instance_cost: pricing_records.extend(rds_instance_cost['pricingRecords']) rdsCost = rdsCost + rds_instance_cost['totalCost'] except Exception as failure: log.error('Error processing RDS instance time costs: %s', failure.message) #Calculate RDS storage cost rds_storage_cost = {} for storage_key in all_db_storage_types.keys(): try: storageType = storage_key.split("|")[0] multiAz = bool(int(storage_key.split("|")[1])) storageGbMonth = all_db_storage_types[storage_key][ 'AllocatedStorage'] iops = all_db_storage_types[storage_key]['Iops'] log.info("Calculating RDS DB Instance Storage") rds_storage_cost = rdspricing.calculate( data.RdsPriceDimension(region=region, storageType=storageType, multiAz=multiAz, storageGbMonth=storageGbMonth, iops=iops)) if 'pricingRecords' in rds_storage_cost: pricing_records.extend(rds_storage_cost['pricingRecords']) rdsCost = rdsCost + rds_storage_cost['totalCost'] except Exception as failure: log.error('Error processing RDS storage costs: %s', failure.message) #RDS Data Transfer - the Lambda function will assume all data transfer happens between RDS and EC2 instances #Lambda functions #TODO: add support for lambda function qualifiers lambdafunctions = resource_manager.get_resources(SERVICE_LAMBDA, RESOURCE_LAMBDA_FUNCTION) for func in lambdafunctions: executions = calculate_lambda_executions(start, end, func) avgduration = calculate_lambda_duration(start, end, func) funcname = '' qualifier = '' fullname = '' funcname = func.id fullname = funcname #if 'qualifier' in func: # qualifier = func['qualifier'] # fullname += ":"+qualifier memory = get_lambda_memory(funcname, qualifier) log.info( "Executions for Lambda function [{}]: [{}] - Memory:[{}] - Avg Duration:[{}]" .format(funcname, executions, memory, avgduration)) if executions and avgduration: try: #Note we're setting data transfer = 0, since we don't have a way to calculate it based on CW metrics alone #TODO:call a single time and include a GB-s price dimension to the Lambda calculator lambdapdim = data.LambdaPriceDimension( region=region, requestCount=executions * calculate_forecast_factor(), avgDurationMs=avgduration, memoryMb=memory, dataTranferOutInternetGb=0, dataTranferOutIntraRegionGb=0, dataTranferOutInterRegionGb=0, toRegion='') lambda_func_cost = lambdapricing.calculate(lambdapdim) if 'pricingRecords' in lambda_func_cost: pricing_records.extend(lambda_func_cost['pricingRecords']) lambdaCost = lambdaCost + lambda_func_cost['totalCost'] except Exception as failure: log.error('Error processing Lambda costs: %s', failure.message) else: log.info( "Skipping pricing calculation for function [{}] - qualifier [{}] due to lack of executions in [{}-minute] time window" .format(fullname, qualifier, METRIC_WINDOW)) #DynamoDB totalRead = 0 totalWrite = 0 ddbtables = resource_manager.get_resources(SERVICE_DYNAMODB, RESOURCE_DDB_TABLE) #Provisioned Capacity Units for t in ddbtables: read, write = get_ddb_capacity_units(t.id) log.info( "Dynamo DB Provisioned Capacity Units - Table:{} Read:{} Write:{}". format(t.id, read, write)) totalRead += read totalWrite += write #TODO: add support for storage if totalRead and totalWrite: ddbpdim = data.DynamoDBPriceDimension( region=region, readCapacityUnitHours=totalRead * HOURS_DICT[FORECAST_PERIOD_MONTHLY], writeCapacityUnitHours=totalWrite * HOURS_DICT[FORECAST_PERIOD_MONTHLY]) ddbtable_cost = ddbpricing.calculate(ddbpdim) if 'pricingRecords' in ddbtable_cost: pricing_records.extend(ddbtable_cost['pricingRecords']) ddbCost = ddbCost + ddbtable_cost['totalCost'] #Kinesis Streams streams = resource_manager.get_resources(SERVICE_KINESIS, RESOURCE_STREAM) totalShards = 0 totalExtendedRetentionCount = 0 totalPutPayloadUnits = 0 for s in streams: log.info("Stream:[{}]".format(s.id)) tmpShardCount, tmpExtendedRetentionCount = get_kinesis_stream_shards( s.id) totalShards += tmpShardCount totalExtendedRetentionCount += tmpExtendedRetentionCount totalPutPayloadUnits += calculate_kinesis_put_payload_units( start, end, s.id) if totalShards: kinesispdim = data.KinesisPriceDimension( region=region, shardHours=totalShards * HOURS_DICT[FORECAST_PERIOD_MONTHLY], extendedDataRetentionHours=totalExtendedRetentionCount * HOURS_DICT[FORECAST_PERIOD_MONTHLY], putPayloadUnits=totalPutPayloadUnits * calculate_forecast_factor()) stream_cost = kinesispricing.calculate(kinesispdim) if 'pricingRecords' in stream_cost: pricing_records.extend(stream_cost['pricingRecords']) kinesisCost = kinesisCost + stream_cost['totalCost'] #Do this after all calculations for all supported services have concluded totalCost = ec2Cost + rdsCost + lambdaCost + ddbCost + kinesisCost result['pricingRecords'] = pricing_records result['totalCost'] = round(totalCost, 2) result['forecastPeriod'] = DEFAULT_FORECAST_PERIOD result['currency'] = CW_METRIC_DIMENSION_CURRENCY_USD #Publish metrics to CloudWatch using the default namespace if tagkey: put_cw_metric_data(end, ec2Cost, CW_METRIC_DIMENSION_SERVICE_NAME_EC2, tagkey, tagvalue) put_cw_metric_data(end, rdsCost, CW_METRIC_DIMENSION_SERVICE_NAME_RDS, tagkey, tagvalue) put_cw_metric_data(end, lambdaCost, CW_METRIC_DIMENSION_SERVICE_NAME_LAMBDA, tagkey, tagvalue) put_cw_metric_data(end, ddbCost, CW_METRIC_DIMENSION_SERVICE_NAME_DYNAMODB, tagkey, tagvalue) put_cw_metric_data(end, kinesisCost, CW_METRIC_DIMENSION_SERVICE_NAME_KINESIS, tagkey, tagvalue) put_cw_metric_data(end, totalCost, CW_METRIC_DIMENSION_SERVICE_NAME_TOTAL, tagkey, tagvalue) log.info(json.dumps(result, sort_keys=False, indent=4)) return result