def write_detailed_cloud_watch_event_logs(context, cw): events = context[c.KEY_AGGREGATOR].events params = [] for event_name, event_count in iteritems(events): params.append({ "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': event_name }], "Timestamp": datetime.datetime.utcnow(), "Value": event_count, "Unit": 'Count' }) if len(params) >= c.CW_MAX_METRIC_SUBMISSIONS: cw.put_metric_data( util.get_cloudwatch_namespace( util.get_cloudwatch_namespace( context[c.KEY_LAMBDA_FUNCTION])), params) params = [] if len(params) > 0: cw.put_metric_data( util.get_cloudwatch_namespace( util.get_cloudwatch_namespace(context[c.KEY_LAMBDA_FUNCTION])), params)
def main(event, lambdacontext): starttime = time.time() queue_url = event.get(c.KEY_SQS_QUEUE_URL, None) print "Started consumer with queue url '{}'".format(queue_url) context = event.get("context", {}) context[c.KEY_SQS_QUEUE_URL] = queue_url context[c.KEY_LAMBDA_FUNCTION] = lambdacontext.function_name if hasattr(lambdacontext, 'function_name') else None context[c.KEY_REQUEST_ID] = lambdacontext.aws_request_id if hasattr(lambdacontext, 'aws_request_id') else None context[c.KEY_IS_LAMBDA_ENV] = context[c.KEY_REQUEST_ID] is not None prefix = util.get_stack_name_from_arn(os.environ[c.ENV_DEPLOYMENT_STACK_ARN]) context[c.KEY_STACK_PREFIX] = prefix context[c.KEY_SQS] = Sqs(context, "{0}_".format(prefix)) context[c.KEY_SQS_AMOEBA] = Sqs(context, "{0}{1}_".format(prefix, c.KEY_SQS_AMOEBA_SUFFIX)) context[c.KEY_SQS_AMOEBA].set_queue_url(lowest_load_queue=True) context[c.KEY_LAMBDA] = Lambda(context) context[c.KEY_CLOUDWATCH] = CloudWatch(context) context[c.KEY_THREAD_POOL] = ThreadPool(context, 8) context[c.KEY_METRIC_BUCKET] = os.environ[c.RES_S3_STORAGE] context[c.KEY_START_TIME] = starttime context[c.CW_ATTR_SAVE_DURATION] = context[c.KEY_CLOUDWATCH].avg_save_duration(util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN])) context[c.CW_ATTR_DELETE_DURATION] = context[c.KEY_CLOUDWATCH].avg_delete_duration(util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN])) context[c.KEY_SUCCEEDED_MSG_IDS] = [] process(context) del context gc.collect() return { 'StatusCode': 200 }
def consumer_sqs_delete_duration(request): cw = CloudWatch() return cw.get_metric( util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN]), c.CW_METRIC_NAME_PROCESSED, c.CW_METRIC_DIMENSION_NAME_CONSUMER, c.CW_METRIC_DIMENSION_DEL, "Average", start=(datetime.datetime.today() - datetime.timedelta(hours=8)))
def write_cloudwatch_metrics(context, save_duration, delete_duration): cw = context[c.KEY_CLOUDWATCH] cw.put_metric_data(util.get_cloudwatch_namespace(context[c.KEY_LAMBDA_FUNCTION]), [ { "MetricName":c.CW_METRIC_NAME_PROCESSED, "Dimensions":[{'Name':c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_ROWS}], "Timestamp":datetime.datetime.utcnow(), "Value":context[c.KEY_AGGREGATOR].rows, "Unit":'Count' }, { "MetricName":c.CW_METRIC_NAME_PROCESSED, "Dimensions":[{'Name':c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_BYTES}], "Timestamp":datetime.datetime.utcnow(), "Value":context[c.KEY_AGGREGATOR].bytes_uncompressed, "Unit":'Bytes' }, { "MetricName":c.CW_METRIC_NAME_PROCESSED, "Dimensions":[{'Name':c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_MSG}], "Timestamp":datetime.datetime.utcnow(), "Value":context[c.KEY_AGGREGATOR].messages, "Unit":'Count' }, { "MetricName":c.CW_METRIC_NAME_PROCESSED, "Dimensions":[{'Name':c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_SAVE}], "Timestamp":datetime.datetime.utcnow(), "Value":save_duration, "Unit":'Seconds' }, { "MetricName":c.CW_METRIC_NAME_PROCESSED, "Dimensions":[{'Name':c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_DEL}], "Timestamp":datetime.datetime.utcnow(), "Value":delete_duration, "Unit":'Seconds' }, ] ) if context.get(c.KEY_WRITE_DETAILED_CLOUDWATCH_EVENTS, False): util.debug_print("Sending detailed CloudWatch events") write_detailed_cloud_watch_event_logs(context, cw)
def sum_delete_duration(request): cw = CloudWatch() return cw.sum_delete_duration( util.get_cloudwatch_namespace(os.environ[c.ENV_DEPLOYMENT_STACK_ARN]))
def write_cloudwatch_metrics(context, save_duration, delete_duration): cw = context[c.KEY_CLOUDWATCH] cw.put_metric_data( util.get_cloudwatch_namespace(context[c.KEY_LAMBDA_FUNCTION]), [ { "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_ROWS }], "Timestamp": datetime.datetime.utcnow(), "Value": context[c.KEY_AGGREGATOR].rows, "Unit": 'Count' }, { "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_BYTES }], "Timestamp": datetime.datetime.utcnow(), "Value": context[c.KEY_AGGREGATOR].bytes_uncompressed, "Unit": 'Bytes' }, { "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_MSG }], "Timestamp": datetime.datetime.utcnow(), "Value": context[c.KEY_AGGREGATOR].messages, "Unit": 'Count' }, { "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_SAVE }], "Timestamp": datetime.datetime.utcnow(), "Value": save_duration, "Unit": 'Seconds' }, { "MetricName": c.CW_METRIC_NAME_PROCESSED, "Dimensions": [{ 'Name': c.CW_METRIC_DIMENSION_NAME_CONSUMER, 'Value': c.CW_METRIC_DIMENSION_DEL }], "Timestamp": datetime.datetime.utcnow(), "Value": delete_duration, "Unit": 'Seconds' }, ])