def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None): """ Query CloudWatch and return the results in a Google Visualizations API-friendly format Arguments: `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information) `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter """ # Initialize data description, columns to be returned, and result set description = { "Timestamp": ("datetime", "Timestamp")} columns = ["Timestamp"] rs = [] current_timezone = timezone('UTC') utc = pytz.utc # Build option list opts = ['unit','metric','namespace','statistics','period', 'dimensions', 'prefix', 'start_time', 'end_time', 'calc_rate', 'region', 'range', 'timezone'] # Set default parameter values from config.py qa = DEFAULTS.copy() # Set passed args for opt in opts: if opt in cloudviz_query: qa[opt] = cloudviz_query[opt] # Convert timestamps to datetimes if necessary for time in ['start_time','end_time']: if time in qa: if type(qa[time]) == str or type(qa[time]) == unicode: qa[time] = datetime.strptime(qa[time].split(".")[0], '%Y-%m-%dT%H:%M:%S') # If both start_time and end_time are specified, do nothing. if 'start_time' in qa and 'end_time' in qa: pass # If only one of the times is specified, fill in the other based on range elif 'start_time' in qa and 'range' in qa: qa['end_time'] = qa['start_time'] + timedelta(hours=qa['range']) elif 'range' in qa and 'end_time' in qa: qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range']) # If neither is specified, use range leading up to current time else: qa['end_time'] = datetime.utcnow() qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range']) if 'timezone' in qa: current_timezone = timezone(qa['timezone']) data_map = {} # Parse, build, and run each CloudWatch query cloudwatch_opts = ['unit', 'metric', 'namespace', 'statistics', 'period', 'dimensions', 'prefix', 'calc_rate', 'region'] for cloudwatch_query in cloudviz_query['cloudwatch_queries']: args = qa.copy() # Override top-level vars for opt in cloudwatch_opts: if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt] # Calculate time range for period determination/sanity-check delta = args['end_time'] - args['start_time'] delta_seconds = ( delta.days * 24 * 60 * 60 ) + delta.seconds + 1 #round microseconds up # Determine min period as the smallest multiple of 60 that won't result in too many data points min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60) if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0: min_period += 60 if 'period' in qa: if args['period'] < min_period: args['period'] = min_period else: args['period'] = min_period # Make sure period isn't smaller than CloudWatch allows if args['period'] < CW_MIN_PERIOD: args['period'] = CW_MIN_PERIOD # Defaulting AWS region to us-east-1 if not 'region' in args: args['region'] = 'us-east-1' # Use AWS keys if provided, otherwise just let the boto look it up if aws_access_key_id and aws_secret_access_key: c = boto.ec2.cloudwatch.connect_to_region( args['region'], aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, is_secure=False) else: c = boto.ec2.cloudwatch.connect_to_region(args['region'], is_secure=False) # Pull data from EC2 results = c.get_metric_statistics( args['period'], args['start_time'], args['end_time'], args['metric'], args['namespace'], args['statistics'], args['dimensions'], args['unit']) # Format/transform results for d in results: # Convert timestamps to datetime objects d.update({u'Timestamp': d[u'Timestamp']}) utc_dt = utc.localize(d[u'Timestamp']) loc_dt = utc_dt.astimezone(current_timezone) d['Timestamp'] = loc_dt try: data_obj = data_map[loc_dt] except KeyError: data_obj = d data_map[loc_dt] = d # If desired, convert Sum to a per-second Rate if args['calc_rate'] == True and 'Sum' in args['statistics']: d.update({u'Rate': d[u'Sum']/args['period']}) # Change key names keys = d.keys() keys.remove('Timestamp') for k in keys: new_k = args['prefix']+k data_obj[new_k] = d[k] del d[k] # rs.extend(results) # Build data description and columns to be return description[args['prefix']+'Samples'] = ('number', args['prefix']+'Samples') if args['unit'] != None: description[args['prefix']+'Unit'] = ('string', args['unit']) for stat in args['statistics']: # If Rate is desired, update label accordingly if stat == 'Sum' and args['calc_rate'] == True: stat = 'Rate' description[args['prefix']+stat] = ('number', args['prefix']+stat) columns.append(args['prefix']+stat) # add a blank data_map[current_timezone.localize(args['end_time'])] = {"Timestamp": current_timezone.localize(args['end_time'])} # Sort data and present data = sorted(data_map.values(), key=operator.itemgetter(u'Timestamp')) data_table = gviz_api.DataTable(description) data_table.LoadData(data) results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id) return results
def setup_arg_parser(): parser = argparse.ArgumentParser(prog='sendyrsspub.py') # # Root level parser # parser.add_argument( '-d', '--database', default=DEFAULTS.get('database', None), help='feed log database location' ) parser.add_argument( '-u', '--sendy-url', default=DEFAULTS.get('sendy_url', None), help='sendy URL' ) parser.add_argument( '-k', '--sendy-api-key', default=DEFAULTS.get('sendy_api_key', None), help='sendy API key' ) subparsers = parser.add_subparsers() # # test_feed command parser # p_test_feed = subparsers.add_parser( 'test_feed', help='output parsed RSS feed data' ) p_test_feed.add_argument( '-f', '--feed-url', help='url of the RSS feed', default=DEFAULTS.get('feed_url', None) ) p_test_feed.add_argument( '-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)' ) p_test_feed.set_defaults(cmd='test_feed') # # Parser for the test_template command # p_test_template = subparsers.add_parser( 'test_template', help='output processed newsletter template' ) p_test_template.add_argument( '-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed' ) p_test_template.add_argument( '-t', '--template', default=DEFAULTS['template'], help='template file name(s) (comma separated)' ) p_test_template.add_argument( '-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)' ) p_test_template.set_defaults(cmd='test_template') # # Parser for the send newsletter command # p_send_newsletter = subparsers.add_parser( 'send_newsletter', help='send a newsletter' ) p_send_newsletter.add_argument( '-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed' ) p_send_newsletter.add_argument( '-t', '--template', default=DEFAULTS.get('template', None), help='template file name(s) (comma separated)' ) p_send_newsletter.add_argument( '-n', '--from-name', default=DEFAULTS['from_name'], help='from name' ) p_send_newsletter.add_argument( '-e', '--from-email', default=DEFAULTS['from_email'], help='from email' ) p_send_newsletter.add_argument( '-r', '--reply-to', default=DEFAULTS['reply_to'], help='reply-to email' ) p_send_newsletter.add_argument( '-s', '--subject', default=DEFAULTS['subject'], help='newsletter subject' ) p_send_newsletter.add_argument( '-l', '--list-ids', default=DEFAULTS['list_ids'], help='list IDs (comma separated)' ) p_send_newsletter.add_argument( '-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)' ) p_send_newsletter.add_argument( '-d', '--disable-log', action='store_true', default=False, help='do not register items as processed in the feed log' ) p_send_newsletter.set_defaults(cmd='send_newsletter') # # Parser for the db_clear command # p_db_clear = subparsers.add_parser( 'db_clear', help='clean all entries in the feed log database' ) p_db_clear.add_argument( '-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed' ) p_db_clear.set_defaults(cmd='db_clear') # # Parser for the db_prune command # p_db_prune = subparsers.add_parser( 'db_prune', help='prune oldest entries from the feed log' ) p_db_prune.add_argument( '-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed' ) p_db_prune.add_argument( '-r', '--remainder', help='number of entries to remain in the feed log (default is 10)' ) p_db_prune.set_defaults(cmd='db_prune') return parser
def setup_arg_parser(): parser = argparse.ArgumentParser(prog='sendyrsspub.py') # # Root level parser # parser.add_argument('-d', '--database', default=DEFAULTS.get('database', None), help='feed log database location') parser.add_argument('-u', '--sendy-url', default=DEFAULTS.get('sendy_url', None), help='sendy URL') parser.add_argument('-k', '--sendy-api-key', default=DEFAULTS.get('sendy_api_key', None), help='sendy API key') subparsers = parser.add_subparsers() # # test_feed command parser # p_test_feed = subparsers.add_parser('test_feed', help='output parsed RSS feed data') p_test_feed.add_argument('-f', '--feed-url', help='url of the RSS feed', default=DEFAULTS.get('feed_url', None)) p_test_feed.add_argument('-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)') p_test_feed.set_defaults(cmd='test_feed') # # Parser for the test_template command # p_test_template = subparsers.add_parser( 'test_template', help='output processed newsletter template') p_test_template.add_argument('-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed') p_test_template.add_argument( '-t', '--template', default=DEFAULTS['template'], help='template file name(s) (comma separated)') p_test_template.add_argument( '-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)') p_test_template.set_defaults(cmd='test_template') # # Parser for the send newsletter command # p_send_newsletter = subparsers.add_parser('send_newsletter', help='send a newsletter') p_send_newsletter.add_argument('-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed') p_send_newsletter.add_argument( '-t', '--template', default=DEFAULTS.get('template', None), help='template file name(s) (comma separated)') p_send_newsletter.add_argument('-n', '--from-name', default=DEFAULTS['from_name'], help='from name') p_send_newsletter.add_argument('-e', '--from-email', default=DEFAULTS['from_email'], help='from email') p_send_newsletter.add_argument('-r', '--reply-to', default=DEFAULTS['reply_to'], help='reply-to email') p_send_newsletter.add_argument('-s', '--subject', default=DEFAULTS['subject'], help='newsletter subject') p_send_newsletter.add_argument('-l', '--list-ids', default=DEFAULTS['list_ids'], help='list IDs (comma separated)') p_send_newsletter.add_argument( '-a', '--all', action='store_true', default=False, help='all feed data (i.e. ignore the feed log)') p_send_newsletter.add_argument( '-d', '--disable-log', action='store_true', default=False, help='do not register items as processed in the feed log') p_send_newsletter.set_defaults(cmd='send_newsletter') # # Parser for the db_clear command # p_db_clear = subparsers.add_parser( 'db_clear', help='clean all entries in the feed log database') p_db_clear.add_argument('-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed') p_db_clear.set_defaults(cmd='db_clear') # # Parser for the db_prune command # p_db_prune = subparsers.add_parser( 'db_prune', help='prune oldest entries from the feed log') p_db_prune.add_argument('-f', '--feed-url', default=DEFAULTS.get('feed_url', None), help='url of the RSS feed') p_db_prune.add_argument( '-r', '--remainder', help='number of entries to remain in the feed log (default is 10)') p_db_prune.set_defaults(cmd='db_prune') return parser
def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None): """ Query CloudWatch and return the results in a Google Visualizations API-friendly format Arguments: `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information) `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter """ # Initialize data description, columns to be returned, and result set description = {"Timestamp": ("datetime", "Timestamp")} columns = ["Timestamp"] rs = [] current_timezone = timezone("UTC") utc = pytz.utc # Build option list opts = [ "unit", "metric", "namespace", "statistics", "period", "dimensions", "prefix", "start_time", "end_time", "calc_rate", "region", "range", "timezone", ] # Set default parameter values from config.py qa = DEFAULTS.copy() # Set passed args for opt in opts: if opt in cloudviz_query: qa[opt] = cloudviz_query[opt] # Convert timestamps to datetimes if necessary for time in ["start_time", "end_time"]: if time in qa: if type(qa[time]) == str or type(qa[time]) == unicode: qa[time] = datetime.strptime(qa[time].split(".")[0], "%Y-%m-%dT%H:%M:%S") # If both start_time and end_time are specified, do nothing. if "start_time" in qa and "end_time" in qa: pass # If only one of the times is specified, fill in the other based on range elif "start_time" in qa and "range" in qa: qa["end_time"] = qa["start_time"] + timedelta(hours=qa["range"]) elif "range" in qa and "end_time" in qa: qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"]) # If neither is specified, use range leading up to current time else: qa["end_time"] = datetime.now() qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"]) if "timezone" in qa: current_timezone = timezone(qa["timezone"]) # Parse, build, and run each CloudWatch query cloudwatch_opts = [ "unit", "metric", "namespace", "statistics", "period", "dimensions", "prefix", "calc_rate", "region", ] for cloudwatch_query in cloudviz_query["cloudwatch_queries"]: args = qa.copy() # Override top-level vars for opt in cloudwatch_opts: if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt] # Calculate time range for period determination/sanity-check delta = args["end_time"] - args["start_time"] delta_seconds = (delta.days * 24 * 60 * 60) + delta.seconds + 1 # round microseconds up # Determine min period as the smallest multiple of 60 that won't result in too many data points min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60) if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0: min_period += 60 if "period" in qa: if args["period"] < min_period: args["period"] = min_period else: args["period"] = min_period # Make sure period isn't smaller than CloudWatch allows if args["period"] < CW_MIN_PERIOD: args["period"] = CW_MIN_PERIOD # Defaulting AWS region to us-east-1 if not "region" in args: args["region"] = "us-east-1" # Use AWS keys if provided, otherwise just let the boto look it up if aws_access_key_id and aws_secret_access_key: c = boto.ec2.cloudwatch.connect_to_region( args["region"], aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, is_secure=False, ) else: c = boto.ec2.cloudwatch.connect_to_region(args["region"], is_secure=False) # Pull data from EC2 results = c.get_metric_statistics( args["period"], args["start_time"], args["end_time"], args["metric"], args["namespace"], args["statistics"], args["dimensions"], args["unit"], ) # Format/transform results for d in results: # Convert timestamps to datetime objects d.update({u"Timestamp": d[u"Timestamp"]}) utc_dt = utc.localize(d[u"Timestamp"]) loc_dt = utc_dt.astimezone(current_timezone) d["Timestamp"] = loc_dt # If desired, convert Sum to a per-second Rate if args["calc_rate"] == True and "Sum" in args["statistics"]: d.update({u"Rate": d[u"Sum"] / args["period"]}) # Change key names keys = d.keys() keys.remove("Timestamp") for k in keys: new_k = args["prefix"] + k d[new_k] = d[k] del d[k] rs.extend(results) # Build data description and columns to be return description[args["prefix"] + "Samples"] = ("number", args["prefix"] + "Samples") description[args["prefix"] + "Unit"] = ("string", args["unit"]) for stat in args["statistics"]: # If Rate is desired, update label accordingly if stat == "Sum" and args["calc_rate"] == True: stat = "Rate" description[args["prefix"] + stat] = ("number", args["prefix"] + stat) columns.append(args["prefix"] + stat) # Sort data and present data = sorted(rs, key=operator.itemgetter(u"Timestamp")) data_table = gviz_api.DataTable(description) data_table.LoadData(data) results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id) return results