def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
"""
Query CloudWatch and return the results in a Google Visualizations API-friendly format
Arguments:
`cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
`request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
"""
# Initialize data description, columns to be returned, and result set
description = { "Timestamp": ("datetime", "Timestamp")}
columns = ["Timestamp"]
rs = []
current_timezone = timezone('UTC')
utc = pytz.utc
# Build option list
opts = ['unit','metric','namespace','statistics','period', 'dimensions', 'prefix',
'start_time', 'end_time', 'calc_rate', 'region', 'range', 'timezone']
# Set default parameter values from config.py
qa = DEFAULTS.copy()
# Set passed args
for opt in opts:
if opt in cloudviz_query: qa[opt] = cloudviz_query[opt]
# Convert timestamps to datetimes if necessary
for time in ['start_time','end_time']:
if time in qa:
if type(qa[time]) == str or type(qa[time]) == unicode:
qa[time] = datetime.strptime(qa[time].split(".")[0], '%Y-%m-%dT%H:%M:%S')
# If both start_time and end_time are specified, do nothing.
if 'start_time' in qa and 'end_time' in qa:
pass
# If only one of the times is specified, fill in the other based on range
elif 'start_time' in qa and 'range' in qa:
qa['end_time'] = qa['start_time'] + timedelta(hours=qa['range'])
elif 'range' in qa and 'end_time' in qa:
qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
# If neither is specified, use range leading up to current time
else:
qa['end_time'] = datetime.utcnow()
qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
if 'timezone' in qa:
current_timezone = timezone(qa['timezone'])
data_map = {}
# Parse, build, and run each CloudWatch query
cloudwatch_opts = ['unit', 'metric', 'namespace', 'statistics', 'period', 'dimensions', 'prefix', 'calc_rate', 'region']
for cloudwatch_query in cloudviz_query['cloudwatch_queries']:
args = qa.copy()
# Override top-level vars
for opt in cloudwatch_opts:
if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt]
# Calculate time range for period determination/sanity-check
delta = args['end_time'] - args['start_time']
delta_seconds = ( delta.days * 24 * 60 * 60 ) + delta.seconds + 1 #round microseconds up
# Determine min period as the smallest multiple of 60 that won't result in too many data points
min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
min_period += 60
if 'period' in qa:
if args['period'] < min_period:
args['period'] = min_period
else:
args['period'] = min_period
# Make sure period isn't smaller than CloudWatch allows
if args['period'] < CW_MIN_PERIOD:
args['period'] = CW_MIN_PERIOD
# Defaulting AWS region to us-east-1
if not 'region' in args:
args['region'] = 'us-east-1'
# Use AWS keys if provided, otherwise just let the boto look it up
if aws_access_key_id and aws_secret_access_key:
c = boto.ec2.cloudwatch.connect_to_region( args['region'], aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key, is_secure=False)
else:
c = boto.ec2.cloudwatch.connect_to_region(args['region'], is_secure=False)
# Pull data from EC2
results = c.get_metric_statistics( args['period'], args['start_time'], args['end_time'],
args['metric'], args['namespace'], args['statistics'],
args['dimensions'], args['unit'])
# Format/transform results
for d in results:
# Convert timestamps to datetime objects
d.update({u'Timestamp': d[u'Timestamp']})
utc_dt = utc.localize(d[u'Timestamp'])
loc_dt = utc_dt.astimezone(current_timezone)
d['Timestamp'] = loc_dt
try:
data_obj = data_map[loc_dt]
except KeyError:
data_obj = d
data_map[loc_dt] = d
# If desired, convert Sum to a per-second Rate
if args['calc_rate'] == True and 'Sum' in args['statistics']: d.update({u'Rate': d[u'Sum']/args['period']})
# Change key names
keys = d.keys()
keys.remove('Timestamp')
for k in keys:
new_k = args['prefix']+k
data_obj[new_k] = d[k]
del d[k]
# rs.extend(results)
# Build data description and columns to be return
description[args['prefix']+'Samples'] = ('number', args['prefix']+'Samples')
if args['unit'] != None:
description[args['prefix']+'Unit'] = ('string', args['unit'])
for stat in args['statistics']:
# If Rate is desired, update label accordingly
if stat == 'Sum' and args['calc_rate'] == True:
stat = 'Rate'
description[args['prefix']+stat] = ('number', args['prefix']+stat)
columns.append(args['prefix']+stat)
# add a blank
data_map[current_timezone.localize(args['end_time'])] = {"Timestamp": current_timezone.localize(args['end_time'])}
# Sort data and present
data = sorted(data_map.values(), key=operator.itemgetter(u'Timestamp'))
data_table = gviz_api.DataTable(description)
data_table.LoadData(data)
results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
return results
def setup_arg_parser():
parser = argparse.ArgumentParser(prog='sendyrsspub.py')
#
# Root level parser
#
parser.add_argument(
'-d', '--database',
default=DEFAULTS.get('database', None),
help='feed log database location'
)
parser.add_argument(
'-u', '--sendy-url',
default=DEFAULTS.get('sendy_url', None),
help='sendy URL'
)
parser.add_argument(
'-k', '--sendy-api-key',
default=DEFAULTS.get('sendy_api_key', None),
help='sendy API key'
)
subparsers = parser.add_subparsers()
#
# test_feed command parser
#
p_test_feed = subparsers.add_parser(
'test_feed',
help='output parsed RSS feed data'
)
p_test_feed.add_argument(
'-f', '--feed-url',
help='url of the RSS feed',
default=DEFAULTS.get('feed_url', None)
)
p_test_feed.add_argument(
'-a', '--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)'
)
p_test_feed.set_defaults(cmd='test_feed')
#
# Parser for the test_template command
#
p_test_template = subparsers.add_parser(
'test_template',
help='output processed newsletter template'
)
p_test_template.add_argument(
'-f', '--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed'
)
p_test_template.add_argument(
'-t', '--template',
default=DEFAULTS['template'],
help='template file name(s) (comma separated)'
)
p_test_template.add_argument(
'-a', '--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)'
)
p_test_template.set_defaults(cmd='test_template')
#
# Parser for the send newsletter command
#
p_send_newsletter = subparsers.add_parser(
'send_newsletter',
help='send a newsletter'
)
p_send_newsletter.add_argument(
'-f', '--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed'
)
p_send_newsletter.add_argument(
'-t', '--template',
default=DEFAULTS.get('template', None),
help='template file name(s) (comma separated)'
)
p_send_newsletter.add_argument(
'-n', '--from-name',
default=DEFAULTS['from_name'],
help='from name'
)
p_send_newsletter.add_argument(
'-e', '--from-email',
default=DEFAULTS['from_email'],
help='from email'
)
p_send_newsletter.add_argument(
'-r', '--reply-to',
default=DEFAULTS['reply_to'],
help='reply-to email'
)
p_send_newsletter.add_argument(
'-s', '--subject',
default=DEFAULTS['subject'],
help='newsletter subject'
)
p_send_newsletter.add_argument(
'-l', '--list-ids',
default=DEFAULTS['list_ids'],
help='list IDs (comma separated)'
)
p_send_newsletter.add_argument(
'-a', '--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)'
)
p_send_newsletter.add_argument(
'-d', '--disable-log',
action='store_true',
default=False,
help='do not register items as processed in the feed log'
)
p_send_newsletter.set_defaults(cmd='send_newsletter')
#
# Parser for the db_clear command
#
p_db_clear = subparsers.add_parser(
'db_clear',
help='clean all entries in the feed log database'
)
p_db_clear.add_argument(
'-f', '--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed'
)
p_db_clear.set_defaults(cmd='db_clear')
#
# Parser for the db_prune command
#
p_db_prune = subparsers.add_parser(
'db_prune',
help='prune oldest entries from the feed log'
)
p_db_prune.add_argument(
'-f', '--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed'
)
p_db_prune.add_argument(
'-r', '--remainder',
help='number of entries to remain in the feed log (default is 10)'
)
p_db_prune.set_defaults(cmd='db_prune')
return parser
def setup_arg_parser():
parser = argparse.ArgumentParser(prog='sendyrsspub.py')
#
# Root level parser
#
parser.add_argument('-d',
'--database',
default=DEFAULTS.get('database', None),
help='feed log database location')
parser.add_argument('-u',
'--sendy-url',
default=DEFAULTS.get('sendy_url', None),
help='sendy URL')
parser.add_argument('-k',
'--sendy-api-key',
default=DEFAULTS.get('sendy_api_key', None),
help='sendy API key')
subparsers = parser.add_subparsers()
#
# test_feed command parser
#
p_test_feed = subparsers.add_parser('test_feed',
help='output parsed RSS feed data')
p_test_feed.add_argument('-f',
'--feed-url',
help='url of the RSS feed',
default=DEFAULTS.get('feed_url', None))
p_test_feed.add_argument('-a',
'--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)')
p_test_feed.set_defaults(cmd='test_feed')
#
# Parser for the test_template command
#
p_test_template = subparsers.add_parser(
'test_template', help='output processed newsletter template')
p_test_template.add_argument('-f',
'--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed')
p_test_template.add_argument(
'-t',
'--template',
default=DEFAULTS['template'],
help='template file name(s) (comma separated)')
p_test_template.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)')
p_test_template.set_defaults(cmd='test_template')
#
# Parser for the send newsletter command
#
p_send_newsletter = subparsers.add_parser('send_newsletter',
help='send a newsletter')
p_send_newsletter.add_argument('-f',
'--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed')
p_send_newsletter.add_argument(
'-t',
'--template',
default=DEFAULTS.get('template', None),
help='template file name(s) (comma separated)')
p_send_newsletter.add_argument('-n',
'--from-name',
default=DEFAULTS['from_name'],
help='from name')
p_send_newsletter.add_argument('-e',
'--from-email',
default=DEFAULTS['from_email'],
help='from email')
p_send_newsletter.add_argument('-r',
'--reply-to',
default=DEFAULTS['reply_to'],
help='reply-to email')
p_send_newsletter.add_argument('-s',
'--subject',
default=DEFAULTS['subject'],
help='newsletter subject')
p_send_newsletter.add_argument('-l',
'--list-ids',
default=DEFAULTS['list_ids'],
help='list IDs (comma separated)')
p_send_newsletter.add_argument(
'-a',
'--all',
action='store_true',
default=False,
help='all feed data (i.e. ignore the feed log)')
p_send_newsletter.add_argument(
'-d',
'--disable-log',
action='store_true',
default=False,
help='do not register items as processed in the feed log')
p_send_newsletter.set_defaults(cmd='send_newsletter')
#
# Parser for the db_clear command
#
p_db_clear = subparsers.add_parser(
'db_clear', help='clean all entries in the feed log database')
p_db_clear.add_argument('-f',
'--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed')
p_db_clear.set_defaults(cmd='db_clear')
#
# Parser for the db_prune command
#
p_db_prune = subparsers.add_parser(
'db_prune', help='prune oldest entries from the feed log')
p_db_prune.add_argument('-f',
'--feed-url',
default=DEFAULTS.get('feed_url', None),
help='url of the RSS feed')
p_db_prune.add_argument(
'-r',
'--remainder',
help='number of entries to remain in the feed log (default is 10)')
p_db_prune.set_defaults(cmd='db_prune')
return parser
def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
"""
Query CloudWatch and return the results in a Google Visualizations API-friendly format
Arguments:
`cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
`request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
"""
# Initialize data description, columns to be returned, and result set
description = {"Timestamp": ("datetime", "Timestamp")}
columns = ["Timestamp"]
rs = []
current_timezone = timezone("UTC")
utc = pytz.utc
# Build option list
opts = [
"unit",
"metric",
"namespace",
"statistics",
"period",
"dimensions",
"prefix",
"start_time",
"end_time",
"calc_rate",
"region",
"range",
"timezone",
]
# Set default parameter values from config.py
qa = DEFAULTS.copy()
# Set passed args
for opt in opts:
if opt in cloudviz_query:
qa[opt] = cloudviz_query[opt]
# Convert timestamps to datetimes if necessary
for time in ["start_time", "end_time"]:
if time in qa:
if type(qa[time]) == str or type(qa[time]) == unicode:
qa[time] = datetime.strptime(qa[time].split(".")[0], "%Y-%m-%dT%H:%M:%S")
# If both start_time and end_time are specified, do nothing.
if "start_time" in qa and "end_time" in qa:
pass
# If only one of the times is specified, fill in the other based on range
elif "start_time" in qa and "range" in qa:
qa["end_time"] = qa["start_time"] + timedelta(hours=qa["range"])
elif "range" in qa and "end_time" in qa:
qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"])
# If neither is specified, use range leading up to current time
else:
qa["end_time"] = datetime.now()
qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"])
if "timezone" in qa:
current_timezone = timezone(qa["timezone"])
# Parse, build, and run each CloudWatch query
cloudwatch_opts = [
"unit",
"metric",
"namespace",
"statistics",
"period",
"dimensions",
"prefix",
"calc_rate",
"region",
]
for cloudwatch_query in cloudviz_query["cloudwatch_queries"]:
args = qa.copy()
# Override top-level vars
for opt in cloudwatch_opts:
if opt in cloudwatch_query:
args[opt] = cloudwatch_query[opt]
# Calculate time range for period determination/sanity-check
delta = args["end_time"] - args["start_time"]
delta_seconds = (delta.days * 24 * 60 * 60) + delta.seconds + 1 # round microseconds up
# Determine min period as the smallest multiple of 60 that won't result in too many data points
min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
min_period += 60
if "period" in qa:
if args["period"] < min_period:
args["period"] = min_period
else:
args["period"] = min_period
# Make sure period isn't smaller than CloudWatch allows
if args["period"] < CW_MIN_PERIOD:
args["period"] = CW_MIN_PERIOD
# Defaulting AWS region to us-east-1
if not "region" in args:
args["region"] = "us-east-1"
# Use AWS keys if provided, otherwise just let the boto look it up
if aws_access_key_id and aws_secret_access_key:
c = boto.ec2.cloudwatch.connect_to_region(
args["region"],
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
is_secure=False,
)
else:
c = boto.ec2.cloudwatch.connect_to_region(args["region"], is_secure=False)
# Pull data from EC2
results = c.get_metric_statistics(
args["period"],
args["start_time"],
args["end_time"],
args["metric"],
args["namespace"],
args["statistics"],
args["dimensions"],
args["unit"],
)
# Format/transform results
for d in results:
# Convert timestamps to datetime objects
d.update({u"Timestamp": d[u"Timestamp"]})
utc_dt = utc.localize(d[u"Timestamp"])
loc_dt = utc_dt.astimezone(current_timezone)
d["Timestamp"] = loc_dt
# If desired, convert Sum to a per-second Rate
if args["calc_rate"] == True and "Sum" in args["statistics"]:
d.update({u"Rate": d[u"Sum"] / args["period"]})
# Change key names
keys = d.keys()
keys.remove("Timestamp")
for k in keys:
new_k = args["prefix"] + k
d[new_k] = d[k]
del d[k]
rs.extend(results)
# Build data description and columns to be return
description[args["prefix"] + "Samples"] = ("number", args["prefix"] + "Samples")
description[args["prefix"] + "Unit"] = ("string", args["unit"])
for stat in args["statistics"]:
# If Rate is desired, update label accordingly
if stat == "Sum" and args["calc_rate"] == True:
stat = "Rate"
description[args["prefix"] + stat] = ("number", args["prefix"] + stat)
columns.append(args["prefix"] + stat)
# Sort data and present
data = sorted(rs, key=operator.itemgetter(u"Timestamp"))
data_table = gviz_api.DataTable(description)
data_table.LoadData(data)
results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
return results
