Exemple #1
0
def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
    """
    Query CloudWatch and return the results in a Google Visualizations API-friendly format

    Arguments:
    `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
    `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
    """
    # Initialize data description, columns to be returned, and result set
    description = { "Timestamp": ("datetime", "Timestamp")}
    columns = ["Timestamp"]
    rs = []
    current_timezone = timezone('UTC')
    utc = pytz.utc

    # Build option list
    opts = ['unit','metric','namespace','statistics','period', 'dimensions', 'prefix', 
            'start_time', 'end_time', 'calc_rate', 'region', 'range', 'timezone']
    
    # Set default parameter values from config.py
    qa = DEFAULTS.copy()
    
    # Set passed args
    for opt in opts:
        if opt in cloudviz_query: qa[opt] = cloudviz_query[opt]
    
    # Convert timestamps to datetimes if necessary
    for time in ['start_time','end_time']:
        if time in qa:
            if type(qa[time]) == str or type(qa[time]) == unicode: 
                qa[time] = datetime.strptime(qa[time].split(".")[0], '%Y-%m-%dT%H:%M:%S')

    # If both start_time and end_time are specified, do nothing.  
    if 'start_time' in qa and 'end_time' in qa:
        pass
    # If only one of the times is specified, fill in the other based on range
    elif 'start_time' in qa and 'range' in qa:
        qa['end_time'] = qa['start_time'] + timedelta(hours=qa['range'])
    elif 'range' in qa and 'end_time' in qa:
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
    # If neither is specified, use range leading up to current time
    else:
        qa['end_time'] = datetime.utcnow()
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
    
    if 'timezone' in qa:
        current_timezone = timezone(qa['timezone'])

    data_map = {}

    # Parse, build, and run each CloudWatch query
    cloudwatch_opts = ['unit', 'metric', 'namespace', 'statistics', 'period', 'dimensions', 'prefix', 'calc_rate', 'region']
    for cloudwatch_query in cloudviz_query['cloudwatch_queries']:
        args = qa.copy()
        # Override top-level vars
        for opt in cloudwatch_opts:
            if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt]
        
        # Calculate time range for period determination/sanity-check
        delta = args['end_time'] - args['start_time']
        delta_seconds = ( delta.days * 24 * 60 * 60 ) + delta.seconds + 1 #round microseconds up

        # Determine min period as the smallest multiple of 60 that won't result in too many data points
        min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
        if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
            min_period += 60
        
        if 'period' in qa:
            if args['period'] < min_period:
                args['period'] = min_period
        else:
            args['period'] = min_period
        
        # Make sure period isn't smaller than CloudWatch allows
        if args['period'] < CW_MIN_PERIOD: 
            args['period'] = CW_MIN_PERIOD
        
        # Defaulting AWS region to us-east-1
        if not 'region' in args: 
            args['region'] = 'us-east-1'
        
        # Use AWS keys if provided, otherwise just let the boto look it up
        if aws_access_key_id and aws_secret_access_key:
            c = boto.ec2.cloudwatch.connect_to_region(  args['region'], aws_access_key_id=aws_access_key_id,
                                                        aws_secret_access_key=aws_secret_access_key, is_secure=False)
        else:
            c = boto.ec2.cloudwatch.connect_to_region(args['region'], is_secure=False)
        
        # Pull data from EC2
        results = c.get_metric_statistics(  args['period'], args['start_time'], args['end_time'], 
                                            args['metric'], args['namespace'], args['statistics'],
                                            args['dimensions'], args['unit'])

        # Format/transform results
        for d in results:
            # Convert timestamps to datetime objects
            d.update({u'Timestamp': d[u'Timestamp']})
            utc_dt = utc.localize(d[u'Timestamp'])
            loc_dt = utc_dt.astimezone(current_timezone)
            d['Timestamp'] = loc_dt

            try:
                data_obj = data_map[loc_dt]
            except KeyError:
                data_obj = d
                data_map[loc_dt] = d

            # If desired, convert Sum to a per-second Rate
            if args['calc_rate'] == True and 'Sum' in args['statistics']: d.update({u'Rate': d[u'Sum']/args['period']})
            # Change key names
            keys = d.keys()
            keys.remove('Timestamp')
            for k in keys:
                new_k = args['prefix']+k
                data_obj[new_k] = d[k]
                del d[k]
    
        # rs.extend(results)
        
        # Build data description and columns to be return
        description[args['prefix']+'Samples'] = ('number', args['prefix']+'Samples')

        if args['unit'] != None:
            description[args['prefix']+'Unit'] = ('string', args['unit'])

        for stat in args['statistics']:
            # If Rate is desired, update label accordingly
            if stat == 'Sum' and args['calc_rate'] == True:
                stat = 'Rate'
            description[args['prefix']+stat] = ('number', args['prefix']+stat)
            columns.append(args['prefix']+stat)

    # add a blank
    data_map[current_timezone.localize(args['end_time'])] = {"Timestamp": current_timezone.localize(args['end_time'])}
    
    # Sort data and present    
    data = sorted(data_map.values(), key=operator.itemgetter(u'Timestamp'))

    data_table = gviz_api.DataTable(description)
    data_table.LoadData(data)

    results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
    return results
def setup_arg_parser():
    parser = argparse.ArgumentParser(prog='sendyrsspub.py')
    #
    # Root level parser
    #
    parser.add_argument(
        '-d', '--database',
        default=DEFAULTS.get('database', None),
        help='feed log database location'
    )
    parser.add_argument(
        '-u', '--sendy-url',
        default=DEFAULTS.get('sendy_url', None),
        help='sendy URL'
    )
    parser.add_argument(
        '-k', '--sendy-api-key',
        default=DEFAULTS.get('sendy_api_key', None),
        help='sendy API key'
    )
    subparsers = parser.add_subparsers()
    #
    # test_feed command parser
    #
    p_test_feed = subparsers.add_parser(
        'test_feed',
        help='output parsed RSS feed data'
    )
    p_test_feed.add_argument(
        '-f', '--feed-url',
        help='url of the RSS feed',
        default=DEFAULTS.get('feed_url', None)
    )
    p_test_feed.add_argument(
        '-a', '--all',
        action='store_true',
        default=False,
        help='all feed data (i.e. ignore the feed log)'
    )
    p_test_feed.set_defaults(cmd='test_feed')
    #
    # Parser for the test_template command
    #
    p_test_template = subparsers.add_parser(
        'test_template',
        help='output processed newsletter template'
    )
    p_test_template.add_argument(
        '-f', '--feed-url',
        default=DEFAULTS.get('feed_url', None),
        help='url of the RSS feed'
    )
    p_test_template.add_argument(
        '-t', '--template',
        default=DEFAULTS['template'],
        help='template file name(s) (comma separated)'
    )
    p_test_template.add_argument(
        '-a', '--all',
        action='store_true',
        default=False,
        help='all feed data (i.e. ignore the feed log)'
    )
    p_test_template.set_defaults(cmd='test_template')
    #
    # Parser for the send newsletter command
    #
    p_send_newsletter = subparsers.add_parser(
        'send_newsletter',
        help='send a newsletter'
    )
    p_send_newsletter.add_argument(
        '-f', '--feed-url',
        default=DEFAULTS.get('feed_url', None),
        help='url of the RSS feed'
    )
    p_send_newsletter.add_argument(
        '-t', '--template',
        default=DEFAULTS.get('template', None),
        help='template file name(s) (comma separated)'
    )
    p_send_newsletter.add_argument(
        '-n', '--from-name',
        default=DEFAULTS['from_name'],
        help='from name'
    )
    p_send_newsletter.add_argument(
        '-e', '--from-email',
        default=DEFAULTS['from_email'],
        help='from email'
    )
    p_send_newsletter.add_argument(
        '-r', '--reply-to',
        default=DEFAULTS['reply_to'],
        help='reply-to email'
    )
    p_send_newsletter.add_argument(
        '-s', '--subject',
        default=DEFAULTS['subject'],
        help='newsletter subject'
    )
    p_send_newsletter.add_argument(
        '-l', '--list-ids',
        default=DEFAULTS['list_ids'],
        help='list IDs (comma separated)'
    )
    p_send_newsletter.add_argument(
        '-a', '--all',
        action='store_true',
        default=False,
        help='all feed data (i.e. ignore the feed log)'
    )
    p_send_newsletter.add_argument(
        '-d', '--disable-log',
        action='store_true',
        default=False,
        help='do not register items as processed in the feed log'
    )
    p_send_newsletter.set_defaults(cmd='send_newsletter')
    #
    # Parser for the db_clear command
    #
    p_db_clear = subparsers.add_parser(
        'db_clear',
        help='clean all entries in the feed log database'
    )
    p_db_clear.add_argument(
        '-f', '--feed-url',
        default=DEFAULTS.get('feed_url', None),
        help='url of the RSS feed'
    )
    p_db_clear.set_defaults(cmd='db_clear')
    #
    # Parser for the db_prune command
    #
    p_db_prune = subparsers.add_parser(
        'db_prune',
        help='prune oldest entries from the feed log'
    )
    p_db_prune.add_argument(
        '-f', '--feed-url',
        default=DEFAULTS.get('feed_url', None),
        help='url of the RSS feed'
    )
    p_db_prune.add_argument(
        '-r', '--remainder',
        help='number of entries to remain in the feed log (default is 10)'
    )
    p_db_prune.set_defaults(cmd='db_prune')
    return parser
Exemple #3
0
def setup_arg_parser():
    parser = argparse.ArgumentParser(prog='sendyrsspub.py')
    #
    # Root level parser
    #
    parser.add_argument('-d',
                        '--database',
                        default=DEFAULTS.get('database', None),
                        help='feed log database location')
    parser.add_argument('-u',
                        '--sendy-url',
                        default=DEFAULTS.get('sendy_url', None),
                        help='sendy URL')
    parser.add_argument('-k',
                        '--sendy-api-key',
                        default=DEFAULTS.get('sendy_api_key', None),
                        help='sendy API key')
    subparsers = parser.add_subparsers()
    #
    # test_feed command parser
    #
    p_test_feed = subparsers.add_parser('test_feed',
                                        help='output parsed RSS feed data')
    p_test_feed.add_argument('-f',
                             '--feed-url',
                             help='url of the RSS feed',
                             default=DEFAULTS.get('feed_url', None))
    p_test_feed.add_argument('-a',
                             '--all',
                             action='store_true',
                             default=False,
                             help='all feed data (i.e. ignore the feed log)')
    p_test_feed.set_defaults(cmd='test_feed')
    #
    # Parser for the test_template command
    #
    p_test_template = subparsers.add_parser(
        'test_template', help='output processed newsletter template')
    p_test_template.add_argument('-f',
                                 '--feed-url',
                                 default=DEFAULTS.get('feed_url', None),
                                 help='url of the RSS feed')
    p_test_template.add_argument(
        '-t',
        '--template',
        default=DEFAULTS['template'],
        help='template file name(s) (comma separated)')
    p_test_template.add_argument(
        '-a',
        '--all',
        action='store_true',
        default=False,
        help='all feed data (i.e. ignore the feed log)')
    p_test_template.set_defaults(cmd='test_template')
    #
    # Parser for the send newsletter command
    #
    p_send_newsletter = subparsers.add_parser('send_newsletter',
                                              help='send a newsletter')
    p_send_newsletter.add_argument('-f',
                                   '--feed-url',
                                   default=DEFAULTS.get('feed_url', None),
                                   help='url of the RSS feed')
    p_send_newsletter.add_argument(
        '-t',
        '--template',
        default=DEFAULTS.get('template', None),
        help='template file name(s) (comma separated)')
    p_send_newsletter.add_argument('-n',
                                   '--from-name',
                                   default=DEFAULTS['from_name'],
                                   help='from name')
    p_send_newsletter.add_argument('-e',
                                   '--from-email',
                                   default=DEFAULTS['from_email'],
                                   help='from email')
    p_send_newsletter.add_argument('-r',
                                   '--reply-to',
                                   default=DEFAULTS['reply_to'],
                                   help='reply-to email')
    p_send_newsletter.add_argument('-s',
                                   '--subject',
                                   default=DEFAULTS['subject'],
                                   help='newsletter subject')
    p_send_newsletter.add_argument('-l',
                                   '--list-ids',
                                   default=DEFAULTS['list_ids'],
                                   help='list IDs (comma separated)')
    p_send_newsletter.add_argument(
        '-a',
        '--all',
        action='store_true',
        default=False,
        help='all feed data (i.e. ignore the feed log)')
    p_send_newsletter.add_argument(
        '-d',
        '--disable-log',
        action='store_true',
        default=False,
        help='do not register items as processed in the feed log')
    p_send_newsletter.set_defaults(cmd='send_newsletter')
    #
    # Parser for the db_clear command
    #
    p_db_clear = subparsers.add_parser(
        'db_clear', help='clean all entries in the feed log database')
    p_db_clear.add_argument('-f',
                            '--feed-url',
                            default=DEFAULTS.get('feed_url', None),
                            help='url of the RSS feed')
    p_db_clear.set_defaults(cmd='db_clear')
    #
    # Parser for the db_prune command
    #
    p_db_prune = subparsers.add_parser(
        'db_prune', help='prune oldest entries from the feed log')
    p_db_prune.add_argument('-f',
                            '--feed-url',
                            default=DEFAULTS.get('feed_url', None),
                            help='url of the RSS feed')
    p_db_prune.add_argument(
        '-r',
        '--remainder',
        help='number of entries to remain in the feed log (default is 10)')
    p_db_prune.set_defaults(cmd='db_prune')
    return parser
Exemple #4
0
def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
    """
    Query CloudWatch and return the results in a Google Visualizations API-friendly format

    Arguments:
    `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
    `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
    """
    # Initialize data description, columns to be returned, and result set
    description = {"Timestamp": ("datetime", "Timestamp")}
    columns = ["Timestamp"]
    rs = []
    current_timezone = timezone("UTC")
    utc = pytz.utc

    # Build option list
    opts = [
        "unit",
        "metric",
        "namespace",
        "statistics",
        "period",
        "dimensions",
        "prefix",
        "start_time",
        "end_time",
        "calc_rate",
        "region",
        "range",
        "timezone",
    ]

    # Set default parameter values from config.py
    qa = DEFAULTS.copy()

    # Set passed args
    for opt in opts:
        if opt in cloudviz_query:
            qa[opt] = cloudviz_query[opt]

    # Convert timestamps to datetimes if necessary
    for time in ["start_time", "end_time"]:
        if time in qa:
            if type(qa[time]) == str or type(qa[time]) == unicode:
                qa[time] = datetime.strptime(qa[time].split(".")[0], "%Y-%m-%dT%H:%M:%S")

    # If both start_time and end_time are specified, do nothing.
    if "start_time" in qa and "end_time" in qa:
        pass
    # If only one of the times is specified, fill in the other based on range
    elif "start_time" in qa and "range" in qa:
        qa["end_time"] = qa["start_time"] + timedelta(hours=qa["range"])
    elif "range" in qa and "end_time" in qa:
        qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"])
    # If neither is specified, use range leading up to current time
    else:
        qa["end_time"] = datetime.now()
        qa["start_time"] = qa["end_time"] - timedelta(hours=qa["range"])

    if "timezone" in qa:
        current_timezone = timezone(qa["timezone"])

    # Parse, build, and run each CloudWatch query
    cloudwatch_opts = [
        "unit",
        "metric",
        "namespace",
        "statistics",
        "period",
        "dimensions",
        "prefix",
        "calc_rate",
        "region",
    ]
    for cloudwatch_query in cloudviz_query["cloudwatch_queries"]:
        args = qa.copy()
        # Override top-level vars
        for opt in cloudwatch_opts:
            if opt in cloudwatch_query:
                args[opt] = cloudwatch_query[opt]

        # Calculate time range for period determination/sanity-check
        delta = args["end_time"] - args["start_time"]
        delta_seconds = (delta.days * 24 * 60 * 60) + delta.seconds + 1  # round microseconds up

        # Determine min period as the smallest multiple of 60 that won't result in too many data points
        min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
        if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
            min_period += 60

        if "period" in qa:
            if args["period"] < min_period:
                args["period"] = min_period
        else:
            args["period"] = min_period

        # Make sure period isn't smaller than CloudWatch allows
        if args["period"] < CW_MIN_PERIOD:
            args["period"] = CW_MIN_PERIOD

        # Defaulting AWS region to us-east-1
        if not "region" in args:
            args["region"] = "us-east-1"

        # Use AWS keys if provided, otherwise just let the boto look it up
        if aws_access_key_id and aws_secret_access_key:
            c = boto.ec2.cloudwatch.connect_to_region(
                args["region"],
                aws_access_key_id=AWS_ACCESS_KEY_ID,
                aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                is_secure=False,
            )
        else:
            c = boto.ec2.cloudwatch.connect_to_region(args["region"], is_secure=False)

        # Pull data from EC2
        results = c.get_metric_statistics(
            args["period"],
            args["start_time"],
            args["end_time"],
            args["metric"],
            args["namespace"],
            args["statistics"],
            args["dimensions"],
            args["unit"],
        )
        # Format/transform results
        for d in results:
            # Convert timestamps to datetime objects
            d.update({u"Timestamp": d[u"Timestamp"]})
            utc_dt = utc.localize(d[u"Timestamp"])
            loc_dt = utc_dt.astimezone(current_timezone)
            d["Timestamp"] = loc_dt
            # If desired, convert Sum to a per-second Rate
            if args["calc_rate"] == True and "Sum" in args["statistics"]:
                d.update({u"Rate": d[u"Sum"] / args["period"]})
            # Change key names
            keys = d.keys()
            keys.remove("Timestamp")
            for k in keys:
                new_k = args["prefix"] + k
                d[new_k] = d[k]
                del d[k]

        rs.extend(results)

        # Build data description and columns to be return
        description[args["prefix"] + "Samples"] = ("number", args["prefix"] + "Samples")
        description[args["prefix"] + "Unit"] = ("string", args["unit"])
        for stat in args["statistics"]:
            # If Rate is desired, update label accordingly
            if stat == "Sum" and args["calc_rate"] == True:
                stat = "Rate"
            description[args["prefix"] + stat] = ("number", args["prefix"] + stat)
            columns.append(args["prefix"] + stat)

    # Sort data and present
    data = sorted(rs, key=operator.itemgetter(u"Timestamp"))
    data_table = gviz_api.DataTable(description)
    data_table.LoadData(data)

    results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
    return results