Esempio n. 1
0
from urlparse import urljoin
from uuid import uuid4
from sqlalchemy import create_engine, MetaData
from sqlalchemy.sql import select, func
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json

# Create flask app
app = Flask(__name__)
app.config.from_object('config')

# Connect to AWS
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect('us-east-1')  # only supported region!
s3 = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate=False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate=False)

# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser

# Initialize browser id login
browser_id = BrowserID()

# Cron-related constants:
CRON_IDX_MIN = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM = 2
Esempio n. 2
0
            except:
                traceback.print_exc()
                config = {}
        else:
            config = {}

        if args.from_email:
            config["notify_from"] = args.from_email

        if args.to_email:
            config["notify_to"] = args.to_email

        if args.subject:
            config["notify_subject"] = args.subject

        if args.dry_run:
            print "Here is what we would have sent:"
            print "   From:", config["notify_from"]
            print "     To:", config["notify_to"]
            print "Subject:", config["notify_subject"]
            print "   Body:", message_body
        else:
            ses = ses_connect('us-east-1') # only supported region!
            ses.send_email(
                source          = config["notify_from"],
                subject         = config["notify_subject"],
                format          = "text",
                body            = message_body,
                to_addresses    = config["notify_to"]
            )
Esempio n. 3
0
def main():
    parser = argparse.ArgumentParser(description="Check Budget Targets")
    parser.add_argument("--day", help="Day to check (YYYYMMDD)", required=True)
    parser.add_argument("--targets-file", help="JSON file containing budget targets", type=file, required=True)
    parser.add_argument("--data-file", help="JSON file containing observed data", type=file, required=True)
    parser.add_argument("--from-email", help="Email 'from:' address", required=True)
    parser.add_argument("--to-email", help="Email 'to:' address (multiple allowed)", action="append", required=True)
    parser.add_argument("--dry-run", help="Print out what would happen instead of sending alert email", action="store_true")
    parser.add_argument("--verbose", help="Print all the messages", action="store_true")
    args = parser.parse_args()

    target_day = args.day
    try:
        targets = json.load(args.targets_file)
    except Exception as e:
        print "Error parsing JSON from {}: {}".format(args.targets_file.name, e)
        return 2

    try:
        data = json.load(args.data_file)
    except Exception as e:
        print "Error parsing JSON from {}: {}".format(args.data_file.name, e)
        return 2

    errors = []
    exit_code = 0
    try:
        s = data["submission"]
        for c in targets.keys():
            if c not in s:
                if args.verbose:
                    print "warning: {} not found in data.".format(c)
                continue
            if target_day not in s[c]:
                if args.verbose:
                    print "warning: {}/{} not found in data.".format(c, target_day)
                continue

            scd = s[c][target_day]
            clients = targets[c]["clients"]
            for docType in targets[c].keys():
                if docType == "clients":
                    continue
                else:
                    if docType not in scd:
                        if args.verbose:
                            print "warning: {}/{}/{} not found in data.".format(c, target_day, docType)
                        continue
                    scdt = scd[docType]
                    expected_size = targets[c][docType]["size"] * targets[c][docType]["count"] * clients
                    actual_size = scdt["size"]
                    if actual_size > expected_size:
                        errors.append(fmt_err(c, docType, actual_size, expected_size))
                    else:
                        if args.verbose:
                            print "ok: {}".format(fmt_err(c, docType, actual_size, expected_size))
    except Exception as e:
        print "Data error: {}".format(e)
        exit_code = 3

    if len(errors) > 0:
        message = "Incoming data for {} exceeded budget targets:\n".format(args.day) + "\n".join(sorted(errors))
        subject = "Incoming Telemetry data exceeded budget targets for {}".format(args.day)
        if args.dry_run:
            print "Dry-run mode. Would have sent:"
            print "=============================="
            print "   From:", args.from_email
            print "     To:", args.to_email
            print "Subject:", subject
            print "   Body:", message
        else:
            # ses = ses_connect('us-east-1')
            ses = ses_connect('us-west-2')
            ses.send_email(
                source       = args.from_email,
                subject      = subject,
                format       = "text",
                body         = message,
                to_addresses = args.to_email
            )
    elif args.dry_run:
        print "Dry-run mode, but would not have sent any alerts."

    return exit_code
Esempio n. 4
0
from flask.ext.login import LoginManager, login_required, current_user
from flask.ext.browserid import BrowserID
from user import User, AnonymousUser
from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from boto.s3 import connect_to_region as s3_connect
from urlparse import urljoin
from uuid import uuid4

# Create flask app
app = Flask(__name__)
app.config.from_object("config")

# Connect to AWS
ec2 = ec2_connect(app.config["AWS_REGION"])
ses = ses_connect("us-east-1")  # only supported region!
s3 = s3_connect(app.config["AWS_REGION"])
bucket = s3.get_bucket(app.config["TEMPORARY_BUCKET"], validate=False)

# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser

# Initialize browser id login
browser_id = BrowserID()


def abs_url_for(rule, **options):
    return urljoin(request.url_root, url_for(rule, **options))

Esempio n. 5
0
from dateutil.parser import parse as parse_date
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json
import re
import os.path

# Create flask app
app = Flask(__name__)
app.config.from_object('config')

# Connect to AWS
emr  = emr_connect(app.config['AWS_REGION'])
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect(app.config['AWS_REGION'])
s3  = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate = False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate = False)

# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser

# Initialize browser id login
browser_id = BrowserID()

# Cron-related constants:
CRON_IDX_MIN  = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM  = 2
def main():
    parser = argparse.ArgumentParser(description="Check Budget Targets")
    parser.add_argument("--day", help="Day to check (YYYYMMDD)", required=True)
    parser.add_argument("--targets-file",
                        help="JSON file containing budget targets",
                        type=file,
                        required=True)
    parser.add_argument("--data-file",
                        help="JSON file containing observed data",
                        type=file,
                        required=True)
    parser.add_argument("--from-email",
                        help="Email 'from:' address",
                        required=True)
    parser.add_argument("--to-email",
                        help="Email 'to:' address (multiple allowed)",
                        action="append",
                        required=True)
    parser.add_argument(
        "--dry-run",
        help="Print out what would happen instead of sending alert email",
        action="store_true")
    parser.add_argument("--verbose",
                        help="Print all the messages",
                        action="store_true")
    args = parser.parse_args()

    target_day = args.day
    try:
        targets = json.load(args.targets_file)
    except Exception as e:
        print "Error parsing JSON from {}: {}".format(args.targets_file.name,
                                                      e)
        return 2

    try:
        data = json.load(args.data_file)
    except Exception as e:
        print "Error parsing JSON from {}: {}".format(args.data_file.name, e)
        return 2

    errors = []
    exit_code = 0
    try:
        s = data["submission"]
        for c in targets.keys():
            if c not in s:
                if args.verbose:
                    print "warning: {} not found in data.".format(c)
                continue
            if target_day not in s[c]:
                if args.verbose:
                    print "warning: {}/{} not found in data.".format(
                        c, target_day)
                continue

            scd = s[c][target_day]
            clients = targets[c]["clients"]
            for docType in targets[c].keys():
                if docType == "clients":
                    continue
                else:
                    if docType not in scd:
                        if args.verbose:
                            print "warning: {}/{}/{} not found in data.".format(
                                c, target_day, docType)
                        continue
                    scdt = scd[docType]
                    expected_size = targets[c][docType]["size"] * targets[c][
                        docType]["count"] * clients
                    actual_size = scdt["size"]
                    if actual_size > expected_size:
                        errors.append(
                            fmt_err(c, docType, actual_size, expected_size))
                    else:
                        if args.verbose:
                            print "ok: {}".format(
                                fmt_err(c, docType, actual_size,
                                        expected_size))
    except Exception as e:
        print "Data error: {}".format(e)
        exit_code = 3

    if len(errors) > 0:
        message = "Incoming data for {} exceeded budget targets:\n".format(
            args.day) + "\n".join(sorted(errors))
        subject = "Incoming Telemetry data exceeded budget targets for {}".format(
            args.day)
        if args.dry_run:
            print "Dry-run mode. Would have sent:"
            print "=============================="
            print "   From:", args.from_email
            print "     To:", args.to_email
            print "Subject:", subject
            print "   Body:", message
        else:
            # ses = ses_connect('us-east-1')
            ses = ses_connect('us-west-2')
            ses.send_email(source=args.from_email,
                           subject=subject,
                           format="text",
                           body=message,
                           to_addresses=args.to_email)
    elif args.dry_run:
        print "Dry-run mode, but would not have sent any alerts."

    return exit_code
#!/usr/bin/env python

from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from dateutil.parser import parse as parse_date
from datetime import datetime
import config

ec2 = ec2_connect(config.AWS_REGION)
ses = ses_connect(config.AWS_REGION)


def main():
    reservations = ec2.get_all_reservations(
        filters={'tag:Application': config.INSTANCE_APP_TAG})
    for reservation in reservations:
        for instance in reservation.instances:
            time = datetime.utcnow() - parse_date(instance.launch_time,
                                                  ignoretz=True)
            if time.days >= 1:
                name = instance.tags.get('name', instance.id)
                ses.send_email(
                    source=config.EMAIL_SOURCE,
                    subject="telemetry-analysis worker instance %s terminated!"
                    % name,
                    body=
                    "We've terminated your instance as it has been running for over 24 hours!",
                    to_addresses=[instance.tags['Owner']])
                instance.terminate()

#!/usr/bin/env python

from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from dateutil.parser import parse as parse_date
from datetime import datetime
import config

ec2 = ec2_connect(config.AWS_REGION)
ses = ses_connect(config.AWS_REGION)

def main():
    reservations = ec2.get_all_reservations(
        filters = {'tag:Application':  config.INSTANCE_APP_TAG}
    )
    for reservation in reservations:
        for instance in reservation.instances:
            time = datetime.utcnow() - parse_date(instance.launch_time, ignoretz = True)
            if time.days >= 1:
                name = instance.tags.get('name', instance.id)
                ses.send_email(
                    source          = config.EMAIL_SOURCE,
                    subject         = "telemetry-analysis worker instance %s terminated!" % name,
                    body            = "We've terminated your instance as it has been running for over 24 hours!",
                    to_addresses    = [instance.tags['Owner']]
                )
                instance.terminate()

if __name__ == '__main__':
    main()
Esempio n. 9
0
from dateutil.parser import parse as parse_date
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json
import re
import os.path

# Create flask app
app = Flask(__name__)
app.config.from_object('config')

# Connect to AWS
emr = emr_connect(app.config['AWS_REGION'])
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect(app.config['AWS_REGION'])
s3 = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate=False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate=False)

# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser

# Initialize browser id login
browser_id = BrowserID()

# Cron-related constants:
CRON_IDX_MIN = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM = 2