from urlparse import urljoin
from uuid import uuid4
from sqlalchemy import create_engine, MetaData
from sqlalchemy.sql import select, func
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json
# Create flask app
app = Flask(__name__)
app.config.from_object('config')
# Connect to AWS
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect('us-east-1') # only supported region!
s3 = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate=False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate=False)
# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser
# Initialize browser id login
browser_id = BrowserID()
# Cron-related constants:
CRON_IDX_MIN = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM = 2
except:
traceback.print_exc()
config = {}
else:
config = {}
if args.from_email:
config["notify_from"] = args.from_email
if args.to_email:
config["notify_to"] = args.to_email
if args.subject:
config["notify_subject"] = args.subject
if args.dry_run:
print "Here is what we would have sent:"
print " From:", config["notify_from"]
print " To:", config["notify_to"]
print "Subject:", config["notify_subject"]
print " Body:", message_body
else:
ses = ses_connect('us-east-1') # only supported region!
ses.send_email(
source = config["notify_from"],
subject = config["notify_subject"],
format = "text",
body = message_body,
to_addresses = config["notify_to"]
)
def main():
parser = argparse.ArgumentParser(description="Check Budget Targets")
parser.add_argument("--day", help="Day to check (YYYYMMDD)", required=True)
parser.add_argument("--targets-file", help="JSON file containing budget targets", type=file, required=True)
parser.add_argument("--data-file", help="JSON file containing observed data", type=file, required=True)
parser.add_argument("--from-email", help="Email 'from:' address", required=True)
parser.add_argument("--to-email", help="Email 'to:' address (multiple allowed)", action="append", required=True)
parser.add_argument("--dry-run", help="Print out what would happen instead of sending alert email", action="store_true")
parser.add_argument("--verbose", help="Print all the messages", action="store_true")
args = parser.parse_args()
target_day = args.day
try:
targets = json.load(args.targets_file)
except Exception as e:
print "Error parsing JSON from {}: {}".format(args.targets_file.name, e)
return 2
try:
data = json.load(args.data_file)
except Exception as e:
print "Error parsing JSON from {}: {}".format(args.data_file.name, e)
return 2
errors = []
exit_code = 0
try:
s = data["submission"]
for c in targets.keys():
if c not in s:
if args.verbose:
print "warning: {} not found in data.".format(c)
continue
if target_day not in s[c]:
if args.verbose:
print "warning: {}/{} not found in data.".format(c, target_day)
continue
scd = s[c][target_day]
clients = targets[c]["clients"]
for docType in targets[c].keys():
if docType == "clients":
continue
else:
if docType not in scd:
if args.verbose:
print "warning: {}/{}/{} not found in data.".format(c, target_day, docType)
continue
scdt = scd[docType]
expected_size = targets[c][docType]["size"] * targets[c][docType]["count"] * clients
actual_size = scdt["size"]
if actual_size > expected_size:
errors.append(fmt_err(c, docType, actual_size, expected_size))
else:
if args.verbose:
print "ok: {}".format(fmt_err(c, docType, actual_size, expected_size))
except Exception as e:
print "Data error: {}".format(e)
exit_code = 3
if len(errors) > 0:
message = "Incoming data for {} exceeded budget targets:\n".format(args.day) + "\n".join(sorted(errors))
subject = "Incoming Telemetry data exceeded budget targets for {}".format(args.day)
if args.dry_run:
print "Dry-run mode. Would have sent:"
print "=============================="
print " From:", args.from_email
print " To:", args.to_email
print "Subject:", subject
print " Body:", message
else:
# ses = ses_connect('us-east-1')
ses = ses_connect('us-west-2')
ses.send_email(
source = args.from_email,
subject = subject,
format = "text",
body = message,
to_addresses = args.to_email
)
elif args.dry_run:
print "Dry-run mode, but would not have sent any alerts."
return exit_code
from flask.ext.login import LoginManager, login_required, current_user
from flask.ext.browserid import BrowserID
from user import User, AnonymousUser
from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from boto.s3 import connect_to_region as s3_connect
from urlparse import urljoin
from uuid import uuid4
# Create flask app
app = Flask(__name__)
app.config.from_object("config")
# Connect to AWS
ec2 = ec2_connect(app.config["AWS_REGION"])
ses = ses_connect("us-east-1") # only supported region!
s3 = s3_connect(app.config["AWS_REGION"])
bucket = s3.get_bucket(app.config["TEMPORARY_BUCKET"], validate=False)
# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser
# Initialize browser id login
browser_id = BrowserID()
def abs_url_for(rule, **options):
return urljoin(request.url_root, url_for(rule, **options))
from dateutil.parser import parse as parse_date
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json
import re
import os.path
# Create flask app
app = Flask(__name__)
app.config.from_object('config')
# Connect to AWS
emr = emr_connect(app.config['AWS_REGION'])
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect(app.config['AWS_REGION'])
s3 = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate = False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate = False)
# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser
# Initialize browser id login
browser_id = BrowserID()
# Cron-related constants:
CRON_IDX_MIN = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM = 2
def main():
parser = argparse.ArgumentParser(description="Check Budget Targets")
parser.add_argument("--day", help="Day to check (YYYYMMDD)", required=True)
parser.add_argument("--targets-file",
help="JSON file containing budget targets",
type=file,
required=True)
parser.add_argument("--data-file",
help="JSON file containing observed data",
type=file,
required=True)
parser.add_argument("--from-email",
help="Email 'from:' address",
required=True)
parser.add_argument("--to-email",
help="Email 'to:' address (multiple allowed)",
action="append",
required=True)
parser.add_argument(
"--dry-run",
help="Print out what would happen instead of sending alert email",
action="store_true")
parser.add_argument("--verbose",
help="Print all the messages",
action="store_true")
args = parser.parse_args()
target_day = args.day
try:
targets = json.load(args.targets_file)
except Exception as e:
print "Error parsing JSON from {}: {}".format(args.targets_file.name,
e)
return 2
try:
data = json.load(args.data_file)
except Exception as e:
print "Error parsing JSON from {}: {}".format(args.data_file.name, e)
return 2
errors = []
exit_code = 0
try:
s = data["submission"]
for c in targets.keys():
if c not in s:
if args.verbose:
print "warning: {} not found in data.".format(c)
continue
if target_day not in s[c]:
if args.verbose:
print "warning: {}/{} not found in data.".format(
c, target_day)
continue
scd = s[c][target_day]
clients = targets[c]["clients"]
for docType in targets[c].keys():
if docType == "clients":
continue
else:
if docType not in scd:
if args.verbose:
print "warning: {}/{}/{} not found in data.".format(
c, target_day, docType)
continue
scdt = scd[docType]
expected_size = targets[c][docType]["size"] * targets[c][
docType]["count"] * clients
actual_size = scdt["size"]
if actual_size > expected_size:
errors.append(
fmt_err(c, docType, actual_size, expected_size))
else:
if args.verbose:
print "ok: {}".format(
fmt_err(c, docType, actual_size,
expected_size))
except Exception as e:
print "Data error: {}".format(e)
exit_code = 3
if len(errors) > 0:
message = "Incoming data for {} exceeded budget targets:\n".format(
args.day) + "\n".join(sorted(errors))
subject = "Incoming Telemetry data exceeded budget targets for {}".format(
args.day)
if args.dry_run:
print "Dry-run mode. Would have sent:"
print "=============================="
print " From:", args.from_email
print " To:", args.to_email
print "Subject:", subject
print " Body:", message
else:
# ses = ses_connect('us-east-1')
ses = ses_connect('us-west-2')
ses.send_email(source=args.from_email,
subject=subject,
format="text",
body=message,
to_addresses=args.to_email)
elif args.dry_run:
print "Dry-run mode, but would not have sent any alerts."
return exit_code
#!/usr/bin/env python
from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from dateutil.parser import parse as parse_date
from datetime import datetime
import config
ec2 = ec2_connect(config.AWS_REGION)
ses = ses_connect(config.AWS_REGION)
def main():
reservations = ec2.get_all_reservations(
filters={'tag:Application': config.INSTANCE_APP_TAG})
for reservation in reservations:
for instance in reservation.instances:
time = datetime.utcnow() - parse_date(instance.launch_time,
ignoretz=True)
if time.days >= 1:
name = instance.tags.get('name', instance.id)
ses.send_email(
source=config.EMAIL_SOURCE,
subject="telemetry-analysis worker instance %s terminated!"
% name,
body=
"We've terminated your instance as it has been running for over 24 hours!",
to_addresses=[instance.tags['Owner']])
instance.terminate()
#!/usr/bin/env python
from boto.ec2 import connect_to_region as ec2_connect
from boto.ses import connect_to_region as ses_connect
from dateutil.parser import parse as parse_date
from datetime import datetime
import config
ec2 = ec2_connect(config.AWS_REGION)
ses = ses_connect(config.AWS_REGION)
def main():
reservations = ec2.get_all_reservations(
filters = {'tag:Application': config.INSTANCE_APP_TAG}
)
for reservation in reservations:
for instance in reservation.instances:
time = datetime.utcnow() - parse_date(instance.launch_time, ignoretz = True)
if time.days >= 1:
name = instance.tags.get('name', instance.id)
ses.send_email(
source = config.EMAIL_SOURCE,
subject = "telemetry-analysis worker instance %s terminated!" % name,
body = "We've terminated your instance as it has been running for over 24 hours!",
to_addresses = [instance.tags['Owner']]
)
instance.terminate()
if __name__ == '__main__':
main()
from dateutil.parser import parse as parse_date
from subprocess import check_output, CalledProcessError
from tempfile import mkstemp
import crontab
import json
import re
import os.path
# Create flask app
app = Flask(__name__)
app.config.from_object('config')
# Connect to AWS
emr = emr_connect(app.config['AWS_REGION'])
ec2 = ec2_connect(app.config['AWS_REGION'])
ses = ses_connect(app.config['AWS_REGION'])
s3 = s3_connect(app.config['AWS_REGION'])
bucket = s3.get_bucket(app.config['TEMPORARY_BUCKET'], validate=False)
code_bucket = s3.get_bucket(app.config['CODE_BUCKET'], validate=False)
# Create login manager
login_manager = LoginManager()
login_manager.anonymous_user = AnonymousUser
# Initialize browser id login
browser_id = BrowserID()
# Cron-related constants:
CRON_IDX_MIN = 0
CRON_IDX_HOUR = 1
CRON_IDX_DOM = 2