def get_rds_instances(self, id=None):
"""Retrieve rds instance(s)."""
if isinstance(id,str) or isinstance(id,unicode):
id = [id]
self.log.info('Geting rds instances (id=%s)..' % id)
conn = boto.connect_rds(self.key, self.secret)
return conn.get_all_dbinstances(id)
def cleanup(label, pgroups):
"""Deletes RDS instances, waiting until they don't exist or timeout
Args:
label Label searched for in RDS instance IDs
pgroups list of parameter group names
"""
rds = boto.connect_rds()
loop = 0
label_rs = True
while loop < 10 and label_rs:
rs = rds.get_all_dbinstances()
label_rs = [d for d in rs if label in d.id]
for inst in label_rs:
if inst.status in ['available', 'failed', 'storage-full',
'incompatible-option-group',
'incompatible-parameters',
'incompatible-restore',
'incompatible-network']:
logging.info("Deleting RDS instance {}".format(inst.id))
rds.delete_dbinstance(inst.id, skip_final_snapshot=True)
if label_rs:
time.sleep(60)
loop += 1
if loop == 10 and rs:
logging.error("Problem deleting RDS instances: timed out")
else:
for pg in pgroups:
rds.delete_parameter_group(pg)
def get_rds_info(indentifier=None):
"""Function for fetching RDS details"""
regions_list = [None]
if all_regions:
regions_list = [ region.name for region in boto.rds.regions() ]
info = []
# Search in all regions, or the default region if all_regions is None
for region in regions_list:
if not region:
rds = boto.connect_rds()
else:
rds = boto.rds.connect_to_region(region)
try:
info.extend(rds.get_all_dbinstances(indentifier))
except boto.exception.BotoServerError:
pass
if indentifier:
if len(info)>0:
return info[0]
return None
return info
def main(title=None):
'''
'''
if not title:
title = u'free storage capacity'
conn = boto.connect_rds()
return monitor(conn, title)
def load_db(number, label, param_grp):
"""Loads a specific database with test data
Args:
number index number of RDS instance
label prefix to apply to name
param_grp param group to use
"""
rds = boto.connect_rds()
logging.info("Loading data into database: {} {} {}\n".format(label, number,
param_grp))
inst = rds.get_all_dbinstances(instance_id='{}-{}-{}'.format(label, number,
param_grp))
db = connect(hostname=inst[0].endpoint[0])
c = db.cursor()
sql = load_data_sql()
try:
for s in sql:
c.execute(s, None)
except:
logging.error("load data error on {} {} {}\n".format(
'{}-{}-{}'.format(label, number, param_grp),
db.errno(), db.error()))
finally:
c.close()
db.close()
def launch_rds_instance(self,
id,
dbname,
allocated_storage,
instance_class,
availability_zone,
multi_az=False,
timeout=0):
"""Launch an rds instance. If timeout is > 0, then this method
won't return until the rds instance is fully running or the timeout
duration expires."""
# TODO: support multiple regions (and multiple security groups)
# import logging
# boto.set_file_logger('rds', '/tmp/boto.log', level=logging.DEBUG)
# conn = boto.connect_rds(self.key, self.secret, debug=2)
conn = boto.connect_rds(self.key, self.secret)
instance = conn.create_dbinstance(id,
allocated_storage,
instance_class,
self.username,
self.password,
db_name=dbname,
availability_zone=availability_zone,
multi_az=multi_az)
self.log.info('Launched rds instance %s' % instance.id)
if timeout:
time.sleep(5.0) # avoids intermittent error
self.wait_until_endpoint(instance, timeout)
return instance
def get_rds_instances(self, id=None):
"""Retrieve rds instance(s)."""
if isinstance(id, str) or isinstance(id, unicode):
id = [id]
self.log.info('Geting rds instances (id=%s)..' % id)
conn = boto.connect_rds(self.key, self.secret)
return conn.get_all_dbinstances(id)
def perform_rds_test(number, label, param_grp, profiles):
"""Performs a test against a specific RDS instance and logs the elapsed time
Args:
number index number of RDS instance
label prefix to apply to name
param_grp param group to use
profiles list of profile types
"""
rds = boto.connect_rds()
logging.info("Performing test on database: {} {} {}\n".format(label, number,
param_grp))
inst = rds.get_all_dbinstances(instance_id='{}-{}-{}'.format(label, number,
param_grp))
try:
db = connect(hostname=inst[0].endpoint[0])
perform_test(db, profiles, inst[0].id)
except Exception as e:
logging.error("test error on {} {} {}\n".format(
'{}-{}-{}'.format(label, number, param_grp),
db.errno(), db.error()))
raise
finally:
db.close()
def __init__(self, *args, **kwargs):
self.connection = boto.connect_rds()
self.storage = kwargs.get('storage', None)
if self.storage is None:
raise RDSException("storage must be set")
self.size = kwargs.get('size', None)
if self.size is None:
raise RDSException("size must be set")
super(AWSRDS, self).__init__(*args, **kwargs)
def update_elb_rds_dns(zone):
"""
Creates elb and rds CNAME records
in a zone for args.stack_name.
Uses the tags of the instances attached
to the ELBs to create the dns name
"""
elb_con = boto.connect_elb()
ec2_con = boto.connect_ec2()
rds_con = boto.connect_rds()
vpc_id = vpc_for_stack_name(args.stack_name)
if not zone and args.noop:
# use a placeholder for zone name
# if it doesn't exist
zone_name = "<zone name>"
else:
zone_name = zone.Name[:-1]
stack_rdss = [rds for rds in rds_con.get_all_dbinstances()
if hasattr(rds.subnet_group, 'vpc_id') and
rds.subnet_group.vpc_id == vpc_id]
for rds in stack_rdss:
fqdn = "{}.{}".format('rds', zone_name)
add_or_update_record(zone, fqdn, 'CNAME', 600,
[stack_rdss[0].endpoint[0]])
stack_elbs = [elb for elb in elb_con.get_all_load_balancers()
if elb.vpc_id == vpc_id]
for elb in stack_elbs:
for inst in elb.instances:
instance = ec2_con.get_all_instances(
instance_ids=[inst.id])[0].instances[0]
try:
env_tag = instance.tags['environment']
if 'play' in instance.tags:
play_tag = instance.tags['play']
else:
# deprecated, for backwards compatibility
play_tag = instance.tags['role']
play_tag = instance.tags['role']
fqdn = "{}-{}.{}".format(env_tag, play_tag, zone_name)
add_or_update_record(zone, fqdn, 'CNAME', 600, [elb.dns_name])
if play_tag == 'edxapp':
# create courses and studio CNAME records for edxapp
for name in ['courses', 'studio']:
fqdn = "{}.{}".format(name, zone_name)
add_or_update_record(zone, fqdn, 'CNAME',
600, [elb.dns_name])
break # only need the first instance for tag info
except KeyError:
print("Instance {}, attached to elb {} does not "
"have tags for environment and play".format(elb, inst))
raise
def main(argv):
arg_Vpcid = ''
arg_Regions = ['us-east-1', 'us-west-1', 'us-west-2']
ww_Regions = ['eu-west-1', 'sa-east-1', 'ap-northeast-1', 'ap-southeast-1', 'ap-southeast-2']
arg_Flags = []
arg_sortBy = 'instance'
SingleInstances = []
try:
opts, args = getopt.getopt(argv, "hv:r:s:Rj",
['help', 'vpcid=', 'region=', 'all-regions', 'sort=', 'json', 'raw'])
except getopt.GetoptError:
print USAGE
sys.exit(2)
for opt, arg in opts:
if opt in ("-h", "--help", "?", "help"):
print USAGE
sys.exit()
elif opt in ("-v", "--vpcid"):
arg_Vpcid = arg
elif opt in ('-f', '--region'):
arg_Regions = [arg]
elif opt in ('-s', '--sort'):
if arg in ['zone', 'size']:
arg_sortBy = arg
elif opt == '--all-regions':
arg_Regions += ww_Regions
elif opt in ('-j', '--json'):
arg_Flags.append('json')
elif opt == '--raw':
arg_Flags.append('raw')
for a in args:
SingleInstances.append(a)
Rds = boto.connect_rds()
RDSList = RdsList(Rds, arg_Regions, arg_Vpcid)
if SingleInstances:
for s_inst in SingleInstances:
printRdsInstance(RDSList, s_inst)
elif arg_Vpcid:
for _r in RDSList:
_r.printInfo()
elif 'raw' in arg_Flags:
for r in RDSList:
print r
elif 'json' in arg_Flags and not 'raw' in arg_Flags:
for r in RDSList:
pprint(r.json())
else:
# default
printRdsInfo(RDSList, arg_sortBy)
def modify_rds_instance(self, id, allocated_storage=None,
instance_class=None, multi_az=None,
apply_immediately=False):
"""Modify an rds instance."""
conn = boto.connect_rds(self.key, self.secret)
conn.modify_dbinstance(id,
allocated_storage = allocated_storage,
instance_class = instance_class,
multi_az = multi_az,
apply_immediately = apply_immediately)
self.log.info('Modified rds instance %s' % id)
def waitForDatabaseBackupToComplete():
rds = boto.connect_rds()
still_backing_up = True
while still_backing_up:
still_backing_up = False
snapshots = rds.get_all_dbsnapshots()
for x in snapshots:
if x.status != "available":
still_backing_up = True
time.sleep(30)
print ".. finished waiting .."
def waitForDatabaseBackupToComplete():
rds = boto.connect_rds()
still_backing_up = True
while still_backing_up:
still_backing_up = False
snapshots = rds.get_all_dbsnapshots()
for x in snapshots:
if x.status != "available":
still_backing_up = True
time.sleep(30)
print ".. finished waiting .."
def update_elb_rds_dns(zone):
"""
Creates elb and rds CNAME records
in a zone for args.stack_name.
Uses the tags of the instances attached
to the ELBs to create the dns name
"""
dns_records = set()
elb_con = boto.connect_elb()
rds_con = boto.connect_rds()
vpc_id = vpc_for_stack_name(args.stack_name)
if not zone and args.noop:
# use a placeholder for zone name
# if it doesn't exist
zone_name = "<zone name>"
else:
zone_name = zone.Name[:-1]
stack_elbs = [elb for elb in elb_con.get_all_load_balancers()
if elb.vpc_id == vpc_id]
for elb in stack_elbs:
if "RabbitMQ" in elb.source_security_group.name or "ElasticSearch" in elb.source_security_group.name:
env_tag,deployment,play_tag = get_security_group_dns(elb.source_security_group.name)
fqdn = "{}-{}.{}".format(env_tag, play_tag, zone_name)
dns_records.add(DNSRecord(zone,fqdn,'CNAME',600,[elb.dns_name]))
else:
env_tag,play_tag = get_dns_from_instances(elb)
fqdn = "{}-{}.{}".format(env_tag, play_tag, zone_name)
dns_records.add(DNSRecord(zone,fqdn,'CNAME',600,[elb.dns_name]))
if extra_play_dns.has_key(play_tag):
for name in extra_play_dns.get(play_tag):
fqdn = "{}-{}.{}".format(env_tag, name, zone_name)
dns_records.add(DNSRecord(zone,fqdn,'CNAME',600,[elb.dns_name]))
stack_rdss = [rds for rds in rds_con.get_all_dbinstances()
if hasattr(rds.subnet_group, 'vpc_id') and
rds.subnet_group.vpc_id == vpc_id]
# TODO the current version of the RDS API doesn't support
# looking up RDS instance tags. Hence, we are using the
# env_tag that was set via the loop over instances above.
for rds in stack_rdss:
fqdn = "{}-{}.{}".format(env_tag,'rds', zone_name)
dns_records.add(DNSRecord(zone,fqdn,'CNAME',600,[stack_rdss[0].endpoint[0]]))
add_or_update_record(dns_records)
def db_status():
"""Returns list of current RDS instances"""
rds = boto.connect_rds()
rs = rds.get_all_dbinstances()
if rs:
for inst in rs:
logging.debug('RDS instance {}, status: {}, endpoint: {}'.format(
inst.id, inst.status, inst.endpoint))
else:
logging.debug('No RDS instances')
return rs
def get_rds_info(indentifier=None):
"""Function for fetching RDS details"""
rds = boto.connect_rds()
try:
if indentifier:
info = rds.get_all_dbinstances(indentifier)[0]
else:
info = rds.get_all_dbinstances()
except boto.exception.BotoServerError:
info = None
return info
def backupDatabase():
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d-%H-%M-%S')
snapshot_id = "mhfowler-" + str(st)
snapshot_id = snapshot_id.replace(" ", "")
rds = boto.connect_rds()
instances = rds.get_all_dbinstances()
db = instances[0]
db.snapshot(snapshot_id)
print("backing up database...")
waitForDatabaseBackupToComplete()
print("database back up successful.")
def backupDatabase():
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d-%H-%M-%S')
snapshot_id = "mhfowler-" + str(st)
snapshot_id = snapshot_id.replace(" ","")
rds = boto.connect_rds()
instances = rds.get_all_dbinstances()
db = instances[0]
db.snapshot(snapshot_id)
print("backing up database...")
waitForDatabaseBackupToComplete()
print("database back up successful.")
def get_rds_info(indentifier=None):
"""Function for fetching RDS details"""
rds = boto.connect_rds()
try:
if indentifier:
info = rds.get_all_dbinstances(indentifier)[0]
else:
info = rds.get_all_dbinstances()
except boto.exception.BotoServerError:
info = None
if not info:
print 'Unable to get RDS details'
sys.exit(1)
return info
def modify_rds_instance(self,
id,
allocated_storage=None,
instance_class=None,
multi_az=None,
apply_immediately=False):
"""Modify an rds instance."""
conn = boto.connect_rds(self.key, self.secret)
conn.modify_dbinstance(id,
allocated_storage=allocated_storage,
instance_class=instance_class,
multi_az=multi_az,
apply_immediately=apply_immediately)
self.log.info('Modified rds instance %s' % id)
def run(self):
end = datetime.datetime.utcnow()
start = end - datetime.timedelta(minutes=5)
try:
self.cw = boto.connect_cloudwatch(self.identity, self.secret, validate_certs=False)
self.rds = None
regions = boto.rds.regions()
for r in regions:
if r.name == self.region:
self.rds = boto.connect_rds(self.identity, self.secret, region=r, validate_certs=False)
except Exception, e:
print "Boto Error: %s" % (e,)
sys.exit(1)
def get_rds_info(indentifier=None):
"""Function for fetching RDS details"""
rds = boto.connect_rds()
try:
if indentifier:
info = rds.get_all_dbinstances(indentifier)[0]
else:
info = rds.get_all_dbinstances()
except boto.exception.BotoServerError:
info = None
if not info:
print 'Unable to get RDS details'
sys.exit(1)
return info
def create_db(number, label, inst_class, param_grp):
"""Creates a RDS instance
Args:
number index number of RDS instance
label prefix to apply to name
inst_class RDS class to sue
param_grp param group to use
Returns:
RDS instance
"""
rds = boto.connect_rds()
logging.info("Creating RDS instance "
"with default database: {} {} {}\n".format(label, inst_class,
param_grp))
inst = rds.create_dbinstance(id='{}-{}-{}'.format(label, number, param_grp),
allocated_storage=10,
instance_class=inst_class,
master_username='******',
master_password='******',
param_group=param_grp,
backup_retention_period='0',
preferred_backup_window='01:00-02:00',
engine_version='5.5',
security_groups=['default'])
time.sleep(30)
inst.update()
while inst.status != 'available':
time.sleep(30)
inst.update()
db = connect(database='', username='******', pwd='changeME',
hostname=inst.endpoint[0])
c = db.cursor()
try:
c.execute("CREATE DATABASE {}".format(DEFAULT_DATABASE), None)
c.execute("grant all on {}.* to ".format(DEFAULT_DATABASE) + \
"{}@'%' identified by '{}'".format(DEFAULT_USER,
DEFAULT_PASSWD))
except:
logging.error("setting up {}, error {} {}\n".format(
'{}-{}-{}'.format(label, number, param_grp),
db.errno(), db.error()))
finally:
c.close()
db.close()
return inst
def create_param_groups(name_prefix, params_to_vary, engine='MySQL5.5'):
"""Creates new sets of parameter groups based on list
Args:
name_prefix prefix of name for new parameter groups
A numerical index is appended to the name
params_to_vary list of list of tuples, each tuple being a
specific DB parameter and its value
engine Version of MySQL to use
An empty list for a parameter set will create a parameter
group based on the default Amazon parameter group
Returns:
list of names of parameter groups
"""
rds = boto.connect_rds()
retVal = []
def set_param(param, val):
pg = rds.get_all_dbparameters(pg_name)
if param in pg:
pg[param].value = val
pg[param].apply()
else:
pg2 = rds.get_all_dbparameters(pg_name, marker=pg.Marker)
pg2[param].value = val
pg2[param].apply()
for index, param_set in enumerate(params_to_vary):
pg_name = "pg{}-{}".format(name_prefix, index)
new_param_group = rds.create_parameter_group(pg_name, engine=engine,
description='{} {}'.format(name_prefix, index))
if isinstance(param_set, list):
for pval in param_set:
set_param(*pval)
else:
set_param(*param_set)
retVal.append(new_param_group.name)
return retVal
def launch_rds_instance(self, id, dbname, allocated_storage, instance_class,
availability_zone, multi_az=False, timeout=0):
"""Launch an rds instance. If timeout is > 0, then this method
won't return until the rds instance is fully running or the timeout
duration expires."""
# TODO: support multiple regions (and multiple security groups)
# import logging
# boto.set_file_logger('rds', '/tmp/boto.log', level=logging.DEBUG)
# conn = boto.connect_rds(self.key, self.secret, debug=2)
conn = boto.connect_rds(self.key, self.secret)
instance = conn.create_dbinstance(id, allocated_storage, instance_class,
self.username, self.password, db_name=dbname,
availability_zone=availability_zone, multi_az=multi_az)
self.log.info('Launched rds instance %s' % instance.id)
if timeout:
time.sleep(5.0) # avoids intermittent error
self.wait_until_endpoint(instance, timeout)
return instance
def connect(self):
""" Creates an RDSConnection instance. """
if self._connection is None:
self._connection = connect_rds(**self._credentials)
return self._connection
from pprint import pprint
import os
import time
import boto
import boto.manage.cmdshell
# connect up parts of the Amazon infrastructure
# notes: to delete snapshots I have made of instances, one has to deregister the AMI first and then delete the snapshot
GLUEJAR_ACCOUNT_ID = 439256357102
ec2 = boto.connect_ec2()
cw = boto.connect_cloudwatch()
rds = boto.connect_rds()
route53 = boto.connect_route53()
iam = boto.connect_iam()
def all_instances():
# "A reservation corresponds to a command to start instances"
# http://boto.readthedocs.org/en/latest/ec2_tut.html?highlight=reservation
reservations = ec2.get_all_instances()
instances = [i for r in reservations for i in r.instances]
return instances
def all_zones():
return ec2.get_all_zones()
def get_entities_for_region(self, region):
rds = boto.connect_rds(self.access_key_id, self.secret_access_key, region=region)
return rds.get_all_dbinstances()
def connect(account_name, connection_type, **args):
"""
Examples of use:
ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False)
ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000)
ec2 = sts_connect.connect(environment, 'ec2')
where environment is ( test, prod, dev )
s3 = sts_connect.connect(environment, 's3')
ses = sts_connect.connect(environment, 'ses')
:param account: Account to connect with (i.e. test, prod, dev)
:raises Exception: RDS Region not valid
AWS Tech not supported.
:returns: STS Connection Object for given tech
:note: To use this method a SecurityMonkey role must be created
in the target account with full read only privledges.
"""
account = Account.query.filter(Account.name == account_name).first()
sts = boto.connect_sts()
role = sts.assume_role('arn:aws:iam::' + account.number + ':role/SecurityMonkey', 'secmonkey')
if connection_type == 'ec2':
return boto.connect_ec2(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'elb':
if 'region' in args:
region = args['region']
del args['region']
else:
region = 'us-east-1'
return boto.ec2.elb.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 's3':
if 'region' in args:
region = args['region']
# drop region key-val pair from args or you'll get an exception
del args['region']
return boto.s3.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_s3(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'ses':
return boto.connect_ses(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'iam':
if 'region' in args:
region = args['region']
# drop region key-val pair from args or you'll get an exception
del args['region']
return boto.iam.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_iam(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'route53':
return boto.connect_route53(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'sns':
if 'region' in args:
region = args['region']
del args['region']
return boto.sns.connect_to_region(
region.name,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_sns(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'sqs':
if 'region' in args:
region = args['region']
del args['region']
return boto.sqs.connect_to_region(
region.name,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_sqs(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'vpc':
return boto.connect_vpc(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'rds':
if 'region' in args:
reg = args['region']
rds_region = None
for boto_region in boto.rds.regions():
if reg.name == boto_region.name:
rds_region = boto_region
if rds_region is None:
raise Exception('The supplied region {0} is not in boto.rds.regions. {1}'.format(reg, boto.rds.regions()))
return boto.connect_rds(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
err_msg = 'The connection_type supplied (%s) is not implemented.' % connection_type
raise Exception(err_msg)
def delete_rds_instance(self, id):
"""Terminate an rds instance."""
self.log.info('Deleting rds instance %s..' % id)
conn = boto.connect_rds(self.key, self.secret)
conn.delete_dbinstance(id, skip_final_snapshot=True)
self.log.info('Deleted rds instance %s' % id)
import boto
import csv
rds = boto.connect_rds(aws_access_key_id='',aws_secret_access_key='')
def add_row(d):
with open('rds.csv', 'a') as f:
a = csv.writer(f, delimiter=',')
a.writerow(d)
instances = rds.get_all_dbinstances()
for i in instances :
data=[i,i.instance_class,i.endpoint[0]]
add_row(data)
def connect(account_name, connection_type, **args):
"""
Examples of use:
ec2 = sts_connect.connect(environment, 'ec2', region=region, validate_certs=False)
ec2 = sts_connect.connect(environment, 'ec2', validate_certs=False, debug=1000)
ec2 = sts_connect.connect(environment, 'ec2')
where environment is ( test, prod, dev )
s3 = sts_connect.connect(environment, 's3')
ses = sts_connect.connect(environment, 'ses')
:param account: Account to connect with (i.e. test, prod, dev)
:raises Exception: RDS Region not valid
AWS Tech not supported.
:returns: STS Connection Object for given tech
:note: To use this method a SecurityMonkey role must be created
in the target account with full read only privledges.
"""
account = Account.query.filter(Account.name == account_name).first()
sts = boto.connect_sts()
role = sts.assume_role(
'arn:aws:iam::' + account.number + ':role/SecurityMonkey', 'secmonkey')
if connection_type == 'ec2':
return boto.connect_ec2(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'elb':
if 'region' in args:
region = args['region']
del args['region']
else:
region = 'us-east-1'
return boto.ec2.elb.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 's3':
if 'region' in args:
region = args['region']
# drop region key-val pair from args or you'll get an exception
del args['region']
return boto.s3.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_s3(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'ses':
return boto.connect_ses(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'iam':
if 'region' in args:
region = args['region']
# drop region key-val pair from args or you'll get an exception
del args['region']
return boto.iam.connect_to_region(
region,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_iam(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'route53':
return boto.connect_route53(
role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'sns':
if 'region' in args:
region = args['region']
del args['region']
return boto.sns.connect_to_region(
region.name,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_sns(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'sqs':
if 'region' in args:
region = args['region']
del args['region']
return boto.sqs.connect_to_region(
region.name,
aws_access_key_id=role.credentials.access_key,
aws_secret_access_key=role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
return boto.connect_sqs(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'vpc':
return boto.connect_vpc(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
if connection_type == 'rds':
if 'region' in args:
reg = args['region']
rds_region = None
for boto_region in boto.rds.regions():
if reg.name == boto_region.name:
rds_region = boto_region
if rds_region is None:
raise Exception(
'The supplied region {0} is not in boto.rds.regions. {1}'.
format(reg, boto.rds.regions()))
return boto.connect_rds(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token,
**args)
err_msg = 'The connection_type supplied (%s) is not implemented.' % connection_type
raise Exception(err_msg)
env.key_name = "lpkey"
env.db_name = "lp"
env.db_user = "******"
AMIs = {
"ubuntu-11-10-64": "ami-00b14b69",
}
env.aws_key = local("echo $LP_AWS_ACCESS_KEY", capture=True)
env.aws_secret = local("echo $LP_AWS_SECRET_KEY", capture=True)
env.owner_id = local("echo $LP_AWS_OWNER_ID", capture=True)
env.key_filename = local("echo $LP_KEY_FILENAME", capture=True)
if not (env.aws_key and env.aws_secret and env.key_filename and env.owner_id):
abort(red("You must set LP_AWS_ACCESS_KEY, LP_AWS_SECRET_KEY, LP_AWS_OWNER_ID, and LP_KEY_FILENAME in your environment."))
env.ec2_conn = boto.connect_ec2(env.aws_key, env.aws_secret)
env.rds_conn = boto.connect_rds(env.aws_key, env.aws_secret)
env.security_groups = dict([(sg.name, sg) for sg in env.ec2_conn.get_all_security_groups()])
DEFAULT_BOOTSTRAP_SCRIPT = "fabfile/bootstrap.sh"
DEFAULT_USER_DATA_FILE = "fabfile/user_data.sh"
def _generate_password(length=36):
return ''.join([random.choice(string.ascii_letters + string.digits) for i in range(length)])
def _slugify(value):
return value.replace(" ", "-")
def _launch_ec2_ami(ami, *args, **kwargs):
kwargs["placement"] = env.zone
user_data_file = kwargs.pop("user_data_file", None)
if user_data_file:
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', '--stack-name', required=True,
help="The name of the cloudformation stack.")
parser.add_argument('-n', '--noop',
help="Don't make any changes.", action="store_true",
default=False)
parser.add_argument('-z', '--zone-name', default="edx.org",
help="The name of the zone under which to "
"create the dns entries.")
parser.add_argument('-f', '--force',
help="Force reuse of an existing name in a zone",
action="store_true", default=False)
parser.add_argument('--aws-id', default=None,
help="read only aws key for fetching instance information"
"the account you wish add entries for")
parser.add_argument('--aws-secret', default=None,
help="read only aws id for fetching instance information for"
"the account you wish add entries for")
args = parser.parse_args()
# Connect to ec2 using the provided credentials on the commandline
ec2_con = boto.connect_ec2(args.aws_id, args.aws_secret)
elb_con = boto.connect_elb(args.aws_id, args.aws_secret)
rds_con = boto.connect_rds(args.aws_id, args.aws_secret)
# Connect to route53 using the user's .boto file
r53 = boto.connect_route53()
zone = get_or_create_hosted_zone(args.zone_name)
update_elb_rds_dns(zone)
def update_elb_rds_dns(zone):
"""
Creates elb and rds CNAME records
in a zone for args.stack_name.
Uses the tags of the instances attached
to the ELBs to create the dns name
"""
dns_records = set()
elb_con = boto.connect_elb()
rds_con = boto.connect_rds()
vpc_id = vpc_for_stack_name(args.stack_name)
if not zone and args.noop:
# use a placeholder for zone name
# if it doesn't exist
zone_name = "<zone name>"
else:
zone_name = zone.Name[:-1]
stack_elbs = [
elb for elb in elb_con.get_all_load_balancers() if elb.vpc_id == vpc_id
]
for elb in stack_elbs:
if "RabbitMQ" in elb.source_security_group.name or "ElasticSearch" in elb.source_security_group.name:
env_tag, deployment, play_tag = get_security_group_dns(
elb.source_security_group.name)
fqdn = "{}-{}.{}".format(env_tag, play_tag, zone_name)
dns_records.add(DNSRecord(zone, fqdn, 'CNAME', 600,
[elb.dns_name]))
else:
env_tag, play_tag = get_dns_from_instances(elb)
fqdn = "{}-{}.{}".format(env_tag, play_tag, zone_name)
dns_records.add(DNSRecord(zone, fqdn, 'CNAME', 600,
[elb.dns_name]))
if extra_play_dns.has_key(play_tag):
for name in extra_play_dns.get(play_tag):
fqdn = "{}-{}.{}".format(env_tag, name, zone_name)
dns_records.add(
DNSRecord(zone, fqdn, 'CNAME', 600, [elb.dns_name]))
stack_rdss = [
rds for rds in rds_con.get_all_dbinstances()
if hasattr(rds.subnet_group, 'vpc_id')
and rds.subnet_group.vpc_id == vpc_id
]
# TODO the current version of the RDS API doesn't support
# looking up RDS instance tags. Hence, we are using the
# env_tag that was set via the loop over instances above.
for rds in stack_rdss:
fqdn = "{}-{}.{}".format(env_tag, 'rds', zone_name)
dns_records.add(
DNSRecord(zone, fqdn, 'CNAME', 600, [stack_rdss[0].endpoint[0]]))
add_or_update_record(dns_records)
def delete_rds_instance(self, id):
"""Terminate an rds instance."""
self.log.info('Deleting rds instance %s..' % id)
conn = boto.connect_rds(self.key, self.secret)
conn.delete_dbinstance(id, skip_final_snapshot=True)
self.log.info('Deleted rds instance %s' % id)
desired_role = 'awsdit-role'
role_description = 'awsditRoleTest'
role = sts.assume_role(
'arn:aws:iam::{0}:role/{1}'.format(account_number, desired_role),
role_description)
## make a connection to an AWS service such as below, and continue with your normal code.
#EC2
ec2 = boto.connect_ec2(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token)
#RDS
rds = boto.connect_rds(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token)
#ELB
elb = boto.connect_elb(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token)
#IAM
iam = boto.connect_iam(role.credentials.access_key,
role.credentials.secret_key,
security_token=role.credentials.session_token)
#S3
s3 = boto.connect_s3(role.credentials.access_key,
role.credentials.secret_key,
env.db_name = "rah"
env.db_user = "******"
AMIs = {
"ubuntu-10.10-32": "ami-b61de9df",
"ubuntu-10.10-64": "ami-548c783d",
}
env.aws_key = local("echo $RAH_AWS_ACCESS_KEY", capture=True)
env.aws_secret = local("echo $RAH_AWS_SECRET_KEY", capture=True)
if not (env.aws_key and env.aws_secret):
abort(
red("You must set RAH_AWS_ACCESS_KEY and RAH_AWS_SECRET_KEY in your environment."
))
env.ec2_conn = boto.connect_ec2(env.aws_key, env.aws_secret)
env.rds_conn = boto.connect_rds(env.aws_key, env.aws_secret)
env.security_groups = dict([(sg.name, sg)
for sg in env.ec2_conn.get_all_security_groups()])
DEFAULT_BOOTSTRAP_SCRIPT = "fabfile/bootstrap.sh"
DEFAULT_USER_DATA_FILE = "fabfile/user_data.sh"
def _generate_password(length=36):
return ''.join([
random.choice(string.ascii_letters + string.digits)
for i in range(length)
])
def _slugify(value):
def rdss_for_stack_name(stack_name):
vpc_id = vpc_for_stack_name(stack_name)
rds = boto.connect_rds()
for instance in rds.get_all_dbinstances():
if hasattr(instance, 'VpcId') and instance.VpcId == vpc_id:
yield instance
parser = argparse.ArgumentParser(description=description)
parser.add_argument('-s', '--stack-name', required=True,
help="The name of the cloudformation stack.")
parser.add_argument('-n', '--noop',
help="Don't make any changes.", action="store_true",
default=False)
parser.add_argument('-z', '--zone-name', default="edx.org",
help="The name of the zone under which to "
"create the dns entries.")
parser.add_argument('-f', '--force',
help="Force reuse of an existing name in a zone",
action="store_true", default=False)
parser.add_argument('--aws-id', default=None,
help="read only aws key for fetching instance information"
"the account you wish add entries for")
parser.add_argument('--aws-secret', default=None,
help="read only aws id for fetching instance information for"
"the account you wish add entries for")
args = parser.parse_args()
# Connect to ec2 using the provided credentials on the commandline
ec2_con = boto.connect_ec2(args.aws_id, args.aws_secret)
elb_con = boto.connect_elb(args.aws_id, args.aws_secret)
rds_con = boto.connect_rds(args.aws_id, args.aws_secret)
# Connect to route53 using the user's .boto file
r53 = boto.connect_route53()
zone = get_or_create_hosted_zone(args.zone_name)
update_elb_rds_dns(zone)
def rdss_for_stack_name(stack_name):
vpc_id = vpc_for_stack_name(stack_name)
rds = boto.connect_rds()
for instance in rds.get_all_dbinstances():
if hasattr(instance, 'VpcId') and instance.VpcId == vpc_id:
yield instance
#account_number = '<12-digit-aws-account-number>'
# make sts call
sts = boto.connect_sts()
desired_role = 'awsdit-role'
role_description = 'awsditRoleTest'
role = sts.assume_role('arn:aws:iam::{0}:role/{1}'.format(account_number, desired_role), role_description)
## make a connection to an AWS service such as below, and continue with your normal code.
#EC2
ec2 = boto.connect_ec2(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#RDS
rds = boto.connect_rds(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#ELB
elb = boto.connect_elb(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#IAM
iam = boto.connect_iam(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#S3
s3 = boto.connect_s3(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#SQS
sns = boto.connect_sqs(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
#SNS
sns = boto.connect_sns(role.credentials.access_key, role.credentials.secret_key, security_token=role.credentials.session_token)
def get_entities_for_region(self, region):
rds = boto.connect_rds(self.access_key_id,
self.secret_access_key,
region=region)
return rds.get_all_dbinstances()
def connect(self):
""" Creates an RDSConnection instance. """
if self._connection is None:
self._connection = connect_rds(**self._credentials)
return self._connection
def get_info(self):
## boto initial connection required before connecting to a region ##
boto.connect_vpc()
awsRegionConn = BotoVPC.connect_to_region(self.region)
for _vid in awsRegionConn.get_all_vpcs():
if _vid.id == self.vpc_id:
self._valid = True
self.cidr_block = _vid.cidr_block
self.region = _vid.region.name
for _sub in awsRegionConn.get_all_subnets(filters=[('vpcId',
_vid.id)]):
self.subnets[_sub.id] = {
'cidr': _sub.cidr_block,
'zone': _sub.availability_zone
}
if 'Name' in _vid.tags.keys():
self.name = _vid.tags['Name']
if 'Description' in _vid.tags.keys():
self.description = _vid.tags['Description']
for _sg in awsRegionConn.get_all_security_groups(
filters={'vpc_id': self.vpc_id}):
self.security_groups[_sg.id] = {
'name': _sg.name,
'description': _sg.description
}
for _resv in awsRegionConn.get_all_instances(
filters={'vpc_id': self.vpc_id}):
_inst = _resv.instances[0]
self.instances[_inst.id] = {
'private_ip': _inst.private_ip_address,
'public_ip': _inst.ip_address,
'state': _inst.state,
'subnet_id': _inst.subnet_id,
'type': _inst.instance_type,
'avail_zone': _inst.placement,
}
if 'Name' in _inst.tags.keys():
self.instances[_inst.id]['Name'] = _inst.tags['Name']
# get RDS info - Call is required before switching regions#
boto.connect_rds()
rdsRegionConn = BotoRDS.connect_to_region(self.region)
for _rds in rdsRegionConn.get_all_dbinstances():
try:
if _rds.VpcId == self.vpc_id:
self.rds[_rds.id] = {
'zone': _rds.availability_zone,
'class': _rds.instance_class,
'size': _rds.allocated_storage,
}
try:
self.rds[_rds.id]['db_name'] = _rds.DBName
except AttributeError as e:
self.rds[_rds.id]['db_name'] = ''
except AttributeError as e:
# VpcId attribute is not available if not in a VPC #
pass
if not self._valid:
raise VpcIdError("VpcId: %s Not found in AWS account!" %
(self.vpc_id))
