def __init__(self, deployment_name, ec2_nodes, rds_nodes, elb_nodes):
"""
``deployment_name`` A string uniquely identifying a deployment.
"""
self.deployment_name = deployment_name
self.deployment_confs = {}
self.deployment_confs['ec2'] = ec2_nodes
self.deployment_confs['rds'] = rds_nodes
self.deployment_confs['elb'] = elb_nodes
aws_credentials = self._get_valid_aws_credentials(
self.deployment_confs,
)
self._pending_gen_id = None
self._active_gen_id = None
self.ec2conn = ec2.EC2Connection(
aws_credentials['access_key_id'],
aws_credentials['secret_access_key'],
)
self.rdsconn = rds.RDSConnection(
aws_credentials['access_key_id'],
aws_credentials['secret_access_key'],
)
def buildApp():
'''Creates a new instance on ec2 and returns the ip address and summary information'''
with settings(warn_only=True):
conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key)
# check if it exists already
if not 'catmaidgroup' in [
group.name for group in conn.get_all_security_groups()
]:
print('Create CATMAID security group')
web = conn.create_security_group('catmaidgroup',
'CATMAID security group')
web.authorize('tcp', 80, 80, '0.0.0.0/0')
web.authorize('tcp', 22, 22, '0.0.0.0/0')
reservation = conn.run_instances(aws_AMI,
instance_type=aws_size,
key_name=aws_keypair_name,
security_groups=['catmaidgroup'])
instance = reservation.instances[0]
print('Starting instance %s' % (instance))
while not instance.update() == 'running':
time.sleep(1)
sys.stdout.write('.')
sys.stdout.flush()
# instance.add_tag('Name', 'incf_catmaid')
print('Instance started: %s' % instance.__dict__['id'])
print('Public DNS: %s\n' % instance.__dict__['public_dns_name'])
print(
'-> Write this public DNS to your configuration.py file (env.host_string)'
)
def mountStatus():
''' unused
'''
with settings(warn_only=True):
v_to_mount = ''
conn = ec2.EC2Connection(env.aws_access_key_id,
env.aws_secret_access_key)
vol = conn.get_all_volumes()
volid = 'vol-8317a983'
for v in vol:
print v
if v.id == volid:
v_to_mount = v
if v_to_mount:
if v_to_mount.attachment_state() == None:
print 'Volume not attached'
else:
print 'Volume attached with status: %s' % v_to_mount.attachment_state(
)
else:
print 'volume not found'
def terminate(): #terminate_instances with settings(warn_only = True): print 'killing last instance' conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key) conn.terminate_instances(env.last_instance) time.sleep(1)
def launch():
'''Creates a new large instance on ec2'''
with settings(warn_only = True):
conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key)
time.sleep(1)
reservation = conn.run_instances(baseAMI, instance_type='m1.small', key_name='ec2-keypair')
time.sleep(1)
instance = reservation.instances[0]
time.sleep(1)
print 'Starting instance %s' %(instance)
while not instance.update() == 'running':
time.sleep(1)
#print 'Instance started.'
instance.add_tag('Name', 'fz-automated-deploy')
time.sleep(1)
print 'Instance started: %s' % instance.__dict__['id']
print 'Private DNS: %s' % instance.__dict__['private_dns_name']
print 'Private IP: %s' % instance.__dict__['private_ip_address']
print 'Public DNS: %s' % instance.__dict__['public_dns_name']
# print '\n------------- ' + strftime("%Y-%m-%d %H:%M:%S", gmtime()) + '-------------'
return instance
def __init__(self, virtapi, read_only=False):
super(EC2Driver, self).__init__(virtapi)
self.host_status_base = {
'vcpus': VCPUS,
'memory_mb': MEMORY_IN_MBS,
'local_gb': DISK_IN_GB,
'vcpus_used': 0,
'memory_mb_used': 0,
'local_gb_used': 100000000000,
'hypervisor_type': 'EC2',
'hypervisor_version': '1.0',
'hypervisor_hostname': CONF.host,
'cpu_info': {},
'disk_available_least': 500000000000,
}
self._mounts = {}
self._interfaces = {}
self.creds = get_nova_creds()
self.nova = client.Client(**self.creds)
region = RegionInfo(name=aws_region, endpoint=aws_endpoint)
self.ec2_conn = ec2.EC2Connection(aws_access_key_id=CONF.ec2driver.ec2_access_key_id,
aws_secret_access_key=CONF.ec2driver.ec2_secret_access_key,
host=host,
port=port,
region=region,
is_secure=secure)
self.cloudwatch_conn = ec2.cloudwatch.connect_to_region(
aws_region, aws_access_key_id=CONF.ec2driver.ec2_access_key_id, aws_secret_access_key=CONF.ec2driver.ec2_secret_access_key)
self.security_group_lock = Lock()
self.instance_rule_refresher = InstanceRuleRefresher(
GroupRuleRefresher(
ec2_connection=self.ec2_conn,
openstack_rule_service=OpenstackRuleService(
group_service=OpenstackGroupService(self.nova.security_groups),
openstack_rule_transformer=OpenstackRuleTransformer()
),
ec2_rule_service=EC2RuleService(
ec2_connection=self.ec2_conn,
ec2_rule_transformer=EC2RuleTransformer(self.ec2_conn)
)
)
)
if not '_EC2_NODES' in globals():
set_nodes([CONF.host])
def mountstatus(): with settings(warn_only=True): v_to_mount = '' conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key) vol = conn.get_all_volumes() for v in vol: if v.id == volid: v_to_mount = v if v_to_mount.attachment_state() == None: print 'Volume not attached' else: print 'Volume attached with status: %s' % v_to_mount.attachment_state()
def __init__(self, *args, **kwargs):
super(EBSDriver, self).__init__(*args, **kwargs)
self.VERSION = '1.0.0'
self._wait_time_sec = 60 * (CONF.AWS.wait_time_min)
self._check_config()
region_name = CONF.AWS.region_name
endpoint = '.'.join(['ec2', region_name, 'amazonaws.com'])
region = RegionInfo(name=region_name, endpoint=endpoint)
self._conn = ec2.EC2Connection(
aws_access_key_id=CONF.AWS.access_key,
aws_secret_access_key=CONF.AWS.secret_key,
region=region)
# resort to first AZ for now. TODO: expose this through API
az = CONF.AWS.az
self._zone = filter(lambda z: z.name == az,
self._conn.get_all_zones())[0]
self.set_initialized()
def _create(size):
'''Creates a new large instance on ec2'''
with settings(warn_only = True):
conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key)
time.sleep(1)
reservation = conn.run_instances(precise_12_04_2, instance_type=size, placement='us-east-1d', key_name=ec2keypairname, security_group=['default', 'irodsweb'])
time.sleep(1)
instance = reservation.instances[0]
time.sleep(1)
print 'Starting instance %s' %(instance)
while not instance.update() == 'running':
time.sleep(1)
instance.add_tag('Name', 'ipython-deploy')
time.sleep(1)
print 'Instance started: %s' % instance.__dict__['id']
print 'Private DNS: %s' % instance.__dict__['private_dns_name']
print 'Private IP: %s' % instance.__dict__['private_ip_address']
print 'Public DNS: %s' % instance.__dict__['public_dns_name']
# write temporary settings in case something goes wrong mid-configuration
import ConfigParser
import sys
parser = ConfigParser.SafeConfigParser()
parser.add_section('lastLaunch')
parser.set('lastLaunch', 'host_string', str(instance.__dict__['public_dns_name']))
parser.set('lastLaunch', 'keypath', localkeypath)
parser.set('lastLaunch', 'username', 'ubuntu')
parser.set('lastLaunch', 'instance', instance.__dict__['id'])
parser.write(open('lastLaunch.ini', 'w'))
env.user = '******'
env.host_string = instance.__dict__['public_dns_name']
env.key_filename = [localkeypath]
print 'Instance has been launched successfully'
print 'To access, open a browser to http://%s' % (instance.__dict__['public_dns_name'])
print 'ssh -i ~/.ssh/ec2-keypair ubuntu@%s' % (instance.__dict__['public_dns_name'])
def unmountebs():
with settings(warn_only=True):
sudo('umount /dev/xvdf')
v_to_unmount = ''
conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key)
vol = conn.get_all_volumes()
for v in vol:
if v.id == volid:
v_to_unmount = v
result = v_to_unmount.detach(force=True)
if result == True:
print 'volume detached successfully'
else:
print 'volume not attached successfully'
print v_to_unmount.attachment_state()
def attachebs(): with settings(warn_only=True): v_to_mount = '' conn = ec2.EC2Connection(aws_access_key_id, aws_secret_access_key) vol = conn.get_all_volumes() for v in vol: if v.id == volid: v_to_mount = v print 'trying to attach volume %s to instance %s' % (v_to_mount, env.last_instance) if v_to_mount.attachment_state() == None: print 'volume not attached, continuing' result = v_to_mount.attach(env.last_instance, '/dev/xvdf') else: print v_to_mount.attachment_state()
def do_setup(self, context):
self._check_config()
region_name = CONF.AWS.region_name
endpoint = '.'.join(['ec2', region_name, 'amazonaws.com'])
region = RegionInfo(name=region_name, endpoint=endpoint)
self._conn = ec2.EC2Connection(
aws_access_key_id=CONF.AWS.access_key,
aws_secret_access_key=CONF.AWS.secret_key,
region=region)
# resort to first AZ for now. TODO(do_setup): expose this through API
az = CONF.AWS.az
try:
self._zone = filter(lambda z: z.name == az,
self._conn.get_all_zones())[0]
except IndexError:
raise AvailabilityZoneNotFound(az=az)
self.set_initialized()
def _connect(self):
endpoint = os.environ.get("EC2_ENDPOINT", "https://ec2.sa-east-1.amazonaws.com")
access_key = os.environ.get("EC2_ACCESS_KEY")
secret_key = os.environ.get("EC2_SECRET_KEY")
from boto import ec2
url = urlparse.urlparse(endpoint)
scheme = url.scheme
host_port = url.netloc.split(":")
host = host_port[0]
if len(host_port) > 1:
port = int(host_port[1])
else:
port = 80 if scheme == "http" else 443
path = url.path or "/"
region = ec2.RegionInfo(name="custom", endpoint=host)
return ec2.EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
host=host, port=port,
is_secure=scheme == "https",
path=path, region=region)
def setUp(self):
print "Establishing connection with AWS"
region = RegionInfo(name=aws_region, endpoint=aws_endpoint)
self.ec2_conn = ec2.EC2Connection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=host,
port=port,
region=region,
is_secure=secure)
self.creds = get_nova_creds()
self.nova = client.Client(**self.creds)
# nova client for cinder
self.creds['service_type'] = 'volume'
self.nova_volume = client.Client(**self.creds)
self.servers = []
self.volumes = []
def __init__(self, virtapi, read_only=False):
super(EC2Driver, self).__init__(virtapi)
self.host_status_base = {
'vcpus': CONF.AWS.max_vcpus,
'memory_mb': CONF.AWS.max_memory_mb,
'local_gb': CONF.AWS.max_disk_gb,
'vcpus_used': 0,
'memory_mb_used': 0,
'local_gb_used': 0,
'hypervisor_type': 'EC2',
'hypervisor_version': '1.0',
'hypervisor_hostname': CONF.host,
'cpu_info': {},
'disk_available_least': CONF.AWS.max_disk_gb,
}
global _EC2_NODES
self._mounts = {}
self._interfaces = {}
self._uuid_to_ec2_instance = {}
self.ec2_flavor_info = EC2_FLAVOR_MAP
aws_region = CONF.AWS.region_name
aws_endpoint = "ec2." + aws_region + ".amazonaws.com"
region = RegionInfo(name=aws_region, endpoint=aws_endpoint)
self.ec2_conn = ec2.EC2Connection(
aws_access_key_id=CONF.AWS.access_key,
aws_secret_access_key=CONF.AWS.secret_key,
region=region)
self.cloudwatch_conn = ec2.cloudwatch.connect_to_region(
aws_region,
aws_access_key_id=CONF.AWS.access_key,
aws_secret_access_key=CONF.AWS.secret_key)
# Allow keypair deletion to be controlled by conf
if CONF.AWS.enable_keypair_notifications:
eventlet.spawn(KeyPairNotifications(self.ec2_conn).run)
LOG.info("EC2 driver init with %s region" % aws_region)
if _EC2_NODES is None:
set_nodes([CONF.host])
from fabric.api import run, local, env, get, sudo, cd
from fabric.colors import yellow, red
from fabric.contrib import console
PROJECT_ROOT = os.path.dirname(os.path.realpath(__file__))
here = lambda *x: os.path.join(PROJECT_ROOT, *x)
config = ConfigParser.ConfigParser()
config.readfp(open(here('settings.cfg')))
# Configuration for the instance
EC2_INSTANCE_TYPE = 't1.micro'
AMI_ID = 'ami-9c78c0f5'
key = config.get('aws', 'key')
secret = config.get('aws', 'secret')
conn = ec2.EC2Connection(key, secret)
SECURITY_GROUP = 'proxy-aws-sg'
KEY_PAIR = config.get('aws', 'key_pair')
KEY_PATH = os.path.expanduser('~/.ssh/%s.pem' % KEY_PAIR)
USER = config.get('vpn', 'user')
PASSWORD = config.get('vpn', 'password')
# Tag name.
TAG_NAME = 'proxy-aws'
def _get_instance_details(instance):
return {
if listofdicts == []:
return None
table = PrettyTable(listofdicts[0].keys())
for i in listofdicts:
table.add_row(i.values())
return table.get_string()
def get_all_addresses(conn):
return conn.get_all_addresses()
def get_all_addresses_with_instance_id(conn):
l = get_all_addresses(conn)
list_of_addresses = []
for a in l:
list_of_addresses.append({
'address': a.public_ip,
'instance_id': a.instance_id
})
return list_of_addresses
aws_config = {
'aws_access_key_id': os.environ['AWS_ACCESS_KEY_ID'],
'aws_secret_access_key': os.environ['AWS_ACCESS_KEY']
}
conn = ec2.EC2Connection(**aws_config)
print make_pretty_instance_table(get_all_instances_uptime(conn))
print make_pretty_ip_table(get_all_addresses_with_instance_id(conn))
