def get_running_instances(access_key=None,
secret_key=None,
security_group=None,
region=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend(
[i for i in s.instances() if i.state == 'running'])
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
return instances_all_regions_list
def get_running_instances(access_key=None,
secret_key=None,
security_group_name=None,
security_group_type=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
for index, region in enumerate(ec2_region_list):
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=ec2_region_list[index])
running_instances = []
for s in conn.get_all_security_groups():
# A bit complex if to handle the different security group types - with backwards compatibility
type_match = (security_group_type.lower() == 'vpc' and s.vpc_id) or \
(security_group_type.lower() == 'ec2' and not s.vpc_id) or not security_group_type
if s.name == security_group_name and type_match:
running_instances.extend(
[i for i in s.instances() if i.state == 'running'])
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
return instances_all_regions_list
def handle(self, *args, **options):
ec2_connection = EC2Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
regions = boto.ec2.elb.regions()
my_priv_ip = get_my_private_ip()
for region in regions:
region_lb_connection = ELBConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=region)
# regions is a list of RegionInfo with connection_cls ELBConnection
# so for our EC2Connection we want to get an EC2Connection RegionInfo with the corresponding region descriptor
region_connection = EC2Connection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=boto.ec2.get_region(region.name))
load_balancers = region_lb_connection.get_all_load_balancers()
instances = [r.instances[0] for r in region_connection.get_all_instances()]
try:
# FIXME: TEST This in dev, stage, prod environments
me = [i for i in instances if i.private_ip_address == my_priv_ip][0]
my_main_group = me.groups[0].name
except IndexError:
me = None
my_main_group = 'dev'
instances = [i for i in instances if i.state != u'stopped' and i.groups[0].name == my_main_group]
load_balancers = [lb for lb in load_balancers if lb.name == my_main_group]
if load_balancers:
print region, load_balancers[0]
for instance in instances:
print instance.tags['Name'], instance.public_dns_name, instance.tags['Name'] + '.c2gops.com' # FIXME assumes basename
def get_running_instances(access_key=None, secret_key=None, security_group=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
if security_group:
for index, region in enumerate(ec2_region_list):
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=ec2_region_list[index])
running_instances = []
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances = [i for i in s.instances() if i.state == 'running']
if running_instances:
for instance in running_instances:
instances_all_regions_list.append(instance)
else:
for index, region in enumerate(ec2_region_list):
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=ec2_region_list[index])
reserved_instances = conn.get_all_instances()
if reserved_instances:
for reservation in reserved_instances:
for instance in reservation.instances:
if instance.stat == 'running':
instances_all_regions_list.append(instance)
return instances_all_regions_list
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = (self._get_aws_config_property()
or self._get_aws_config_from_file())
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
if 'ec2_region_name' in aws_config:
del (aws_config['ec2_region_name'])
# for backward compatibility,
# delete this key before passing config to Boto
if 'ec2_region_endpoint' in aws_config:
del (aws_config["ec2_region_endpoint"])
return EC2Connection(**aws_config)
def get_running_instances(access_key=None,
secret_key=None,
security_group=None,
region=None,
safe_mode=False,
delay=0):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
instances_all_regions_list = []
if region is None:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
ec2_region_list = conn.get_all_regions()
else:
ec2_region_list = [get_region(region)]
for region in ec2_region_list:
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region=region)
running_instances = []
date_format = '%Y-%m-%dT%H:%M:%S.%fZ'
try:
for s in conn.get_all_security_groups():
if s.name == security_group:
running_instances.extend([
i for i in s.instances() if i.state == 'running' and
(datetime.utcnow() - datetime.strptime(
i.launch_time, date_format)).seconds > delay
])
except boto.exception.EC2ResponseError:
logging.error('Region [' + region.name + '] inaccessible')
if safe_mode:
logging.error(
'Safe mode enabled. No new haproxy cfg is generated. Exit now.'
)
exit(1)
if running_instances:
for instance in running_instances:
logging.info(instance)
print(instance)
print("============================")
instances_all_regions_list.append(instance)
logging.info("**********************************")
logging.info(instances_all_regions_list)
instances_all_regions_list_sorted = sorted(instances_all_regions_list)
print(instances_all_regions_list_sorted)
return instances_all_regions_list_sorted
def _connect_to_region(self, **kwargs):
if self._isRegionInfo:
return EC2Connection(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
**kwargs)
for region in self.all_region(self.name):
if region.name == self.region:
self.region = region
return EC2Connection(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region=self.region, **kwargs)
def __init__(self, **kwargs):
super(EC2System, self).__init__(kwargs)
username = kwargs.get('username')
password = kwargs.get('password')
connection_config = Config(
signature_version='s3v4',
retries=dict(
max_attempts=10
)
)
regionname = kwargs.get('region')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.sqs_connection = connection.SQSConnection(username, password, region=_regions(
regionmodule=sqs, regionname=regionname))
self.elb_connection = ELBConnection(username, password, region=_regions(
regionmodule=elb, regionname=regionname))
self.s3_connection = boto3.resource('s3', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.ec2_connection = boto3.client('ec2', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.stackapi = CloudFormationConnection(username, password, region=_regions(
regionmodule=cloudformation, regionname=regionname))
self.cloudformation_connection = boto3.client('cloudformation', aws_access_key_id=username,
aws_secret_access_key=password, region_name=regionname, config=connection_config)
self.sns_connection = boto3.client('sns', region_name=regionname)
self.kwargs = kwargs
def main():
args = parser.parse_args()
jobs.setup_logger(None)
#
# Prepare init script for new EC2 instance to run.
#
with open(join(dirname(__file__), 'templates', 'user-data.sh')) as file:
user_data = file.read().format(**args.__dict__)
_L.info('Prepared {} bytes of instance user data'.format(len(user_data)))
#
# Figure out how much we're willing to bid on a spot instance.
#
ec2_access_key = args.ec2_access_key or environ.get(
'EC2_ACCESS_KEY_ID', args.access_key)
ec2_secret_key = args.ec2_secret_key or environ.get(
'EC2_SECRET_ACCESS_KEY', args.secret_key)
ec2 = EC2Connection(ec2_access_key, ec2_secret_key)
bid = get_bid_amount(ec2, args.instance_type, args.bid_strategy)
_L.info('Bidding ${:.4f}/hour for {} instance'.format(
bid, args.instance_type))
#
# Request a spot instance with 200GB storage.
#
device_sda1 = BlockDeviceType(size=200, delete_on_termination=True)
device_map = BlockDeviceMapping()
device_map['/dev/sda1'] = device_sda1
spot_args = dict(instance_type=args.instance_type,
user_data=user_data,
key_name=args.ssh_keypair,
block_device_map=device_map,
security_groups=[args.security_group])
spot_req = ec2.request_spot_instances(bid, args.machine_image,
**spot_args)[0]
_L.info(
'https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#SpotInstances:search={}'
.format(spot_req.id))
#
# Wait while EC2 does its thing, unless the user interrupts.
#
try:
wait_it_out(spot_req, time() + 12 * 60 * 60)
finally:
spot_req = ec2.get_all_spot_instance_requests(spot_req.id)[0]
if spot_req.instance_id:
print 'Shutting down instance {} early'.format(
spot_req.instance_id)
ec2.terminate_instances(spot_req.instance_id)
spot_req.cancel()
def __init__(self, **kwargs):
username = kwargs.get('username')
password = kwargs.get('password')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.stackapi = CloudFormationConnection(username, password)
self.kwargs = kwargs
def steal_elastic_ip(access_key=None, secret_key=None, ip=None):
'''
Assign an elastic IP to this instance.
'''
logging.debug('steal_elastic_ip()')
instance_id = get_self_instance_id()
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
conn.associate_address(instance_id=instance_id, public_ip=ip)
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = self._get_aws_config_property()
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
aws_config['region'] = region_object
del (aws_config['ec2_region_name'])
else:
aws_config = aws_config_property.copy()
return EC2Connection(**aws_config)
def test_utils_log_available_resources(self):
ctx = self.mock_ctx(
'test_utils_log_available_resources')
current_ctx.set(ctx=ctx)
client = EC2Connection()
key_pairs = client.get_all_key_pairs()
utils.log_available_resources(key_pairs)
def __init__(self, **kwargs):
super(EC2System, self).__init__(**kwargs)
self._username = kwargs.get('username')
self._password = kwargs.get('password')
connection_config = Config(signature_version='s3v4',
retries=dict(max_attempts=10))
self._region_name = kwargs.get('region')
self._region = get_region(self._region_name)
self.api = EC2Connection(self._username,
self._password,
region=self._region)
self.sqs_connection = boto3client('sqs',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.elb_connection = ELBConnection(self._username,
self._password,
region=_regions(
regionmodule=elb,
regionname=self._region_name))
self.s3_connection = boto3resource(
's3',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.ec2_connection = boto3client('ec2',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.ecr_connection = boto3client('ecr',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.cloudformation_connection = boto3client(
'cloudformation',
aws_access_key_id=self._username,
aws_secret_access_key=self._password,
region_name=self._region_name,
config=connection_config)
self.sns_connection = boto3client('sns', region_name=self._region_name)
self.kwargs = kwargs
def test_install_workflow(self, cfy_local):
""" Tests the install workflow using the built in
workflows.
"""
ec2_client = EC2Connection()
ec2_client.create_security_group('test_group2', 'so hard to describe')
kp = ec2_client.create_key_pair('test_key2')
kp.save(TEST_KEY_FOLDER)
# execute install workflow
cfy_local.execute('install', task_retries=0)
def client(self):
"""Represents the EC2Connection Client
"""
aws_config_property = (self._get_aws_config_property()
or self._get_aws_config_from_file())
if not aws_config_property:
return EC2Connection()
elif aws_config_property.get('ec2_region_name'):
region_object = \
get_region(aws_config_property['ec2_region_name'])
aws_config = aws_config_property.copy()
if region_object and 'ec2_region_endpoint' in aws_config_property:
region_object.endpoint = \
aws_config_property['ec2_region_endpoint']
aws_config['region'] = region_object
else:
aws_config = aws_config_property.copy()
aws_config = self.aws_config_cleanup(aws_config)
return EC2Connection(**aws_config)
def __init__(self, **kwargs):
access_key_id = kwargs['ec2_key_id']
secret_access_key = kwargs['ec2_secret']
endpoint = kwargs.get('nova_hostname', kwargs['hostname'])
port = kwargs.get('nova_port', 8773)
service_path = '/services/Cloud'
region = RegionInfo(None, 'openstack', endpoint)
self.api = EC2Connection(access_key_id,
secret_access_key,
region=region,
port=port,
path=service_path,
is_secure=False)
def main():
parser = optparse.OptionParser()
parser.add_option('-n', '--num-instances', type='int', default=1)
parser.add_option('-i', '--image')
options, extras = parser.parse_args()
if not options.image:
parser.error('--image is required')
if len(extras) != 0:
parser.error('Unrecognized options: %s' % ' '.join(extras))
parameters = [
'AWS_ACCESS_KEY_ID',
'AWS_PREFIX',
'AWS_SECRET_ACCESS_KEY',
'DATABASE_HOST',
'DATABASE_PASSWORD',
'DATABASE_PORT',
'DATABASE_USER',
'DEPLOYMENT_MODE',
'S3_HOST',
'S3_IS_SECURE',
'SOLR_HOST',
'SOLR_IS_SECURE',
'SOLR_PORT',
'SQS_HOST',
'SQS_IS_SECURE',
'SQS_VISIBILITY_TIMEOUT',
'SUBVERSION_USERNAME',
'SUBVERSION_PASSWORD',
]
user_data = '&'.join(
['%s=%s' % (p, getattr(settings, p)) for p in parameters])
conn = EC2Connection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
images = conn.get_all_images(options.image)
if len(images) != 1:
parser.error('Image %s not valid' % options.image_id)
reservation = images[0].run(min_count=1,
max_count=options.num_instances,
user_data=user_data)
for i in reservation.instances:
print repr(i), i.state
def _get_autoscaling_group():
'''
Returns None if the current instance is not in an autoscaling group.
'''
global _autoscaling_group
if not _autoscaling_group:
# Get the autoscaling group name
ec2conn = EC2Connection()
tags = ec2conn.get_all_tags({ 'key': 'aws:autoscaling:groupName',
'resource-id': _get_instance_id() })
_autoscaling_group = tags[0].value if tags else None
return _autoscaling_group
def __init__(self, **kwargs):
super(EC2System, self).__init__(kwargs)
username = kwargs.get('username')
password = kwargs.get('password')
regionname = kwargs.get('region')
region = get_region(kwargs.get('region'))
self.api = EC2Connection(username, password, region=region)
self.sqs_connection = connection.SQSConnection(username, password, region=_regions(
regionmodule=sqs, regionname=regionname))
self.elb_connection = ELBConnection(username, password, region=_regions(
regionmodule=elb, regionname=regionname))
self.s3_connection = boto.connect_s3(username, password)
self.stackapi = CloudFormationConnection(username, password, region=_regions(
regionmodule=cloudformation, regionname=regionname))
self.kwargs = kwargs
def get_running_instances(access_key=None,
secret_key=None,
security_group=None):
'''
Get all running instances. Only within a security group if specified.
'''
logging.debug('get_running_instances()')
conn = EC2Connection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
if security_group:
sg = SecurityGroup(connection=conn, name=security_group)
instances = [i for i in sg.instances() if i.state == 'running']
return instances
else:
instances = conn.get_all_instances()
return instances
def get_input():
'''
'''
github_client_id = environ['GITHUB_CLIENT_ID']
github_client_secret = environ['GITHUB_CLIENT_SECRET']
gdocs_client_id = environ['GDOCS_CLIENT_ID']
gdocs_client_secret = environ['GDOCS_CLIENT_SECRET']
print('--> Enter Github details:')
username = raw_input(' Github username: '******' Github password: '******' New Github repository name: ')
if not match(r'\w+(-\w+)*$', reponame):
raise RuntimeError('Repository "{}" does not match "\w+(-\w+)*$"'.format(reponame))
ec2 = EC2Connection()
return github_client_id, github_client_secret, \
gdocs_client_id, gdocs_client_secret, \
username, password, reponame, ec2
def get_client(self):
return EC2Connection()
def instance_run_jobs(code,
image_id,
key_name,
aws_access_key_id,
aws_secret_access_key,
shutdown_behavior='terminate',
instance_type='t2.micro',
dry_run=False,
**kwargs):
""" A simple wrapper around boto tools
image_id : str
The ID of the AMI.
key_name : str
The name of the ssh security keypair as you would see it in the AWS
console.
code : str
The code that you want to run, essentially the contents of a script.
Note that unlike the inputs to aws ec2 cli this does not accept
files, it must be the file contents with a shebang at the top.
aws_access_key_id : str
The key.
aws_secret_access_key : str
The secret key.
shutdown_behavior : str
see `instance_initiated_shutdown_behavior` in boto.
instance_type : str
see `instance_type` in boto.
But my favorites are:
t2.micro : nice for testing
c3.x2large : has SSDs on epehmeral0 and ephemeral1
c3.x4large : 2 times c3.x2large
c3.x8large : 4 c3.x2large
dry_run : bool
The dry run for testing.
block_device_map : instance BlockDeviceMapping
A particular dict-subclass from boto that maps the storage
devices to mount points.
This example
mapping = BlockDeviceMapping()
mapping["/dev/sdb"] = BlockDeviceType(ephemeral_name='ephemeral0')
Is equivalent to this JSON syntax:
[
{
"DeviceName": "/dev/sdb",
"VirtualName": "ephemeral0"
}
]
"""
ec2con = EC2Connection(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
out = ec2con.run_instances(
image_id=image_id,
key_name=key_name,
instance_type=instance_type,
user_data=code,
instance_initiated_shutdown_behavior=shutdown_behavior,
dry_run=dry_run,
**kwargs)
return out
def build_instance_list(reservation):
map(write_instances, reservation.instances)
def write_instances(instance):
# Beginning of alternate method if we choose to break out tags:
# print 'test1\n'
# for (tagname, tagvalue) in instance.tags.items():
# print tagname, tagvalue, '\n'
# print 'test2\n'
# csv_file.write("%s,%s,%s,%s,%s,%s,%s,%s,\"%s\"\n"%(instance.tags['Name'],instance.id,instance.public_dns_name,
# instance.state,instance.placement,instance.architecture,instance.instance_type,instance.launch_time,str(instance.tags)))
csv_file.write(
"%s,%s,%s,%s,%s,%s,%s,%s,\"%s\"\n" %
(instance.tags['Name'], instance.id, instance.public_dns_name,
instance.state, instance.placement,
instance.architecture, instance.instance_type, instance.launch_time,
json.JSONEncoder().encode(instance.tags)))
csv_file.flush()
if __name__ == "__main__":
connection = EC2Connection()
process_instance_list(connection)
csv_file.close()
def ec2_client(self):
credentials = self._client_credentials()
return EC2Connection(**credentials)
def ec2_connection(self):
if not self._ec2_connection:
conn = EC2Connection(self.access_key, self.secret_key)
self._ec2_connection = conn
return self._ec2_connection
def _get_ec2_client(self):
aws_config = self._get_aws_config()
aws_config.pop('ec2_region_name')
aws_config.pop('elb_region_name')
aws_config['region'] = get_region(self.env.ec2_region_name)
return EC2Connection(**aws_config)
def test_nested(self):
self.setup_sqs_queue()
conn = EC2Connection()
conn.run_instances('ami-123456')
def ec2(self):
from boto.ec2 import EC2Connection
self._ec2 = EC2Connection(self.access_key_id, self.secret_access_key)
return self._ec2