def find_instance(ec2, instance_id, module):
""" Attempt to find the EC2 instance and return it """
try:
reservations = ec2.get_all_reservations(instance_ids=[instance_id])
except boto.exception.EC2ResponseError, e:
module.fail_json(msg=str(e))
def get_instances_info(ec2, vpc_info):
reservations = ec2.get_all_reservations()
count, total = init_status(None, 'Instances')
for reservation in reservations:
total = total + len(reservation.instances)
for i in reservation.instances:
vpc_id = i.vpc_id if i.vpc_id else 'no-vpc'
manage_dictionary(vpc_info[vpc_id], 'instances', {})
manage_dictionary(vpc_info[vpc_id]['instances'], i.id, {})
vpc_info[vpc_id]['instances'][
i.id]['reservation_id'] = reservation.id
# Get instance variables (see http://boto.readthedocs.org/en/latest/ref/ec2.html#module-boto.ec2.instance to see what else is available)
for key in [
'id', 'public_dns_name', 'private_dns_name', 'key_name',
'launch_time', 'private_ip_address', 'ip_address',
'instance_type'
]:
vpc_info[vpc_id]['instances'][i.id][key] = i.__dict__[key]
# FIXME ... see why it's not working when added in the list above
vpc_info[vpc_id]['instances'][i.id]['state'] = i.state
vpc_info[vpc_id]['instances'][
i.id]['profile_arn'] = i.instance_profile[
'arn'] if i.instance_profile else ''
manage_dictionary(vpc_info[vpc_id]['instances'][i.id],
'security_groups', [])
for sg in i.groups:
vpc_info[vpc_id]['instances'][i.id]['security_groups'].append(
sg.id)
# Network interfaces
vpc_info[vpc_id]['instances'][i.id]['interfaces'] = []
for interface in i.interfaces:
vpc_info[vpc_id]['instances'][i.id]['interfaces'].append(
interface.id)
count = update_status(count, total, 'Instances')
close_status(count, total, 'Instances')
def find_instance(ec2, instance_id, module):
""" Attempt to find the EC2 instance and return it """
try:
reservations = ec2.get_all_reservations(instance_ids=[instance_id])
except boto.exception.EC2ResponseError, e:
module.fail_json(msg=str(e))
def trackRWorkers():
print "Tracking workers"
while True:
reservations = ec2.get_all_reservations(filters={"instance-state-code" : [0, 16]})
max_workers = 16
#dynamic
#print "Number of requests " + str(requests.count()) + " and number of reservations " + str(len(reservations))
if requests.count() > 0 and len(reservations) == 2: #there are requests, but no workers
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
elif len(reservations) < max_workers:
if requests.count() % 2 == 0 and len(reservations)-2 < requests.count():
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
elif requests.count() % 2 == 1 and len(reservations)-2 < requests.count()-1:
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
def get_instances_info(ec2, vpc_info):
reservations = ec2.get_all_reservations()
count, total = init_status(None, 'Instances')
for reservation in reservations:
total = total + len(reservation.instances)
for i in reservation.instances:
vpc_id = i.vpc_id if i.vpc_id else 'no-vpc'
manage_dictionary(vpc_info[vpc_id], 'instances', {})
manage_dictionary(vpc_info[vpc_id]['instances'], i.id, {})
vpc_info[vpc_id]['instances'][i.id]['reservation_id'] = reservation.id
# Get instance variables (see http://boto.readthedocs.org/en/latest/ref/ec2.html#module-boto.ec2.instance to see what else is available)
for key in ['id', 'public_dns_name', 'private_dns_name', 'key_name', 'launch_time', 'private_ip_address', 'ip_address', 'instance_type']:
vpc_info[vpc_id]['instances'][i.id][key] = i.__dict__[key]
# Get instance name
if 'Name' in i.tags and i.tags['Name'] != '':
vpc_info[vpc_id]['instances'][i.id]['name'] = i.tags['Name']
else:
vpc_info[vpc_id]['instances'][i.id]['name'] = i.id
read_tags(vpc_info[vpc_id]['instances'][i.id], i)
# FIXME ... see why it's not working when added in the list above
vpc_info[vpc_id]['instances'][i.id]['state'] = i.state
vpc_info[vpc_id]['instances'][i.id]['profile_arn'] = i.instance_profile['arn'] if i.instance_profile else ''
manage_dictionary(vpc_info[vpc_id]['instances'][i.id], 'security_groups', [])
for sg in i.groups:
vpc_info[vpc_id]['instances'][i.id]['security_groups'].append(sg.id)
# Network interfaces
vpc_info[vpc_id]['instances'][i.id]['interfaces'] = []
for interface in i.interfaces:
vpc_info[vpc_id]['instances'][i.id]['interfaces'].append(interface.id)
count = update_status(count, total, 'Instances')
close_status(count, total, 'Instances')
def print_instances():
reservations = ec2.get_all_reservations()
for reservation in reservations:
for instance in reservation.instances:
if 'Name' in instance.tags:
print "%s (%s) [%s]" % (instance.tags['Name'], instance.id, instance.state)
else:
print "%s [%s]" % (instance.id, instance.state)
def check_id(choice):
success = False
reservations = ec2.get_all_reservations()
for reservation in reservations:
for instance in reservation.instances:
if str(instance.id) == choice:
success = True
return success
def get_filtered_servers(region, filters):
ec2 = boto.ec2.connect_to_region(region)
reservations = ec2.get_all_reservations(filters=filters)
filtered = []
for r in reservations:
for i in r.instances:
filtered.append(i.id)
return filtered
def spawn(hints, ec2):
logging.info("About to launch {0} instance(s). Press Control-C to abort!".format(hints['count']))
for t in range(5,0,-1):
logging.info("Launching in {0}...".format(t))
time.sleep(1)
profile_suffix = '' if hints['tags']['environment'] == 'prod' else '_' + hints['tags']['environment']
instance_profile = hints['tags']['application'] + "_profile" + profile_suffix
logging.info("OK, sending launch request!")
# Make the reservation call.
if hints['public_ip']:
# http://stackoverflow.com/questions/19029588/how-to-auto-assign-public-ip-to-ec2-instance-with-boto
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id=hints['subnet_id'],
groups=hints['security_group_ids'],
associate_public_ip_address=True)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
reservation = ec2.run_instances(
hints['ami_id'],
instance_type = hints['instance_type'],
network_interfaces=interfaces,
instance_profile_name = instance_profile
)
else:
reservation = ec2.run_instances(
hints['ami_id'],
min_count = hints['count'],
max_count = hints['count'],
instance_type = hints['instance_type'],
subnet_id = hints['subnet_id'],
security_group_ids = hints['security_group_ids'],
instance_profile_name = instance_profile,
)
# Sleep a little bit.
logging.info("Launch request sent. Waiting for instances to be created...")
time.sleep(5)
reservation_id = reservation.id
sentinel = 0
while len(reservation.instances) != hints['count'] or [i for i in reservation.instances if i.private_ip_address is None]:
print '.'
time.sleep(2)
sentinel += 1
if sentinel > 30:
logging.error("Giving up!")
exit(11)
reservation = ec2.get_all_reservations(filters={"reservation-id":reservation_id})
hints['ips'] = {}
# Apply tags to the instances.
for idx in range(0, len(hints['names'])):
instance = reservation.instances[idx]
logging.info ("Tagging instance {0} as {1}".format(instance.id, hints['names'][idx]))
instance.add_tags(hints['tags'])
instance.add_tag('Name', value = hints['names'][idx])
# Store private IP by name
hints['ips'][hints['names'][idx]] = instance.private_ip_address
def get_running_instances(region): ec2 = boto.ec2.connect_to_region(region) reservations = ec2.get_all_reservations() running = [] for r in reservations: for i in r.instances: if i.state == 'running': running.append(i) return running
def get_stopped_instances(region): ec2 = boto.ec2.connect_to_region(region) reservations = ec2.get_all_reservations() stopped = [] for r in reservations: for i in r.instances: if i.state == 'stopped': stopped.append(i.id) return stopped
def check_ec2(account, region, ec2):
logging.info('checking ec2')
try:
for res in ec2.get_all_reservations():
for inst in res.instances:
logging.info('instance id: {}'.format(inst.id))
check_instance_tags(account, region, 'ec2', inst.id, inst.tags)
except EC2ResponseError as e:
logging.warning('exception: {}'.format(e.error_message))
def check_state(choice):
reservations = ec2.get_all_reservations(
filters={'instance-id': choice})
for reservation in reservations:
for instance in reservation.instances:
state = instance.update()
while state not in ('running', 'stopped', 'terminated'):
time.sleep(2)
state = instance.update()
print " state:", state
def check_ec2(account, region, ec2):
logging.info('checking ec2')
try:
for res in ec2.get_all_reservations():
for inst in res.instances:
logging.info('instance id: {}'.format(inst.id))
check_instance_tags(account, region, 'ec2', inst.id, inst.tags)
except EC2ResponseError as e:
logging.warning('exception: {}'.format(e.error_message))
def find_instance(ec2, instance_id):
""" Attempt to find the EC2 instance and return it """
reservations = ec2.get_all_reservations(instance_ids=[instance_id])
if len(reservations) == 1:
instances = reservations[0].instances
if len(instances) == 1:
return instances[0]
raise EIPException("could not find instance" + instance_id)
def find_instance(ec2, instance_id):
""" Attempt to find the EC2 instance and return it """
reservations = ec2.get_all_reservations(instance_ids=[instance_id])
if len(reservations) == 1:
instances = reservations[0].instances
if len(instances) == 1:
return instances[0]
raise EIPException("could not find instance" + instance_id)
def tag_instances():
for region in regions():
region_name = region.name
if region_name != 'cn-north-1' and region_name != 'us-gov-west-1': # Skip these regions
print "Region: %s" % (region_name)
# Connect
ec2 = boto.ec2.connect_to_region(
region_name,
aws_access_key_id=access_key,
aws_secret_access_key=access_secret)
cloudtrail = boto.cloudtrail.connect_to_region(
region_name,
aws_access_key_id=access_key,
aws_secret_access_key=access_secret)
reservations = ec2.get_all_reservations()
tags = ec2.get_all_tags()
for reservation in reservations:
for instance in reservation.instances:
events_dict = cloudtrail.lookup_events(
lookup_attributes=[{
'AttributeKey': 'ResourceName',
'AttributeValue': instance.id
}])
if len(events_dict['Events']) == 0:
print("No CloudTrail events for instance: %s - %s" %
(instance.id, instance.instance_type))
else:
for data in events_dict['Events']:
json_file = json.loads(data['CloudTrailEvent'])
# Only interested in RunInstances (e.g. created instances) to find owners
# There's also StartInstances, but that event is fired if someone else
# restarts an instance, which isn't what we're really looking for
if json_file['eventName'] == 'RunInstances':
arn = json_file['userIdentity']['arn']
username = json_file['userIdentity'][
'userName']
user_type = json_file['userIdentity']['type']
print(
"Tagging Instance: %s, Username: %s, ARN: %s, Type: %s, eventName: %s"
% (instance.id, username, arn, user_type,
json_file['eventName']))
# Tag the instance
ec2.create_tags(
[instance.id], {
"IAM Username": username,
"IAM ARN": arn,
"IAM Type": user_type
})
# CloudTrail calls are throttled if there's more than 1 req/s
time.sleep(1)
def _fetch_ec2_instances():
'''
Returns list of all DevEnv VMs running in Amazon EC2.
'''
# Prices are as of 10/17/2017 based on "grep t2.type | grep per" of this file:
# https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/us-west-2/index.json
# with EBS costs added.
ebs_per_gigabyte_hour = 0.10 / 24 / 30
prices = {
"t2.micro": 0.0116 + 25 * ebs_per_gigabyte_hour,
"t2.small": 0.0232 + 25 * ebs_per_gigabyte_hour,
"t2.medium": 0.0464 + 25 * ebs_per_gigabyte_hour,
"t2.large": 0.0928 + ebs_per_gigabyte_hour,
}
results = []
now = datetime.datetime.now()
region = __salt__['ktpillar.get']('salt:cloud:providers:aws:location')
ec2 = boto.ec2.connect_to_region(region)
for reservation in ec2.get_all_reservations():
for vm in reservation.instances:
result = {"provider": "aws", "hostname": "Unknown"}
is_devenv_vm = False
for tag in vm.tags:
if tag == "Name":
result["hostname"] = vm.tags[tag]
if tag == "DevEnv" and vm.tags[tag] == "True":
is_devenv_vm = True
# Skip the loop if this is not a DevEnv VM
if not is_devenv_vm or "DEL" in result["hostname"]:
continue
result["ip"] = vm.private_ip_address
result["specs"] = vm.instance_type
result["state"] = vm.state
result["cost_hr"] = prices.get(vm.instance_type, 0)
launch_time = datetime.datetime.strptime(vm.launch_time,
"%Y-%m-%dT%H:%M:%S.000Z")
age = now.replace(tzinfo=None) - launch_time.replace(tzinfo=None)
age_hours = (24 * age.days) + (age.seconds / 3600)
result["cost_total"] = age_hours * result["cost_hr"]
results.append(result)
return results
def find_device(ec2, device_id, isinstance=True):
""" Attempt to find the EC2 instance and return it """
if isinstance:
try:
reservations = ec2.get_all_reservations(instance_ids=[device_id])
except boto.exception.EC2ResponseError, e:
module.fail_json(msg=str(e))
if len(reservations) == 1:
instances = reservations[0].instances
if len(instances) == 1:
return instances[0]
def find_device(ec2, device_id, isinstance=True):
""" Attempt to find the EC2 instance and return it """
if isinstance:
try:
reservations = ec2.get_all_reservations(instance_ids=[device_id])
except boto.exception.EC2ResponseError, e:
module.fail_json(msg=str(e))
if len(reservations) == 1:
instances = reservations[0].instances
if len(instances) == 1:
return instances[0]
def list_instances():
instances = {}
ec2 = boto.ec2.connect_to_region(env.ec2_region)
for reservation in ec2.get_all_reservations():
for instance in reservation.instances:
if instance.vpc_id == env.ec2_vpc_id:
(instances.setdefault(instance.placement_group, [])
.append(instance))
print ''
for pg, instances in instances.iteritems():
for instance in instances:
print ' %s (%s): %s' % (instance.id,
instance.instance_type,
instance.private_ip_address)
def my_booted_machine(reservation_id):
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None, None
try:
reservations = ec2.get_all_reservations()
reservation = [r for r in reservations if r.id == reservation_id]
if len(reservation) > 0 and len(reservation[0].instances) > 0:
return reservation[0].instances[0].id, reservation[0].instances[0].ip_address
except Exception, e:
logging.exception('Could not get reservations for %s (%s)' % (reservation_id, e))
return None, None
def active_instance_count():
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None
try:
total = 0
reservations = ec2.get_all_reservations()
for reservation in reservations:
total += len(reservation.instances)
return total
except Exception, e:
logging.exception('Could not get attributes (%s)' % e)
return None
def active_instance_count():
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None
try:
total = 0
reservations = ec2.get_all_reservations()
for reservation in reservations:
total += len(reservation.instances)
return total
except Exception, e:
logging.exception('Could not get attributes (%s)' % e)
return None
def get_instance_map(cluster, existing_machines_def_file):
instance_map = {}
if existing_machines_def_file is not None:
instance_map = {}
existing_machines_def = file(existing_machines_def_file)
existing_machines = json.load(existing_machines_def)
for node in existing_machines:
node_detail = existing_machines[node]
new_instance = {}
new_instance['private_ip_address'] = node_detail['ip_address']
if 'is_bastion' in node_detail and node_detail[
'is_bastion'] is True:
new_instance['ip_address'] = node_detail['public_ip_address']
else:
new_instance['ip_address'] = None
new_instance['node_type'] = node_detail['node_type']
try:
new_instance['node_idx'] = int(node.split('-')[-1])
except ValueError:
new_instance['node_idx'] = ''
new_instance['name'] = node_detail['ip_address']
instance_map[cluster + '-' + node] = new_instance
existing_machines_def.close()
else:
CONSOLE.debug('Checking details of created instances')
region = PNDA_ENV['ec2_access']['AWS_REGION']
ec2 = boto.ec2.connect_to_region(region)
reservations = ec2.get_all_reservations()
instance_map = {}
for reservation in reservations:
for instance in reservation.instances:
if 'pnda_cluster' in instance.tags and instance.tags[
'pnda_cluster'] == cluster and instance.state == 'running':
CONSOLE.debug(instance.private_ip_address, ' ',
instance.tags['Name'])
instance_map[instance.tags['Name']] = {
"public_dns": instance.public_dns_name,
"ip_address": instance.ip_address,
"private_ip_address": instance.private_ip_address,
"name": instance.tags['Name'],
"node_idx": instance.tags['node_idx'],
"node_type": instance.tags['node_type']
}
return instance_map
def my_booted_machine(reservation_id):
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None, None
try:
reservations = ec2.get_all_reservations()
reservation = [r for r in reservations if r.id == reservation_id]
if len(reservation) > 0 and len(reservation[0].instances) > 0:
return reservation[0].instances[0].id, reservation[0].instances[
0].ip_address
except Exception, e:
logging.exception('Could not get reservations for %s (%s)' %
(reservation_id, e))
return None, None
def get_inventory():
"""
Retrieve the inventory from a set of regions in an Ansible Dynamic
Inventory compliant format (see
http://docs.ansible.com/ansible/developing_inventory.html#script-conventions).
Instances are filtered through instance_filter, grouped by the
ec2_pod_instance_name tag, and contain host specific variables
according to get_instance_vars.
"""
inventory = {}
instances = []
# Gather all instances that pass instance_filter into instances
for region in get_regions():
ec2 = boto.ec2.connect_to_region(
region,
aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'))
region_instances = sum(
[x.instances for x in ec2.get_all_reservations()], [])
instances += [x for x in region_instances if instance_filter(x)]
# Build up main inventory, instance_name is something like "head" or "node"
# instance_name_instances are the boto.ec2.instance objects that have an
# ec2_pod_instance_name tag value of instance_name
for (instance_name,
instance_name_instances) in instances_by_name(instances):
inventory[instance_name] = {
'hosts': [x.ip_address for x in instance_name_instances]
}
# Build up _meta/hostvars for individual instances
hostvars = {
instance.ip_address: get_instance_vars(instance)
for instance in instances
}
if hostvars:
inventory['_meta'] = {'hostvars': hostvars}
return inventory
def get_inventory():
"""
Retrieve the inventory from a set of regions in an Ansible Dynamic
Inventory compliant format (see
http://docs.ansible.com/ansible/developing_inventory.html#script-conventions).
Instances are filtered through instance_filter, grouped by the
ec2_pod_instance_name tag, and contain host specific variables
according to get_instance_vars.
"""
inventory = {}
instances = []
# Gather all instances that pass instance_filter into instances
for region in get_regions():
ec2 = boto.ec2.connect_to_region(
region,
aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'))
region_instances = sum([x.instances for x in
ec2.get_all_reservations()], [])
instances += [x for x in region_instances if instance_filter(x)]
# Build up main inventory, instance_name is something like "head" or "node"
# instance_name_instances are the boto.ec2.instance objects that have an
# ec2_pod_instance_name tag value of instance_name
for (instance_name, instance_name_instances) in instances_by_name(
instances):
inventory[instance_name] = {
'hosts': [x.ip_address for x in instance_name_instances]
}
# Build up _meta/hostvars for individual instances
hostvars = {instance.ip_address: get_instance_vars(instance)
for instance in instances}
if hostvars:
inventory['_meta'] = {
'hostvars': hostvars
}
return inventory
def get_instance_map(cluster):
CONSOLE.debug('Checking details of created instances')
region = pnda_env['ec2_access']['AWS_REGION']
ec2 = boto.ec2.connect_to_region(region)
reservations = ec2.get_all_reservations()
instance_map = {}
for reservation in reservations:
for instance in reservation.instances:
if 'pnda_cluster' in instance.tags and instance.tags[
'pnda_cluster'] == cluster and instance.state == 'running':
CONSOLE.debug(instance.private_ip_address, ' ',
instance.tags['Name'])
instance_map[instance.tags['Name']] = {
"public_dns": instance.public_dns_name,
"ip_address": instance.ip_address,
"private_ip_address": instance.private_ip_address,
"name": instance.tags['Name'],
"node_idx": instance.tags['node_idx'],
"node_type": instance.tags['node_type']
}
return instance_map
def create_from_current_instances(cls):
servers = []
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
rs = ec2.get_all_reservations()
for reservation in rs:
for instance in reservation.instances:
try:
Server.find(instance_id=instance.id).next()
boto.log.info('Server for %s already exists' % instance.id)
except StopIteration:
s = cls()
s.ec2 = ec2
s.name = instance.id
s.region_name = region.name
s.instance_id = instance.id
s._reservation = reservation
s.put()
servers.append(s)
return servers
def create_from_current_instances(cls):
servers = []
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
rs = ec2.get_all_reservations()
for reservation in rs:
for instance in reservation.instances:
try:
next(Server.find(instance_id=instance.id))
boto.log.info('Server for %s already exists' % instance.id)
except StopIteration:
s = cls()
s.ec2 = ec2
s.name = instance.id
s.region_name = region.name
s.instance_id = instance.id
s._reservation = reservation
s.put()
servers.append(s)
return servers
def create_from_instance_id(cls, instance_id, name, description=''):
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
try:
rs = ec2.get_all_reservations([instance_id])
except:
rs = []
if len(rs) == 1:
s = cls()
s.ec2 = ec2
s.name = name
s.description = description
s.region_name = region.name
s.instance_id = instance_id
s._reservation = rs[0]
for instance in s._reservation.instances:
if instance.id == instance_id:
s._instance = instance
s.put()
return s
return None
def create_from_instance_id(cls, instance_id, name, description=''):
regions = boto.ec2.regions()
for region in regions:
ec2 = region.connect()
try:
rs = ec2.get_all_reservations([instance_id])
except:
rs = []
if len(rs) == 1:
s = cls()
s.ec2 = ec2
s.name = name
s.description = description
s.region_name = region.name
s.instance_id = instance_id
s._reservation = rs[0]
for instance in s._reservation.instances:
if instance.id == instance_id:
s._instance = instance
s.put()
return s
return None
def find_device(ec2, module, device_id, isinstance=True):
""" Attempt to find the EC2 instance and return it """
if isinstance:
try:
reservations = ec2.get_all_reservations(instance_ids=[device_id])
except boto.exception.EC2ResponseError as e:
module.fail_json(msg=str(e))
if len(reservations) == 1:
instances = reservations[0].instances
if len(instances) == 1:
return instances[0]
else:
try:
interfaces = ec2.get_all_network_interfaces(network_interface_ids=[device_id])
except boto.exception.EC2ResponseError as e:
module.fail_json(msg=str(e))
if len(interfaces) == 1:
return interfaces[0]
raise EIPException("could not find instance" + device_id)
def main():
"Test passing arguments to Python script"
global region # pylint: disable=invalid-name
parser = argparse.ArgumentParser( description="This is an AWS test script")
parser.add_argument("-r", "--region", dest="region", help="AWS region", default="ap-southeast-2")
args = parser.parse_args()
region = args.region
try:
print( "connecting to region '", region, "'", sep='' )
ec2 = boto.ec2.connect_to_region( region )
if (not ec2):
raise
except:
print( "Error connecting to region '", region, "'", sep='' )
sys.exit( 3 )
reservations = ec2.get_all_reservations()
if ( reservations ):
print( reservations )
else:
print( "No instances created in region %s" % (region))
def metric_series(instance_id):
metric = cloudwatch.get_metric_statistics(
'300',
datetime.datetime.utcnow() - datetime.timedelta(seconds=1800),
datetime.datetime.utcnow(),
'CPUUtilization',
'AWS/EC2',
'Average',
dimensions={'InstanceId':instance_id},
unit='Percent'
)
return metric
reservations = ec2.get_all_reservations(filters={'instance-state-name': 'running', 'tag:Usage' : tag})
ave_count = 0
cpu_value = 0
for reservation in reservations:
for instance in reservation.instances:
for metric in (metric_series(instance.id)):
cpu_value = cpu_value + metric['Average']
ave_count = ave_count + 1
average = cpu_value / ave_count
print ave_count
print cpu_value
print average
# # Currently unused options
# valid_from=None, valid_until=None, launch_group=None,
# availability_zone_group=None,
# addressing_type=None, # only the shadow knows
# placement=None, kernel_id=None, ramdisk_id=None,
# monitoring_enabled=False, subnet_id=None, placement_group=None,
# instance_profile_arn=None, instance_profile_name=None,
# security_group_ids=None, ebs_optimized=False, network_interfaces=None,
block_device_map=bdm)
# get the request ids to wait on
request_ids = [req.id for req in requests]
# wait for the requests to be fulfilled
instance_ids = wait_for_fulfillment(ec2, request_ids,
copy.deepcopy(request_ids))
reservation = ec2.get_all_reservations(instance_ids=instance_ids)
else:
reservation = [
ec2.run_instances(ami,
min_count=count,
max_count=count,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
block_device_map=bdm,
user_data=user_data)
]
# The instance has been launched but it's not yet up and
# running. Let's wait for its state to change to 'running'.
backup_retention = ami_backup_config.backup_retention
for server in servers :
server_name = server['name']
account_profile = server['profile']
server_pattern = server['pattern']
server_region = server['region']
# create ec2 connection using boto config profiles
ec2 = boto.ec2.connect_to_region(server_region, profile_name = account_profile)
if ( ec2 is None ):
print "ERROR - " + server_name + ": unable to connect"
logger.error( server_name + ": unable to connect to region " + server_region + " with profile " + account_profile )
continue
# filter instances by tag name
reservations = ec2.get_all_reservations(filters = {'tag:Name':server_pattern, 'instance-state-name':'*'})
if ( len(reservations) == 0 ):
print "ERROR - " + server_name + ": unable to find server " + server_pattern
logger.error( server_name + ": unable to find server " + server_pattern )
continue
# loop through reservations and instances
for reservation in reservations:
for instance in reservation.instances:
#print instance.__dict__.keys()
instance_name = instance.tags['Name']
instance_id = instance.id
print "\n" + server_name + ": " + instance_name + " (" + instance_id + ")"
current_datetime = datetime.datetime.now()
date_stamp = current_datetime.strftime("%Y-%m-%d_%H-%M-%S")
ami_name = instance_name + signature + date_stamp
try:
def get_instances(ec2):
instances = ec2.get_all_reservations()
return instances[0].instances
regions['rds'] = [r for r in boto.rds2.regions() if r.name in args.regions]
profiles = [None] if args.profiles is None else args.profiles
if args.ec2:
print "EC2 Reservation Report"
print args.delimiter.join(
['Instance', 'Placement', 'Run', 'Reserve', 'Diff'])
for region in regions['ec2']:
running_instances = defaultdict(int)
reserved_instances = defaultdict(int)
for profile in profiles:
ec2 = region.connect(profile_name=profile)
running = ec2.get_all_reservations(
filters={'instance-state-name': 'running'})
for r in running:
for i in r.instances:
running_instances[i.instance_type + args.delimiter +
i.placement] += 1
for ri in ec2.get_all_reserved_instances(
filters={'state': ['active', 'payment-pending']}):
reserved_instances[ri.instance_type + args.delimiter +
ri.availability_zone] += ri.instance_count
print_results(running_instances, reserved_instances, args.delimiter)
# Redshift
if args.redshift:
print "Redshift Reservation Report"
regions['redshift'] = [r for r in boto.redshift.regions() if r.name in args.regions]
regions['rds'] = [r for r in boto.rds2.regions() if r.name in args.regions]
profiles = [None] if args.profiles is None else args.profiles
if args.ec2:
print "EC2 Reservation Report"
print args.delimiter.join(['Instance', 'Placement', 'Run', 'Reserve', 'Diff'])
for region in regions['ec2']:
running_instances = defaultdict(int)
reserved_instances = defaultdict(int)
for profile in profiles:
ec2 = region.connect(profile_name=profile)
running = ec2.get_all_reservations(filters={'instance-state-name': 'running'})
for r in running:
for i in r.instances:
running_instances[i.instance_type + args.delimiter + i.placement] += 1
for ri in ec2.get_all_reserved_instances(filters={'state': ['active', 'payment-pending']}):
reserved_instances[ri.instance_type + args.delimiter + ri.availability_zone] += ri.instance_count
print_results(running_instances, reserved_instances, args.delimiter)
# Redshift
if args.redshift:
print "Redshift Reservation Report"
print args.delimiter.join(['NodeType', 'Region', 'Running', 'Reserve', 'Diff'])
for region in regions['redshift']:
running_nodes = defaultdict(int)
import sys
import re
import subprocess
#Variables
playbook1 = 'webserver.yml'
playbook2 = ''
playbook3 = ''
ec2 = boto.ec2.connect_to_region('ap-southeast-1')
inst = boto.ec2.autoscale.connect_to_region('ap-southeast-1')
grp = inst.get_all_groups(['ASG-1'])[0]
inst_ids = [i.instance_id for i in grp.instances]
res = ec2.get_all_reservations(inst_ids)
str = ""
str1 = ""
if len(res) != 1:
for i in range(len(res)):
subres = res[i].instances
#for i in range(len(subres)):
subfam = subres[0]
if str == "":
if subfam.state == "running":
str = subfam.ip_address
else:
if subfam.state == "running":
str = str + "\n" + subfam.ip_address
#for i in range(len(subres)):
def get_instances():
return reduce(lambda a, b: a + b,
[a.instances for a in ec2.get_all_reservations()], [])
def get_slave_vms(owner=None, rendersession=None, pool=None, state=None, instance_type=None):
print(colored("DEBUG: DQCCcloud.get_slave_vms()", 'green'))
usable_slaves = []
# build filters
filter_ami = {'image-id': DQCCconfig.ec2_slave_ami}
## filtering with custom tags will be supported in OpenStack Havana
## see https://blueprints.launchpad.net/nova/+spec/ec2-tags-api
filter_owner = {'tag:user_id': owner}
filter_rendersession = {'tag:rendersession': rendersession}
filter_pool = {'tag:pool_list': '*' + str(pool) + '*'}
filter_state = {'instance-state-name': state}
filter_instance_type = {'instance-type': instance_type}
# chain filters
filters = {}
filters.update(filter_ami)
## filtering with custom tags will be supported in OpenStack Havana
## see https://blueprints.launchpad.net/nova/+spec/ec2-tags-api
if owner != None:
filters.update(filter_owner)
if rendersession != None:
filters.update(filter_rendersession)
if pool != None:
filters.update(filter_pool)
if state != None:
filters.update(filter_state)
if instance_type != None:
filters.update(filter_instance_type)
print("DEBUG: Active filters = " + str(filters))
# walk through all registered VMs
for reservation in ec2.get_all_reservations(filters=filters):
for instance in reservation.instances:
# we are not interested in terminated/stopping VMs
if ("running" in instance.state) or ("pending" in instance.state):
# check age of VMs
launch_timestamp = DQCCcloud.datestring_to_timestamp(instance.launch_time)
print("DEBUG: Instance " + instance.id + " was started " + str( int(time.time() - launch_timestamp) ) + " seconds ago.")
# check tags of instance
print("DEBUG: Existing tags of VM " + instance.id + ":")
for tkey, tvalue in instance.tags.items():
print(" " + str(tkey) + ": " + str(tvalue))
#instance.remove_tag(tkey)
# check if VPN mode is enabled
if DQCCconfig.ec2_vpn_enabled:
# get VPN IP from private IP
vpn_ip = DQCCcloud.lookup_vpn_ip(instance.private_ip_address)
instance.add_tag('vpn_ip', vpn_ip)
if vpn_ip == None:
print("DEBUG: Could not look up VPN IP of VM " + instance.id + ".")
# stop VM if stuck
if DQCCcloud.check_max_wait(instance):
if DQCCconfig.testmode == False:
continue
else:
print("DEBUG: VPN IP of VM " + instance.id + " is " + vpn_ip + ".")
# VM uses VPN ip address for connecting to master
instance.add_tag('client_ip', vpn_ip)
else:
# VM uses private ip address for connecting to master
instance.add_tag('client_ip', instance.private_ip_address)
# get DrQueue computer info from client IP (either vpn_ip or private_ip)
if instance.tags['client_ip'] != None:
queue_info = DQCCimport.DQCCqueue.get_slave_info(instance.tags['client_ip'])
if queue_info == None:
print("DEBUG: Could not get queue info of VM " + instance.id + ".")
# stop VM if stuck
if DQCCcloud.check_max_wait(instance):
if DQCCconfig.testmode == False:
continue
else:
instance.add_tag('queue_info', str(queue_info))
print("DEBUG: Queue info of VM " + instance.id + " is \n" + str(queue_info) + ".")
# get list of pools from DrQueue computer info
pool_list = DQCCimport.DQCCqueue.concat_pool_names_of_computer(instance)
instance.add_tag('pool_list', pool_list)
print("DEBUG: Metadata for VM " + instance.id + " is updated.")
usable_slaves.append(instance)
else:
print("DEBUG: VM " + instance.id + " is not usable because its state is \'" + instance.state + "\'")
return usable_slaves
instance_type=instance_type, #instance type
# # Currently unused options
# valid_from=None, valid_until=None, launch_group=None,
# availability_zone_group=None,
# addressing_type=None, # only the shadow knows
# placement=None, kernel_id=None, ramdisk_id=None,
# monitoring_enabled=False, subnet_id=None, placement_group=None,
# instance_profile_arn=None, instance_profile_name=None,
# security_group_ids=None, ebs_optimized=False, network_interfaces=None,
block_device_map=bdm)
# get the request ids to wait on
request_ids = [req.id for req in requests]
# wait for the requests to be fulfilled
instance_ids = wait_for_fulfillment(ec2, request_ids, copy.deepcopy(request_ids))
reservation = ec2.get_all_reservations(instance_ids=instance_ids)
else:
reservation = [ec2.run_instances(ami,
min_count=count, max_count=count,
key_name=key_name,
security_group_ids=[group.id],
instance_type=instance_type,
block_device_map=bdm,
subnet_id='subnet-a3cb3bfa',
user_data=user_data)]
# The instance has been launched but it's not yet up and
# running. Let's wait for its state to change to 'running'.
print 'waiting for instances'
dry_run = True
message += "Terminating older than %s hour(s) old; Warning older than %s hour(s) old\n\n" % (a[1],a[2])
now = int(time.time())
#CALCULATE SECONDS FOR INTERVALS
delta_kill = datetime.timedelta(hours=int(a[1])).total_seconds()
delta_warn = datetime.timedelta(hours=int(a[2])).total_seconds()
#CONNECT TO REGION AS ARGUMENT 3
ec2 = boto.ec2.connect_to_region(a[3])
#PROCESS
reservations = ec2.get_all_reservations()
kill = []
warn = []
ok = []
for r in reservations:
for i in r.instances:
if i.subnet_id in exempt:
continue
if 'Name' in i.tags:
name = i.tags['Name']
else:
name = i.public_dns_name