def regions_with_azs(self):
e = Ec2()
regions = e.get_regions()
for region in regions:
name = region.get('RegionName')
print "Region: %s" % (name, )
r = Ec2(region=name)
azs = r.get_all_availability_zones(filter=[{
'Name': 'region-name',
'Values': [name]
}])
for a in azs:
print " - %s: %s" % (a.get('ZoneName'), a.get('State'))
def query_ips(self,
environment=None,
puppet_role=None,
xively_service=None):
filters = []
if environment:
filters.append({
'Name': 'tag:Environment',
'Values': [environment]
})
if puppet_role:
filters.append({
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
})
if xively_service:
filters.append({
'Name': 'tag:Xively_service',
'Values': [xively_service]
})
e = Ec2()
instances = e.get_all_instances(filters=filters)
ips = []
for instance in instances:
if instance['State']['Name'] == 'running':
ips.append({
'Public':
instance['PublicIpAddress']
if 'PublicIpAddress' in instance else '-',
'Private:':
instance['PrivateIpAddress']
if 'PrivateIpAddress' in instance else '-'
})
return ips
def get_snappy_index(self, num, vpcid):
from wrapper.ec2 import Ec2
ec2 = Ec2(session=self.session)
indexes = []
current = 1
while len(indexes) < num:
logger.info("Gathering snappyindex %s" % current)
filters = [{
'Name': 'tag:Snappy_index',
'Values': [str(current)]
}, {
'Name': 'instance-state-name',
'Values': ["running", "pending"]
}, {
'Name': 'vpc-id',
'Values': [vpcid]
}]
inst = ec2.information_ec2_instances(filters=filters)
if len(inst) == 0:
indexes.append(current)
logger.info("Found snappy_index not in use: %s" % current)
else:
logger.info("Snappy index in use")
current += 1
return indexes
def info_all(self, filter=None):
e = Ec2()
instances = e.get_all_instances(filters=filter)
output = []
for i in instances:
model = Misc.parse_instance(i)
output.append(model)
return output
def get_ami_stacks(self):
"""
This function returns all ami stacks in an account
:return: a dict containing the puppet_roles
"""
from wrapper.ec2 import Ec2
from wrapper.iam import Iam
ec2 = Ec2(session=self.session)
iam = Iam(session=self.session)
account_id = iam.get_account_id()
return ec2.get_ami_stacks(account_id=account_id)
def get_ami_from_tag(self, puppet_role):
from wrapper.ec2 import Ec2
from wrapper.iam import Iam
ec2 = Ec2(session=self.session)
iam_modul = Iam(session=self.session)
account_id = iam_modul.get_account_id()
baseami_object = ec2.get_images(account_id=account_id,
filters=[{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}])[0]
return baseami_object
def terminate_instance(self, dryrun, instanceids):
"""
This function is used to terminate instance-s within AWS
:param dryrun: No change should be performed, only validation is done
:type dryrun: bool
:param instanceids: an array of instance id's to terminate
:type instanceids: dict
:return: An array with dicts
:rtype: array
"""
from wrapper.ec2 import Ec2
ec2 = Ec2(session=self.session)
ret = ec2.terminate_instance(dryrun=dryrun, instanceids=instanceids)
return ret
def start_instances(self,
environment=None,
puppet_role=None,
xively_service=None,
dry_run=None):
filters = [{
'Name': 'tag:Environment',
'Values': [environment]
}, {
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}]
if xively_service:
filters.append({
'Name': 'tag:Xively_service',
'Values': [xively_service]
})
e = Ec2()
res = e.get_all_instances(filters=filters)
candidate_ids = []
for instance in res:
if instance['State']['Name'] == 'stopped':
candidate_ids.append(instance['InstanceId'])
if len(candidate_ids) == 0:
logger.warning("Couldn't find any instances to start")
return []
if not dry_run:
logger.info("Instances would start: {0}".format(candidate_ids))
res = e.start_instances(instance_ids=candidate_ids)
if not ('StartingInstances' in res):
logger.error(
"No instance started, error happened, result:{0}".format(
res))
return []
result = []
for instance in res['StartingInstances']:
result.append({
'InstanceId': instance['InstanceId'],
'State': instance['CurrentState']['Name']
})
return result
else:
logger.info("Instances would start: {0}".format(candidate_ids))
return []
def information_ec2_instances(self, columns, filters):
'''
This function gathers information about ec2 instances
:param columns: The requested columns that should be returned
:type columns: array
:param filters: The boto3 filters that should be used for filtering the results
:type filters: array
:return: an array with parsed instances for printing
:rtype: array
'''
from wrapper.ec2 import Ec2
from misc import Misc
ec2 = Ec2(session=self.session)
result = ec2.information_ec2_instances(filters=filters)
ret = []
for instance in result:
ret.append(
Misc.parse_object(object=instance,
columns=columns,
service="ec2"))
return ret
def create_launch_config(self,
launch_config_name=None,
env=None,
xively_service=None,
stack=None):
e = Ec2()
v = Vpc()
vpc = v.get_vpc_from_env(env=env)
keyname = Misc.get_value_from_array_hash(dictlist=vpc['Tags'],
key='Keypair')
baseami = e.query_base_image(stack=stack)
ostype = Misc.get_value_from_array_hash(dictlist=baseami['Tags'],
key='Os')
instance_type = Misc.get_value_from_array_hash(
dictlist=baseami['Tags'], key='Instancetype')
image = baseami.get('ImageId')
sgs = e.get_security_group_ids_for_launch(
vpcid=vpc.get('VpcId'),
stack=stack,
ostype=ostype,
xively_service=xively_service)
iam = "ec2"
y = Misc.get_app_ports_yaml('app_ports')
port = y[xively_service]
userdata = Misc.get_autoscale_userdata_for_os(ostype=ostype).format(
env=env, stack=stack, xively_service=xively_service, port=port)
monitoring = {}
monitoring['Enabled'] = True
self.autoscale.create_launch_configuration(
LaunchConfigurationName=launch_config_name,
ImageId=image,
KeyName=keyname,
SecurityGroups=sgs,
UserData=userdata,
InstanceType=instance_type,
InstanceMonitoring=monitoring,
IamInstanceProfile=iam)
def create_instance_lamdba(self, args=None):
"""
This function is invoked by lambda to provision multiple at same time
:param args: A dict with keypairs needed
:return: instance objects created
"""
from wrapper.ec2 import Ec2
from wrapper.iam import Iam
ec2 = Ec2(session=self.session)
iam = Iam(session=self.session)
account_id = iam.get_account_id()
inst_name = args['instance_name'].pop()
# linux or windows userdata formated for start
if 'snappyindex' in args:
logger.debug("Deploying a snappyindex")
snappyindex = args['snappyindex'].pop()
userdata = args['userdata']
userdata = userdata.format(index=snappyindex,
accountid=args['accountid'],
channelname=args['channelname'],
newrelic=args['newrelic'],
broker=args['broker'],
hostname=inst_name,
env=args['env'],
devicestring=args['devicestring'],
branch=args['branch'])
elif 'userdata' in args:
userdata = args['userdata']
else:
userdata = Misc.get_userdata_for_os(ostype=args['ostype']).format(
hostname=inst_name, env=args['env'], account=account_id)
instance = ec2.run_instance(baseamiid=args['baseamiid'],
key_name=args['keypair'],
securitygroup=args['securitygroup'],
instancetype=args['instance_type'],
subnet=args['subnet'].pop(),
user_data=userdata,
shutdown=args['shutdown'],
monitoring=args['monitoring'],
ebsoptimized=args['ebsoptimized'],
dry_run=args['dry_run'],
iam_name=args['iam'])
# add snappyindex to tag
if 'snappyindex' in args:
ec2.tag_resource(instanceid=instance.get('InstanceId'),
tags={
'Name': inst_name,
'Requester': args['requester'],
'Puppet_role': args['puppet_role'],
'Xively_service': args['xively_service'],
'Customer': args['customer'],
'Snappy_index': str(snappyindex),
'Environment': args['env']
})
else:
ec2.tag_resource(instanceid=instance.get('InstanceId'),
tags={
'Name': inst_name,
'Requester': args['requester'],
'Puppet_role': args['puppet_role'],
'Xively_service': args['xively_service'],
'Customer': args['customer'],
'Environment': args['env']
})
return instance
def create_ec2_instance(self,
puppet_role,
env,
requester,
customer,
xively_service,
base_ami,
iam,
instance_type,
dry_run,
shutdown,
monitoring,
fillup,
num,
keypair,
availability=None):
"""
This function creates an ec2 instance
:param puppet_role: the Puppet_role that should be used
:param env: the environment where we should provision to
:param requester: the user/team requesting the machine
:param customer: For future use only
:param xively_service: the Xively_service that should be used
:param base_ami: the base_ami that should be used. Can default to puppet_role
:param iam: The iam role that should be attached, defaults to ec2-base
:param instance_type: the type of instance requested
:param dry_run: No changes should be done
:param shutdown: The shutdown behavior to use
:param monitoring: Should monitoring be enabled
:param fillup: Should fillup algorithym be used or round robin
:param num: the number of instances
:return: a list of instance objects
"""
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
from misc import Misc
from core.stackdata import stackdata
stackdata_object = stackdata(session=self.session)
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
lambda_function_args = {
'env': env,
'puppet_role': puppet_role,
'requester': requester,
'xively_service': xively_service,
'customer': customer,
'shutdown': shutdown,
'dry_run': dry_run
}
stack_data = stackdata_object.get_stack_data(
puppet_role=puppet_role, xively_service=xively_service)
vpc_obj = vpc.get_vpc_from_env(env=env)
## Find the baseami object that needs to be used
if base_ami:
base_ami = base_ami
elif 'ami' in stack_data:
base_ami = stack_data['ami']
else:
logger.info("Falling back to puppet_role as AMI name")
base_ami = puppet_role
logger.info("The baseami that is going to be used: %s" % (base_ami, ))
baseami_object = self.get_ami_from_tag(puppet_role=base_ami)
## Get values for lambda function
lambda_function_args['baseamiid'] = baseami_object.get('ImageId')
if (availability == None):
availability = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Availability')
lambda_function_args['ostype'] = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Os')
if keypair is not None:
lambda_function_args['keypair'] = keypair
else:
lambda_function_args['keypair'] = Misc.get_value_from_array_hash(
dictlist=vpc_obj.get('Tags'), key='Keypair')
## Find the instance_type that needs to be used
if instance_type:
inst_type_final = instance_type
elif 'instance_type' in stack_data and env in stack_data[
'instance_type']:
inst_type_final = stack_data['instance_type'][env]
else:
inst_type_final = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Instancetype')
logger.info("Instance type that will be used: %s" %
(inst_type_final, ))
lambda_function_args['instance_type'] = inst_type_final
## Find the instance profile that needs to be used
if iam:
iam_name = iam
elif 'iam' in stack_data:
iam_name = "%s-%s" % (env, stack_data['iam']['name_postfix'])
else:
iam_name = "ec2-base"
logger.info("Base iam instance profile name: %s" % (iam_name, ))
lambda_function_args['iam'] = iam_name
## Find value for ebsoptimized
if 'ebsoptimized' in stack_data and env in stack_data['ebsoptimized']:
lambda_function_args['ebsoptimized'] = Misc.str2bool(
stack_data['ebsoptimized'][env])
else:
lambda_function_args['ebsoptimized'] = False
## Find value for monitoring enablement
if monitoring:
lambda_function_args['monitoring'] = monitoring
elif 'monitoring' in stack_data and env in stack_data['monitoring']:
lambda_function_args['monitoring'] = Misc.str2bool(
stack_data['monitoring'][env])
else:
lambda_function_args['monitoring'] = False
## Generate instance names for all required instances
lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name(
env=env, puppet_role=puppet_role, num=num)
## Gather all security groups needed for creating an instance stack
lambda_function_args[
'securitygroup'] = ec2.get_security_group_ids_for_stack(
vpcid=vpc_obj.get('VpcId'),
puppet_role=puppet_role,
ostype=lambda_function_args['ostype'],
xively_service=xively_service)
# We need to retrieve the subnets from Vpc object, and pass it to Ec2 object
subnets = vpc.get_all_subnets(filters=[{
'Name': 'tag:Network',
'Values': [availability]
}, {
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}])
lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym(
puppet_role=puppet_role,
subnets=subnets,
num=num,
fillup=fillup,
xively_service=xively_service)
instances = Misc.parallel_map_reduce(
lambda x: self.create_instance_lamdba(args=lambda_function_args),
lambda x, y: x + [y], xrange(0, num), [])
return instances
def deploy_snappy(self, env, num, dryrun, accountid, newrelic, channelname,
devicestring, branch):
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
vpc_obj = vpc.get_vpc_from_env(env=env)
num = int(num)
snappyindex = self.get_snappy_index(num=num,
vpcid=vpc_obj.get('VpcId'))
lambda_function_args = {
'env': "infra",
'puppet_role': 'benchmarkslave',
'requester': "benchmark",
'xively_service': "benchmark_slave",
'customer': "",
'shutdown': "stop",
'dry_run': dryrun,
'base_ami': "benchmarkslave",
'instance_type': 'c3.xlarge',
'snappyindex': snappyindex,
'accountid': accountid,
'channelname': channelname,
'newrelic': newrelic,
'iam': 'infra-benchmarkslave',
'ebsoptimized': False,
'monitoring': False,
'devicestring': devicestring,
'branch': branch
}
lambda_function_args['userdata'] = Misc.get_userdata_for_os(
ostype="snappy")
baseami_object = self.get_ami_from_tag(
puppet_role=lambda_function_args['base_ami'])
lambda_function_args['baseamiid'] = baseami_object.get('ImageId')
availability = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Availability')
lambda_function_args['ostype'] = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Os')
lambda_function_args['keypair'] = Misc.get_value_from_array_hash(
dictlist=vpc_obj.get('Tags'), key='Keypair')
lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name(
env=env, puppet_role="benchmarkslave", num=num)
lambda_function_args[
'securitygroup'] = ec2.get_security_group_ids_for_stack(
vpcid=vpc_obj.get('VpcId'),
puppet_role="benchmarkslave",
ostype=lambda_function_args['ostype'],
xively_service="benchmark_slave")
subnets = vpc.get_all_subnets(filters=[{
'Name': 'tag:Network',
'Values': [availability]
}, {
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}])
lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym(
puppet_role="benchmarkslave",
subnets=subnets,
num=num,
fillup=False,
xively_service="benchmark_slave")
## Get broker IP address
broker = ec2.get_ec2_instances(
filters=[{
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}, {
'Name': 'tag:Xively_service',
'Values': ['benchmark_master']
}, {
'Name': 'tag:Puppet_role',
'Values': ['linuxbase']
}])
lambda_function_args['broker'] = broker[0].get(
'PrivateIpAddress') + ":8883"
instances = Misc.parallel_map_reduce(
lambda x: self.create_instance_lamdba(args=lambda_function_args),
lambda x, y: x + [y], xrange(0, num), [])
return instances
def terminate_instance(self, dryrun=None, instanceids=None):
e = Ec2()
ret = e.terminate_instance(dryrun=dryrun, instanceids=instanceids)
return ret
def prepare_deployment(self, puppet_role, xively_service, env, num,
instance_type, base_ami, iam, requester, customer,
dry_run):
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
vpc_obj = vpc.get_vpc_from_env(env=env)
filter_base = [{'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')]}]
if xively_service:
old_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': ["%s_old" % xively_service]
}])
rollback_machines = ec2.get_ec2_instances(
filters=filter_base +
[{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': ["%s_rollback" % xively_service]
}])
current_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': [xively_service]
}])
else:
old_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': ["%s_old" % puppet_role]
}])
rollback_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': ["%s_rollback" % puppet_role]
}])
current_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}])
for old_machine in old_machines:
logger.info(msg="Going to stop old machine %s" %
old_machine.get('InstanceId'))
ec2.stop_instances(dryrun=dry_run,
instanceids=[old_machine.get('InstanceId')])
for rollback_machine in rollback_machines:
logger.info(msg="Going to stop old machine %s" %
rollback_machine.get('InstanceId'))
ec2.stop_instances(
dryrun=dry_run,
instanceids=[rollback_machine.get('InstanceId')])
if xively_service:
ec2.tag_resource(
instanceid=rollback_machine.get('InstanceId'),
tags={'Xively_service': "%s_old" % xively_service})
else:
ec2.tag_resource(instanceid=rollback_machine.get('InstanceId'),
tags={'Puppet_role': "%s_old" % puppet_role})
for current_machine in current_machines:
logger.info(msg="Going to retag current machine %s" %
current_machine.get('InstanceId'))
if xively_service:
ec2.tag_resource(
instanceid=current_machine.get('InstanceId'),
tags={'Xively_service': "%s_rollback" % xively_service})
else:
ec2.tag_resource(
instanceid=current_machine.get('InstanceId'),
tags={'Puppet_role': "%s_rollback" % puppet_role})
def get_image_stacks(self):
e = Ec2()
return e.get_active_images_stacks()
def get_console(self):
e = Ec2()
out = e.get_console_output(instance_id='i-c25879ed')
print out