def route53_info(self, env=None):
envs = super(awsrequests, self).get_needed_envs(env=env)
v = Vpc()
r = Route53()
res = []
for environment in envs:
logger.debug("Working in env: %s" % environment)
vpc = v.get_vpc_from_env(env=environment)
domain = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'),
key="Domain")
zoneid = r.get_zoneid_from_domain(domain=domain)
records = r.list_zone_records(zoneid=zoneid)
for record in records:
rec = record.pop('ResourceRecords', [])
values = []
for rr in rec:
values.append(rr['Value'])
record['Values'] = values
if 'AliasTarget' in record:
aliastarget = record.pop('AliasTarget')
record['TTL'] = 'alias'
record['Values'] = [aliastarget['DNSName']]
record['Env'] = environment
res.append(record)
logger.debug("Processed record is: %s" % record, )
return res
def get_elb_from_env(self, env=None):
v = Vpc()
vpc = v.get_vpc_from_env(env=env)
elbs = self.get_all_elbs()
ret = []
for elb in elbs:
if elb['VPCId'] == vpc.get('VpcId'):
ret.append(elb)
logger.debug("Elbs in env %s : %s" % (env, ret))
return ret
def get_sgs_for_elb(self, env=None, name=None):
ec2 = Ec2()
vpc = Vpc()
v = vpc.get_vpc_from_env(env=env)
sgs = ec2.get_security_groups(
filters=[{'Name': 'tag:ELB', 'Values': [name]}, {'Name': 'vpc-id', 'Values': [v.get('VpcId')]}])
res = []
for sg in sgs:
res.append(sg.get('GroupId'))
logger.debug("Sgs for the elb are %s" % res, )
return res
def get_server_cert_for_env(self, env=None):
certs = self.get_server_certs()
v = Vpc()
vpc = v.get_vpc_from_env(env=env)
domain = Misc.get_value_from_array_hash(dictlist=vpc['Tags'], key='Domain')
cert_name = "star." + domain
logger.debug("Searching for cert domain %s" % cert_name, )
for c in certs:
logger.debug("Investigateing Cert %s" % c, )
if c['ServerCertificateName'] == cert_name:
logger.debug("We found the server certificate we are looking for")
return c
logger.warning("Could not find the certificate for %s" % env, )
return None
def info_all(self):
elbs = self.get_all_elbs()
result = []
yaml = Misc.get_aws_yaml(yaml_file="elb")
V = Vpc()
for lb in elbs:
tmp = lb['LoadBalancerName'].split('-')
if len(tmp) >= 3:
elb_env = tmp.pop(0)
short_env = tmp.pop(0)
elb_stack = "-".join(tmp)
elb_facing = lb['Scheme']
if elb_stack in yaml and elb_facing in yaml[elb_stack]:
yaml_info = yaml[elb_stack][elb_facing]
v = V.get_vpc_from_env(env=elb_env)
domain = Misc.get_value_from_array_hash(dictlist=v.get('Tags'), key="Domain")
if elb_facing == "internet-facing":
elb_dns_name = yaml_info['dns'] + "." + Misc.change_domain_to_local(domain=domain)
elif elb_facing == "internal":
elb_dns_name = yaml_info['dns'] + "." + Misc.change_domain_to_local(domain=domain)
else:
elb_dns_name = None
info = {}
if elb_dns_name is not None:
info['DNS cname'] = elb_dns_name
else:
info['DNS cname'] = "This elb is not in automatisation framework. Will be decomissioned"
info['Xively_service'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Xively_service")
info['Puppet_role'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Puppet_role")
info['Env'] = Misc.get_value_from_array_hash(dictlist=lb.get('Tags'), key="Environment")
info['Real endpoint'] = lb['DNSName']
info['Vpcid'] = lb['VPCId']
info['Name'] = lb['LoadBalancerName']
info['CreateTime'] = lb['CreatedTime'].strftime("%Y-%m-%d %H:%M")
info['Facing'] = elb_facing
info['Availability Zones'] = lb['AvailabilityZones']
info['Securitygroups'] = lb['SecurityGroups']
instance = []
for i in lb['Instances']:
instance.append(i['InstanceId'])
info['InstanceIds'] = instance
listen = []
for listener in lb['ListenerDescriptions']:
listener = listener['Listener']
listen.append(
"%s-%s-%s" % (listener['LoadBalancerPort'], listener['InstancePort'], listener['Protocol']))
info['From-To-Protocol'] = listen
result.append(info)
return result
def get_subnets_for_elb(self, scheme=None, env=None):
vpc = Vpc()
v = vpc.get_vpc_from_env(env=env)
azs = Misc.get_azs_from_yaml(region=self.region)
if scheme == "internal":
avail = "private"
else:
avail = "public"
res = []
sub = vpc.get_all_subnets(
filters=[{'Name': 'tag:Availability', 'Values': [avail]}, {'Name': 'availabilityZone', 'Values': azs},
{'Name': 'vpc-id', 'Values': [v.get('VpcId')]}])
for s in sub:
logger.debug("Adding sub: %s" % sub, )
res.append(s.get('SubnetId'))
logger.debug("Subnets for elb are: %s" % res, )
return res
def deploy_stack_to_env(self, env, file, dryrun):
stack_json = Misc.parse_file_to_json(file=file)
from misc import Validator
stack_json = Validator.validate_kerrigan_json(stack_data=stack_json,
env=env)
if 'cloudformation' in stack_json:
cloudformation_json = stack_json.pop("cloudformation")
from misc import Translater
from wrapper.vpc import Vpc
from wrapper.cloudformation import Cloudformation
vpc = Vpc(session=self.session)
cloudformation = Cloudformation(session=self.session)
env_vpc = vpc.get_vpc_from_env(env=env)
env_cidr = env_vpc['CidrBlock']
ami = self.get_ami_from_tag(puppet_role=stack_json['ami'])
else:
cloudformation_json = None
self.upload_stack_to_dynamodb(item=stack_json)
# Do changes from kerrigan
if cloudformation_json:
logger.info(msg="Doing security group transformations")
cloudformation_json = Translater.translate_security_group_ip_address_in_cloudformation(
cloudformation_json=cloudformation_json, env_cidr=env_cidr)
cloudformation_parameters = Translater.gather_information_for_cloudofrmation_parameters(
stack_data=stack_json, vpc=env_vpc, ami=ami)
# Do pre kerrigan tasks
# Do cloudformation
if cloudformation_json:
stackname = "%s-%s-%s" % (env, stack_json['puppet_role'],
stack_json['xively_service'])
if cloudformation.stack_exists(stackname=stackname):
cloudformation.update_stack(
stackname=stackname,
templatebody=cloudformation_json,
dryrun=dryrun,
parameters=cloudformation_parameters)
else:
cloudformation.create_stack(
stackname=stackname,
templatebody=cloudformation_json,
dryrun=dryrun,
parameters=cloudformation_parameters)
def envs_with_domains(self):
v = Vpc()
envs = v.get_active_envs()
res = {}
for env in envs:
res[env] = []
vpc = v.get_vpc_from_env(env=env)
domain = Misc.get_value_from_array_hash(dictlist=vpc.get('Tags'),
key='Domain')
res[env].append(domain)
logger.debug('Working on env %s and domain %s' % (env, domain))
split_domain = Misc.string_to_array(string=domain)
last_dn = split_domain.pop()
# local entry
if env == "prod":
local_tmp = ["prod"] + split_domain + ['local']
else:
local_tmp = split_domain + ['local']
res[env].append(
Misc.join_list_to_string(list=local_tmp, join_with='.'))
return res
def create_launch_config(self,
launch_config_name=None,
env=None,
xively_service=None,
stack=None):
e = Ec2()
v = Vpc()
vpc = v.get_vpc_from_env(env=env)
keyname = Misc.get_value_from_array_hash(dictlist=vpc['Tags'],
key='Keypair')
baseami = e.query_base_image(stack=stack)
ostype = Misc.get_value_from_array_hash(dictlist=baseami['Tags'],
key='Os')
instance_type = Misc.get_value_from_array_hash(
dictlist=baseami['Tags'], key='Instancetype')
image = baseami.get('ImageId')
sgs = e.get_security_group_ids_for_launch(
vpcid=vpc.get('VpcId'),
stack=stack,
ostype=ostype,
xively_service=xively_service)
iam = "ec2"
y = Misc.get_app_ports_yaml('app_ports')
port = y[xively_service]
userdata = Misc.get_autoscale_userdata_for_os(ostype=ostype).format(
env=env, stack=stack, xively_service=xively_service, port=port)
monitoring = {}
monitoring['Enabled'] = True
self.autoscale.create_launch_configuration(
LaunchConfigurationName=launch_config_name,
ImageId=image,
KeyName=keyname,
SecurityGroups=sgs,
UserData=userdata,
InstanceType=instance_type,
InstanceMonitoring=monitoring,
IamInstanceProfile=iam)
def get_vpcid_from_env(self, env=None):
v = Vpc()
vpc = v.get_vpc_from_env(env=env)
return vpc.get('VpcId')
def launch_auto_scaling_group(self,
env=None,
stack=None,
min_size=None,
max_size=None,
xively_service=None,
requester=None,
load_balancer_name=None,
health_check=None,
health_check_grace_period=None,
availability=None,
customer=None):
a = AutoScaling()
v = Vpc()
c = CloudWatch()
logger.info("Starting creation of auto-scaling group")
auto_scaling_group_name = a.generate_auto_scaling_group_name(
env=env, stack=stack, xively_service=xively_service)
launch_config_name = a.generate_launch_config_name(
env=env, stack=stack, xively_service=xively_service)
lc_exists = a.check_launch_config_exists(env=env,
xively_service=xively_service,
stack=stack)
if lc_exists is False:
logger.info("Starting to Create Launch Configuration: %s" %
launch_config_name)
a.create_launch_config(launch_config_name=launch_config_name,
env=env,
xively_service=xively_service,
stack=stack)
else:
logger.info("Launch Configuration %s Already Exists" %
launch_config_name)
vpc = v.get_vpc_from_env(env=env)
subnet_filter = v.get_all_subnets(
filters=[{
"Name": "tag:Availability",
"Values": [
availability,
]
}, {
"Name": "vpc-id",
"Values": [
vpc.get('VpcId'),
]
}])
vpc_zones = ""
for s in subnet_filter:
vpc_zones += str(s['SubnetId'])
vpc_zones += str(",")
tags = [{
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": False,
"Key": "Name",
"Value": auto_scaling_group_name
}, {
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": True,
"Key": "Requester",
"Value": requester
}, {
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": True,
"Key": "Puppet_role",
"Value": stack
}, {
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": True,
"Key": "Xively_service",
"Value": xively_service
}, {
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": True,
"Key": "Environment",
"Value": env
}, {
"ResourceId": auto_scaling_group_name,
"ResourceType": "auto-scaling-group",
"PropagateAtLaunch": True,
"Key": "Customer",
"Value": customer
}]
asg = []
asg.append(auto_scaling_group_name)
asg_exists = a.check_auto_scaling_group_exists(
auto_scaling_group_name=asg)
if asg_exists is False:
logger.info("Starting to Create Auto Scaling Group: %s" %
launch_config_name)
a.run_auto_scaling_group(
auto_scaling_group_name=auto_scaling_group_name,
min_size=min_size,
max_size=max_size,
launch_config_name=launch_config_name,
load_balancer_name=load_balancer_name,
health_check=health_check,
health_check_grace_period=health_check_grace_period,
vpc_zones=vpc_zones,
tags=tags)
# resp = a.get_status_auto_scaling_group(auto_scaling_group_name=auto_scaling_group_name)
# logger.info(resp)
else:
logger.info("Auto Scaling Group %s Already Exists" %
launch_config_name)
a.create_scaling_policy(env=env,
stack=stack,
xively_service=xively_service)
c.create_alarm_for_auto_scaling_group(env=env,
stack=stack,
xively_service=xively_service)
def prepare_deployment(self, puppet_role, xively_service, env, num,
instance_type, base_ami, iam, requester, customer,
dry_run):
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
vpc_obj = vpc.get_vpc_from_env(env=env)
filter_base = [{'Name': 'vpc-id', 'Values': [vpc_obj.get('VpcId')]}]
if xively_service:
old_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': ["%s_old" % xively_service]
}])
rollback_machines = ec2.get_ec2_instances(
filters=filter_base +
[{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': ["%s_rollback" % xively_service]
}])
current_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}, {
'Name': 'tag:Xively_service',
'Values': [xively_service]
}])
else:
old_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': ["%s_old" % puppet_role]
}])
rollback_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': ["%s_rollback" % puppet_role]
}])
current_machines = ec2.get_ec2_instances(
filters=filter_base + [{
'Name': 'tag:Puppet_role',
'Values': [puppet_role]
}])
for old_machine in old_machines:
logger.info(msg="Going to stop old machine %s" %
old_machine.get('InstanceId'))
ec2.stop_instances(dryrun=dry_run,
instanceids=[old_machine.get('InstanceId')])
for rollback_machine in rollback_machines:
logger.info(msg="Going to stop old machine %s" %
rollback_machine.get('InstanceId'))
ec2.stop_instances(
dryrun=dry_run,
instanceids=[rollback_machine.get('InstanceId')])
if xively_service:
ec2.tag_resource(
instanceid=rollback_machine.get('InstanceId'),
tags={'Xively_service': "%s_old" % xively_service})
else:
ec2.tag_resource(instanceid=rollback_machine.get('InstanceId'),
tags={'Puppet_role': "%s_old" % puppet_role})
for current_machine in current_machines:
logger.info(msg="Going to retag current machine %s" %
current_machine.get('InstanceId'))
if xively_service:
ec2.tag_resource(
instanceid=current_machine.get('InstanceId'),
tags={'Xively_service': "%s_rollback" % xively_service})
else:
ec2.tag_resource(
instanceid=current_machine.get('InstanceId'),
tags={'Puppet_role': "%s_rollback" % puppet_role})
def deploy_snappy(self, env, num, dryrun, accountid, newrelic, channelname,
devicestring, branch):
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
vpc_obj = vpc.get_vpc_from_env(env=env)
num = int(num)
snappyindex = self.get_snappy_index(num=num,
vpcid=vpc_obj.get('VpcId'))
lambda_function_args = {
'env': "infra",
'puppet_role': 'benchmarkslave',
'requester': "benchmark",
'xively_service': "benchmark_slave",
'customer': "",
'shutdown': "stop",
'dry_run': dryrun,
'base_ami': "benchmarkslave",
'instance_type': 'c3.xlarge',
'snappyindex': snappyindex,
'accountid': accountid,
'channelname': channelname,
'newrelic': newrelic,
'iam': 'infra-benchmarkslave',
'ebsoptimized': False,
'monitoring': False,
'devicestring': devicestring,
'branch': branch
}
lambda_function_args['userdata'] = Misc.get_userdata_for_os(
ostype="snappy")
baseami_object = self.get_ami_from_tag(
puppet_role=lambda_function_args['base_ami'])
lambda_function_args['baseamiid'] = baseami_object.get('ImageId')
availability = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Availability')
lambda_function_args['ostype'] = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Os')
lambda_function_args['keypair'] = Misc.get_value_from_array_hash(
dictlist=vpc_obj.get('Tags'), key='Keypair')
lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name(
env=env, puppet_role="benchmarkslave", num=num)
lambda_function_args[
'securitygroup'] = ec2.get_security_group_ids_for_stack(
vpcid=vpc_obj.get('VpcId'),
puppet_role="benchmarkslave",
ostype=lambda_function_args['ostype'],
xively_service="benchmark_slave")
subnets = vpc.get_all_subnets(filters=[{
'Name': 'tag:Network',
'Values': [availability]
}, {
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}])
lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym(
puppet_role="benchmarkslave",
subnets=subnets,
num=num,
fillup=False,
xively_service="benchmark_slave")
## Get broker IP address
broker = ec2.get_ec2_instances(
filters=[{
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}, {
'Name': 'tag:Xively_service',
'Values': ['benchmark_master']
}, {
'Name': 'tag:Puppet_role',
'Values': ['linuxbase']
}])
lambda_function_args['broker'] = broker[0].get(
'PrivateIpAddress') + ":8883"
instances = Misc.parallel_map_reduce(
lambda x: self.create_instance_lamdba(args=lambda_function_args),
lambda x, y: x + [y], xrange(0, num), [])
return instances
def create_ec2_instance(self,
puppet_role,
env,
requester,
customer,
xively_service,
base_ami,
iam,
instance_type,
dry_run,
shutdown,
monitoring,
fillup,
num,
keypair,
availability=None):
"""
This function creates an ec2 instance
:param puppet_role: the Puppet_role that should be used
:param env: the environment where we should provision to
:param requester: the user/team requesting the machine
:param customer: For future use only
:param xively_service: the Xively_service that should be used
:param base_ami: the base_ami that should be used. Can default to puppet_role
:param iam: The iam role that should be attached, defaults to ec2-base
:param instance_type: the type of instance requested
:param dry_run: No changes should be done
:param shutdown: The shutdown behavior to use
:param monitoring: Should monitoring be enabled
:param fillup: Should fillup algorithym be used or round robin
:param num: the number of instances
:return: a list of instance objects
"""
from wrapper.ec2 import Ec2
from wrapper.vpc import Vpc
from misc import Misc
from core.stackdata import stackdata
stackdata_object = stackdata(session=self.session)
ec2 = Ec2(session=self.session)
vpc = Vpc(session=self.session)
lambda_function_args = {
'env': env,
'puppet_role': puppet_role,
'requester': requester,
'xively_service': xively_service,
'customer': customer,
'shutdown': shutdown,
'dry_run': dry_run
}
stack_data = stackdata_object.get_stack_data(
puppet_role=puppet_role, xively_service=xively_service)
vpc_obj = vpc.get_vpc_from_env(env=env)
## Find the baseami object that needs to be used
if base_ami:
base_ami = base_ami
elif 'ami' in stack_data:
base_ami = stack_data['ami']
else:
logger.info("Falling back to puppet_role as AMI name")
base_ami = puppet_role
logger.info("The baseami that is going to be used: %s" % (base_ami, ))
baseami_object = self.get_ami_from_tag(puppet_role=base_ami)
## Get values for lambda function
lambda_function_args['baseamiid'] = baseami_object.get('ImageId')
if (availability == None):
availability = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Availability')
lambda_function_args['ostype'] = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Os')
if keypair is not None:
lambda_function_args['keypair'] = keypair
else:
lambda_function_args['keypair'] = Misc.get_value_from_array_hash(
dictlist=vpc_obj.get('Tags'), key='Keypair')
## Find the instance_type that needs to be used
if instance_type:
inst_type_final = instance_type
elif 'instance_type' in stack_data and env in stack_data[
'instance_type']:
inst_type_final = stack_data['instance_type'][env]
else:
inst_type_final = Misc.get_value_from_array_hash(
dictlist=baseami_object.get('Tags'), key='Instancetype')
logger.info("Instance type that will be used: %s" %
(inst_type_final, ))
lambda_function_args['instance_type'] = inst_type_final
## Find the instance profile that needs to be used
if iam:
iam_name = iam
elif 'iam' in stack_data:
iam_name = "%s-%s" % (env, stack_data['iam']['name_postfix'])
else:
iam_name = "ec2-base"
logger.info("Base iam instance profile name: %s" % (iam_name, ))
lambda_function_args['iam'] = iam_name
## Find value for ebsoptimized
if 'ebsoptimized' in stack_data and env in stack_data['ebsoptimized']:
lambda_function_args['ebsoptimized'] = Misc.str2bool(
stack_data['ebsoptimized'][env])
else:
lambda_function_args['ebsoptimized'] = False
## Find value for monitoring enablement
if monitoring:
lambda_function_args['monitoring'] = monitoring
elif 'monitoring' in stack_data and env in stack_data['monitoring']:
lambda_function_args['monitoring'] = Misc.str2bool(
stack_data['monitoring'][env])
else:
lambda_function_args['monitoring'] = False
## Generate instance names for all required instances
lambda_function_args['instance_name'] = ec2.generate_ec2_unique_name(
env=env, puppet_role=puppet_role, num=num)
## Gather all security groups needed for creating an instance stack
lambda_function_args[
'securitygroup'] = ec2.get_security_group_ids_for_stack(
vpcid=vpc_obj.get('VpcId'),
puppet_role=puppet_role,
ostype=lambda_function_args['ostype'],
xively_service=xively_service)
# We need to retrieve the subnets from Vpc object, and pass it to Ec2 object
subnets = vpc.get_all_subnets(filters=[{
'Name': 'tag:Network',
'Values': [availability]
}, {
'Name': 'vpc-id',
'Values': [vpc_obj.get('VpcId')]
}])
lambda_function_args['subnet'] = ec2.get_subnet_with_algorithym(
puppet_role=puppet_role,
subnets=subnets,
num=num,
fillup=fillup,
xively_service=xively_service)
instances = Misc.parallel_map_reduce(
lambda x: self.create_instance_lamdba(args=lambda_function_args),
lambda x, y: x + [y], xrange(0, num), [])
return instances