def _run_command(args):
# Always autoscale!
args.autoscale = True
if args.launch and args.autoscale:
print "Error: Can't use --launch and --autoscale together."
sys.exit(1)
name = get_app_name()
tier_name = get_tier_name()
conf = get_drift_config(tier_name=tier_name,
deployable_name=name,
drift_app=load_flask_config())
aws_region = conf.tier['aws']['region']
print "AWS REGION:", aws_region
print "DOMAIN:\n", json.dumps(conf.domain.get(), indent=4)
print "DEPLOYABLE:\n", json.dumps(conf.deployable, indent=4)
ec2_conn = boto.ec2.connect_to_region(aws_region)
iam_conn = boto.iam.connect_to_region(aws_region)
if conf.tier['is_live']:
print "NOTE! This tier is marked as LIVE. Special restrictions may apply. Use --force to override."
autoscaling = {
"min": 1,
"max": 1,
"desired": 1,
"instance_type": args.instance_type,
}
autoscaling.update(conf.deployable.get('autoscaling', {}))
release = conf.deployable.get('release', '')
if args.launch and autoscaling and not args.force:
print "--launch specified, but tier config specifies 'use_autoscaling'. Use --force to override."
sys.exit(1)
if args.autoscale and not autoscaling and not args.force:
print "--autoscale specified, but tier config doesn't specify 'use_autoscaling'. Use --force to override."
sys.exit(1)
print "Launch an instance of '{}' on tier '{}'".format(name, tier_name)
if release:
print "Using AMI with release tag: ", release
else:
print "Using the newest AMI baked (which may not be what you expect)."
ami = _find_latest_ami(name, release)
print "Latest AMI:", ami
if args.ami:
print "Using a specified AMI:", args.ami
ec2 = boto3.resource('ec2', region_name=aws_region)
if ami.id != args.ami:
print "AMI found is different from AMI specified on command line."
if conf.tier['is_live'] and not args.force:
print "This is a live tier. Can't run mismatched AMI unless --force is specified"
sys.exit(1)
try:
ami = ec2.Image(args.ami)
except Exception as e:
raise RuntimeError("Ami '%s' not found or broken: %s" %
(args.ami, e))
if not ami:
sys.exit(1)
ami_info = dict(
ami_id=ami.id,
ami_name=ami.name,
ami_created=ami.creation_date,
ami_tags={d['Key']: d['Value']
for d in ami.tags},
)
print "AMI Info:\n", pretty(ami_info)
if autoscaling:
print "Autoscaling group:\n", pretty(autoscaling)
else:
print "EC2:"
print "\tInstance Type:\t{}".format(args.instance_type)
ec2 = boto3.resource('ec2', region_name=aws_region)
# Get all 'private' subnets
filters = {'tag:tier': tier_name, 'tag:realm': 'private'}
subnets = list(ec2.subnets.filter(Filters=filterize(filters)))
if not subnets:
print "Error: No subnet available matching filter", filters
sys.exit(1)
print "Subnets:"
for subnet in subnets:
print "\t{} - {}".format(fold_tags(subnet.tags)['Name'], subnet.id)
# Get the "one size fits all" security group
filters = {
'tag:tier': tier_name,
'tag:Name': '{}-private-sg'.format(tier_name)
}
security_group = list(
ec2.security_groups.filter(Filters=filterize(filters)))[0]
print "Security Group:\n\t{} [{} {}]".format(
fold_tags(security_group.tags)["Name"], security_group.id,
security_group.vpc_id)
# The key pair name for SSH
key_name = conf.tier['aws']['ssh_key']
if "." in key_name:
key_name = key_name.split(
".",
1)[0] # TODO: Distinguish between key name and .pem key file name
print "SSH Key:\t", key_name
'''
autoscaling group:
Name LIVENORTH-themachines-backend-auto
api-port 10080
api-target themachines-backend
service-name themachines-backend
service-type rest-api
tier LIVENORTH
ec2:
Name DEVNORTH-drift-base
launched-by nonnib
api-port 10080
api-target drift-base
service-name drift-base
service-type rest-api
tier DEVNORTH
'''
target_name = "{}-{}".format(tier_name, name)
if autoscaling:
target_name += "-auto"
# To auto-generate Redis cache url, we create the Redis backend using our config,
# and then ask for a url representation of it:
drift_config_url = get_redis_cache_backend(conf.table_store,
tier_name).get_url()
# Specify the app
app_root = '/etc/opt/{service_name}'.format(service_name=name)
tags = {
"Name": target_name,
"tier": tier_name,
"service-name": name,
"service-type": conf.drift_app.get('service_type', 'web-app'),
"config-url": drift_config_url,
"app-root": app_root,
"launched-by": iam_conn.get_user().user_name,
}
if tags['service-type'] == 'web-app':
# Make instance part of api-router round-robin load balancing
tags.update({
"api-target": name,
"api-port": str(conf.drift_app.get('PORT', 10080)),
"api-status": "online",
})
tags.update(fold_tags(ami.tags))
print "Tags:"
for k in sorted(tags.keys()):
print " %s: %s" % (k, tags[k])
user_data = '''#!/bin/bash
# Environment variables set by drift-admin run command:
export DRIFT_CONFIG_URL={drift_config_url}
export DRIFT_TIER={tier_name}
export DRIFT_APP_ROOT={app_root}
export DRIFT_SERVICE={service_name}
export AWS_REGION={aws_region}
# Shell script from ami-run.sh:
'''.format(drift_config_url=drift_config_url,
tier_name=tier_name,
app_root=app_root,
service_name=name,
aws_region=aws_region)
user_data += pkg_resources.resource_string(__name__, "ami-run.sh")
custom_script_name = os.path.join(conf.drift_app['app_root'], 'scripts',
'ami-run.sh')
if os.path.exists(custom_script_name):
print "Using custom shell script", custom_script_name
user_data += "\n# Custom shell script from {}\n".format(
custom_script_name)
user_data += open(custom_script_name, 'r').read()
else:
print "Note: No custom ami-run.sh script found for this application."
print "user_data:"
from drift.utils import pretty as poo
print poo(user_data, 'bash')
if args.preview:
print "--preview specified, exiting now before actually doing anything."
sys.exit(0)
if autoscaling:
client = boto3.client('autoscaling', region_name=aws_region)
launch_config_name = '{}-{}-launchconfig-{}-{}'.format(
tier_name, name, datetime.utcnow(), release)
launch_config_name = launch_config_name.replace(':', '.')
kwargs = dict(
LaunchConfigurationName=launch_config_name,
ImageId=ami.id,
KeyName=key_name,
SecurityGroups=[security_group.id],
InstanceType=autoscaling['instance_type'] or args.instance_type,
IamInstanceProfile=IAM_ROLE,
InstanceMonitoring={'Enabled': True},
UserData=user_data,
)
print "Creating launch configuration using params:\n", pretty(kwargs)
client.create_launch_configuration(**kwargs)
# Update current autoscaling group or create a new one if it doesn't exist.
groups = client.describe_auto_scaling_groups(
AutoScalingGroupNames=[target_name])
kwargs = dict(
AutoScalingGroupName=target_name,
LaunchConfigurationName=launch_config_name,
MinSize=autoscaling['min'],
MaxSize=autoscaling['max'],
DesiredCapacity=autoscaling['desired'],
VPCZoneIdentifier=','.join([subnet.id for subnet in subnets]),
)
if not groups['AutoScalingGroups']:
print "Creating a new autoscaling group using params:\n", pretty(
kwargs)
client.create_auto_scaling_group(**kwargs)
else:
print "Updating current autoscaling group", target_name
client.update_auto_scaling_group(**kwargs)
# Prepare tags which get propagated to all new instances
tagsarg = [{
'ResourceId': tags['Name'],
'ResourceType': 'auto-scaling-group',
'Key': k,
'Value': v,
'PropagateAtLaunch': True,
} for k, v in tags.items()]
print "Updating tags on autoscaling group that get propagated to all new instances."
client.create_or_update_tags(Tags=tagsarg)
# Define a 2 min termination cooldown so api-router can drain the connections.
response = client.put_lifecycle_hook(
LifecycleHookName='Wait-2-minutes-on-termination',
AutoScalingGroupName=target_name,
LifecycleTransition='autoscaling:EC2_INSTANCE_TERMINATING',
HeartbeatTimeout=120,
DefaultResult='CONTINUE')
print "Configuring lifecycle hook, response:", response.get(
'ResponseMetadata')
print "Done!"
print "YOU MUST TERMINATE THE OLD EC2 INSTANCES YOURSELF!"
else:
# Pick a random subnet from list of available subnets
subnet = random.choice(subnets)
print "Randomly picked this subnet to use: ", subnet
print "Launching EC2 instance..."
reservation = ec2_conn.run_instances(
ami.id,
instance_type=args.instance_type,
subnet_id=subnet.id,
security_group_ids=[security_group.id],
key_name=key_name,
instance_profile_name=IAM_ROLE,
user_data=user_data,
)
if len(reservation.instances) == 0:
print "No instances in reservation!"
sys.exit(1)
instance = reservation.instances[0]
print "{} starting up...".format(instance)
# Check up on its status every so often
status = instance.update()
while status == 'pending':
time.sleep(10)
status = instance.update()
if status == 'running':
for k, v in tags.items():
instance.add_tag(k, v)
print "{} running at {}".format(instance,
instance.private_ip_address)
slackbot.post_message(
"Started up AMI '{}' for '{}' on tier '{}' with ip '{}'".
format(ami.id, name, tier_name, instance.private_ip_address))
else:
print "Instance was not created correctly"
sys.exit(1)
def run_command(args):
service_info = get_service_info()
tier_config = get_tier_config()
ec2_conn = boto.ec2.connect_to_region(tier_config["region"])
vpc_conn = boto.vpc.connect_to_region(tier_config["region"])
iam_conn = boto.iam.connect_to_region(tier_config["region"])
tier_name = tier_config["tier"].upper() # Canonical name of tier
print "Launch an instance of '{}' on tier '{}'".format(
service_info["name"], tier_config["tier"])
for deployable in tier_config["deployables"]:
if deployable["name"] == service_info["name"]:
break
else:
print "Error: Deployable '{}' not found in tier config:".format(
service_info["name"])
print json.dumps(tier_config, indent=4)
sys.exit(1)
if args.ami is None:
# Pick the most recent image baked by the caller
print "No source AMI specified. See if your organization has baked one recently..."
print "Searching AMI's with the following tags:"
print " service-name:", service_info["name"]
print " tier:", tier_name
amis = ec2_conn.get_all_images(
owners=['self'], # The current organization
filters={
'tag:service-name': service_info["name"],
'tag:tier': tier_name,
},
)
if not amis:
print "No AMI's found that match this service and tier."
print "Bake a new one using this command: {} bakeami".format(
sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creationDate"))
print "{} AMI(s) found.".format(len(amis))
else:
ami = ec2_conn.get_image(args.ami)
print "AMI Info:"
print "\tAMI ID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creationDate
import pprint
print "\tTags:\t"
for k, v in ami.tags.items():
print "\t\t", k, ":", v
print "EC2:"
print "\tInstance Type:\t{}".format(args.instance_type)
# Find the appropriate subnet to run on.
# TODO: The subnet should be tagged more appropriately. For now we deploy
# all drift apps to private-subnet-2, and keep special purpose services on
# private-subnet-1, like RabbitMQ.
for subnet in vpc_conn.get_all_subnets():
tier_match = subnet.tags.get("tier", "").upper() == tier_name
name_match = "private-subnet-2" in subnet.tags.get("Name", "").lower()
if tier_match and name_match:
break
else:
print "Can't find a subnet to run on."
sys.exit(1)
print "\tSubnet:\t{} [{} {}]".format(subnet.tags["Name"], subnet.id,
subnet.vpc_id)
print "\tCIDR:\t", subnet.cidr_block
# Find the appropriate security group.
# TODO: For now we just have a "one size fits all" group which allows all
# traffic from 10.x.x.x. This security group was created manually but needs
# to be added to the tier provisioning script.
for security_group in vpc_conn.get_all_security_groups():
tier_match = security_group.tags.get("tier", "").upper() == tier_name
name_match = "private-sg" in security_group.tags.get("Name",
"").lower()
vpc_match = security_group.vpc_id == subnet.vpc_id
if tier_match and name_match and vpc_match:
break
else:
print "Can't find a security group to run on."
sys.exit(1)
print "\tSecurity Group: {} [{} {}]".format(security_group.tags["Name"],
security_group.id,
security_group.vpc_id)
# The key pair name for SSH
key_name = deployable["ssh_key"]
if "." in key_name:
key_name = key_name.split(
".",
1)[0] # TODO: Distinguish between key name and .pem key file name
print "\tSSH Key:\t", key_name
tags = {
"Name": "{}-{}".format(tier_name, service_info["name"]),
"tier": tier_name,
"service-name": service_info["name"],
"launched-by": iam_conn.get_user().user_name,
# Make instance part of api-router round-robin load balancing
"api-target": service_info["name"],
"api-port": "10080",
}
print "Tags:"
print json.dumps(tags, indent=4)
reservation = ec2_conn.run_instances(
ami.id,
instance_type=args.instance_type,
subnet_id=subnet.id,
security_group_ids=[security_group.id],
key_name=key_name,
instance_profile_name=IAM_ROLE)
if len(reservation.instances) == 0:
print "No instances in reservation!"
sys.exit(1)
instance = reservation.instances[0]
print "{} starting up...".format(instance)
# Check up on its status every so often
status = instance.update()
while status == 'pending':
time.sleep(10)
status = instance.update()
if status == 'running':
for k, v in tags.items():
instance.add_tag(k, v)
print "{} running at {}".format(instance, instance.private_ip_address)
slackbot.post_message(
"Started up AMI '{}' for '{}' on tier '{}' with ip '{}'".format(
ami.id, service_info["name"], tier_config["tier"],
instance.private_ip_address))
else:
print "Instance was not created correctly"
sys.exit(1)
def _bake_command(args):
if args.ubuntu:
name = UBUNTU_BASE_IMAGE_NAME
else:
name = get_app_name()
name = get_app_name()
tier_name = get_tier_name()
conf = get_drift_config(tier_name=tier_name,
deployable_name=name,
drift_app=load_flask_config())
domain = conf.domain.get()
aws_region = domain['aws']['ami_baking_region']
ec2 = boto3.resource('ec2', region_name=aws_region)
print "DOMAIN:\n", json.dumps(domain, indent=4)
if not args.ubuntu:
print "DEPLOYABLE:", name
print "AWS REGION:", aws_region
# Create a list of all regions that are active
if args.ubuntu:
# Get all Ubuntu images from the appropriate region and pick the most recent one.
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
print "Finding the latest AMI on AWS that matches", UBUNTU_RELEASE
filters = [
{
'Name': 'name',
'Values': [UBUNTU_RELEASE]
},
]
amis = list(
ec2.images.filter(Owners=[AMI_OWNER_CANONICAL], Filters=filters))
if not amis:
print "No AMI found matching '{}'. Not sure what to do now.".format(
UBUNTU_RELEASE)
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
else:
filters = [
{
'Name': 'tag:service-name',
'Values': [UBUNTU_BASE_IMAGE_NAME]
},
{
'Name': 'tag:domain-name',
'Values': [domain['domain_name']]
},
]
amis = list(ec2.images.filter(Owners=['self'], Filters=filters))
if not amis:
criteria = {d['Name']: d['Values'][0] for d in filters}
print "No '{}' AMI found using the search criteria {}.".format(
UBUNTU_BASE_IMAGE_NAME, criteria)
print "Bake one using this command: {} ami bake --ubuntu".format(
sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creation_date
if args.ubuntu:
manifest = None
packer_vars = {
'setup_script':
pkg_resources.resource_filename(__name__, "ubuntu-packer.sh"),
'ubuntu_release':
UBUNTU_RELEASE,
}
else:
current_branch = get_branch()
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
# Wrap git branch modification in RAII.
checkout(args.tag)
try:
setup_script = ""
setup_script_custom = ""
with open(
pkg_resources.resource_filename(__name__,
"driftapp-packer.sh"),
'r') as f:
setup_script = f.read()
custom_script_name = os.path.join(conf.drift_app['app_root'],
'scripts', 'ami-bake.sh')
if os.path.exists(custom_script_name):
print "Using custom bake shell script", custom_script_name
setup_script_custom = "echo Executing custom bake shell script from {}\n".format(
custom_script_name)
setup_script_custom += open(custom_script_name, 'r').read()
setup_script_custom += "\necho Custom bake shell script completed\n"
else:
print "Note: No custom ami-bake.sh script found for this application."
# custom setup needs to happen first because we might be installing some requirements for the regular setup
setup_script = setup_script_custom + setup_script
tf = tempfile.NamedTemporaryFile(delete=False)
tf.write(setup_script)
tf.close()
setup_script_filename = tf.name
manifest = create_deployment_manifest('ami', comment=None)
packer_vars = {
'version': get_app_version(),
'setup_script': setup_script_filename,
}
if not args.preview:
cmd = ['python', 'setup.py', 'sdist', '--formats=zip']
ret = subprocess.call(cmd)
if ret != 0:
print "Failed to execute build command:", cmd
sys.exit(ret)
cmd = ["zip", "-r", "dist/aws.zip", "aws"]
ret = subprocess.call(cmd)
if ret != 0:
print "Failed to execute build command:", cmd
sys.exit(ret)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
user = boto.iam.connect_to_region(
aws_region).get_user() # The current IAM user running this command
packer_vars.update({
"service": name,
"region": aws_region,
"source_ami": ami.id,
"user_name": user.user_name,
"domain_name": domain['domain_name'],
})
print "Packer variables:\n", pretty(packer_vars)
# See if Packer is installed and generate sensible error code if something is off.
# This will also write the Packer version to the terminal which is useful info.
try:
subprocess.call(['packer', 'version'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "'packer version' command failed. Please install it if it's missing."
sys.exit(127)
cmd = "packer build "
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in packer_vars.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
start_time = time.time()
try:
# Execute Packer command and parse the output to find the ami id.
p = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while True:
line = p.stdout.readline()
print line,
if line == '' and p.poll() is not None:
break
# The last lines from the packer execution look like this:
# ==> Builds finished. The artifacts of successful builds are:
# --> amazon-ebs: AMIs were created:
#
# eu-west-1: ami-0ee5eb68
if 'ami-' in line:
ami_id = line[line.rfind('ami-'):].strip()
ami = ec2.Image(ami_id)
print ""
print "AMI ID: %s" % ami.id
print ""
finally:
pkg_resources.cleanup_resources()
if p.returncode != 0:
print "Failed to execute packer command:", cmd
sys.exit(p.returncode)
duration = time.time() - start_time
if manifest:
print "Adding manifest tags to AMI:"
pretty(manifest)
prefix = "drift:manifest:"
tags = []
for k, v in manifest.iteritems():
tag_name = "{}{}".format(prefix, k)
tags.append({'Key': tag_name, 'Value': v or ''})
ami.create_tags(DryRun=False, Tags=tags)
if not args.skipcopy:
_copy_image(ami.id)
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' in %.0f seconds".format(
name, duration))
def _run_command(args):
if args.launch and args.autoscale:
print "Error: Can't use --launch and --autoscale together."
sys.exit(1)
service_info = get_service_info()
tier_config = get_tier_config()
ec2_conn = boto.ec2.connect_to_region(tier_config["region"])
iam_conn = boto.iam.connect_to_region(tier_config["region"])
tier_name = tier_config["tier"].upper() # Canonical name of tier
print "Launch an instance of '{}' on tier '{}'".format(
service_info["name"], tier_config["tier"])
if tier_config.get('is_live', True):
print "NOTE! This tier is marked as LIVE. Special restrictions may apply. Use --force to override."
for deployable in tier_config["deployables"]:
if deployable["name"] == service_info["name"]:
break
else:
print "Error: Deployable '{}' not found in tier config:".format(
service_info["name"])
print pretty(tier_config)
sys.exit(1)
print "Deployable:\n", pretty(deployable)
autoscaling = deployable.get('autoscaling')
release = deployable.get('release', '')
if args.launch and autoscaling and not args.force:
print "--launch specified, but tier config specifies 'use_autoscaling'. Use --force to override."
sys.exit(1)
if args.autoscale and not autoscaling and not args.force:
print "--autoscale specified, but tier config doesn't specify 'use_autoscaling'. Use --force to override."
sys.exit(1)
if args.autoscale and not autoscaling:
# Fill using default params
autoscaling = {
"min": 1,
"max": 2,
"desired": 2,
"instance_type": args.instance_type,
}
# Find AMI
filters = {
'tag:service-name': service_info["name"],
'tag:tier': tier_name,
}
if release:
filters['tag:release'] = release
print "Searching for AMIs matching the following tags:\n", pretty(filters)
amis = ec2_conn.get_all_images(
owners=['self'], # The current organization
filters=filters,
)
if not amis:
print "No AMI's found that match the tags."
print "Bake a new one using this command: {} ami bake {}".format(
sys.argv[0], release)
ami = None
else:
print "{} AMI(s) found.".format(len(amis))
ami = max(amis, key=operator.attrgetter("creationDate"))
if args.ami:
print "Using a specified AMI:", args.ami
if ami.id != args.ami:
print "AMI found is different from AMI specified on command line."
if tier_config.get('is_live', True) and not args.force:
print "This is a live tier. Can't run mismatched AMI unless --force is specified"
sys.exit(1)
ami = ec2_conn.get_image(args.ami)
if not ami:
sys.exit(1)
ami_info = dict(
ami_id=ami.id,
ami_name=ami.name,
ami_created=ami.creationDate,
ami_tags=ami.tags,
)
print "AMI Info:\n", pretty(ami_info)
if autoscaling:
print "Autoscaling group:\n", pretty(autoscaling)
else:
print "EC2:"
print "\tInstance Type:\t{}".format(args.instance_type)
ec2 = boto3.resource('ec2', region_name=tier_config["region"])
# Get all 'private' subnets
filters = {'tag:tier': tier_name, 'tag:realm': 'private'}
subnets = list(ec2.subnets.filter(Filters=filterize(filters)))
if not subnets:
print "Error: No subnet available matching filter", filters
sys.exit(1)
print "Subnets:"
for subnet in subnets:
print "\t{} - {}".format(fold_tags(subnet.tags)['Name'], subnet.id)
# Get the "one size fits all" security group
filters = {
'tag:tier': tier_name,
'tag:Name': '{}-private-sg'.format(tier_name)
}
security_group = list(
ec2.security_groups.filter(Filters=filterize(filters)))[0]
print "Security Group:\n\t{} [{} {}]".format(
fold_tags(security_group.tags)["Name"], security_group.id,
security_group.vpc_id)
# The key pair name for SSH
key_name = deployable["ssh_key"]
if "." in key_name:
key_name = key_name.split(
".",
1)[0] # TODO: Distinguish between key name and .pem key file name
print "SSH Key:\t", key_name
'''
autoscaling group:
Name LIVENORTH-themachines-backend-auto
api-port 10080
api-target themachines-backend
service-name themachines-backend
service-type rest-api
tier LIVENORTH
ec2:
Name DEVNORTH-drift-base
launched-by nonnib
api-port 10080
api-target drift-base
service-name drift-base
service-type rest-api
tier DEVNORTH
'''
target_name = "{}-{}".format(tier_name, service_info["name"])
if autoscaling:
target_name += "-auto"
tags = {
"Name": target_name,
"tier": tier_name,
"service-name": service_info["name"],
"service-type":
"rest-api", # TODO: Assume there are more types to come.
"launched-by": iam_conn.get_user().user_name,
# Make instance part of api-router round-robin load balancing
"api-target": service_info["name"],
"api-port": "10080",
"api-status": "online",
}
if args.preview:
print "--preview specified, exiting now before actually doing anything."
sys.exit(0)
if autoscaling:
client = boto3.client('autoscaling', region_name=tier_config["region"])
launch_config_name = '{}-{}-launchconfig-{}-{}'.format(
tier_name, service_info["name"], datetime.utcnow(), release)
launch_config_name = launch_config_name.replace(':', '.')
launch_script = '''#!/bin/bash\nsudo bash -c "echo TIERCONFIGPATH='${TIERCONFIGPATH}' >> /etc/environment"'''
kwargs = dict(
LaunchConfigurationName=launch_config_name,
ImageId=ami.id,
KeyName=key_name,
SecurityGroups=[security_group.id],
InstanceType=autoscaling['instance_type'] or args.instance_type,
IamInstanceProfile=IAM_ROLE,
InstanceMonitoring={'Enabled': True},
UserData=launch_script,
)
print "Creating launch configuration using params:\n", pretty(kwargs)
client.create_launch_configuration(**kwargs)
# Update current autoscaling group or create a new one if it doesn't exist.
groups = client.describe_auto_scaling_groups(
AutoScalingGroupNames=[target_name])
if not groups['AutoScalingGroups']:
tagsarg = [{
'ResourceId': tags['Name'],
'ResourceType': 'auto-scaling-group',
'Key': k,
'Value': v,
'PropagateAtLaunch': True,
} for k, v in tags.items()]
kwargs = dict(
AutoScalingGroupName=target_name,
LaunchConfigurationName=launch_config_name,
MinSize=autoscaling['min'],
MaxSize=autoscaling['max'],
DesiredCapacity=autoscaling['desired'],
VPCZoneIdentifier=','.join([subnet.id for subnet in subnets]),
Tags=tagsarg,
)
print "Creating a new autoscaling group using params:\n", pretty(
kwargs)
client.create_auto_scaling_group(**kwargs)
else:
print "Updating current autoscaling group", target_name
kwargs = dict(
AutoScalingGroupName=target_name,
LaunchConfigurationName=launch_config_name,
MinSize=autoscaling['min'],
MaxSize=autoscaling['max'],
DesiredCapacity=autoscaling['desired'],
VPCZoneIdentifier=','.join([subnet.id for subnet in subnets]),
)
client.update_auto_scaling_group(**kwargs)
print "Done!"
print "YOU MUST TERMINATE THE OLD EC2 INSTANCES YOURSELF!"
else:
# Pick a random subnet from list of available subnets
subnet = random.choice(subnets)
print "Randomly picked this subnet to use: ", subnet
print "Launching EC2 instance..."
reservation = ec2_conn.run_instances(
ami.id,
instance_type=args.instance_type,
subnet_id=subnet.id,
security_group_ids=[security_group.id],
key_name=key_name,
instance_profile_name=IAM_ROLE)
if len(reservation.instances) == 0:
print "No instances in reservation!"
sys.exit(1)
instance = reservation.instances[0]
print "{} starting up...".format(instance)
# Check up on its status every so often
status = instance.update()
while status == 'pending':
time.sleep(10)
status = instance.update()
if status == 'running':
for k, v in tags.items():
instance.add_tag(k, v)
print "{} running at {}".format(instance,
instance.private_ip_address)
slackbot.post_message(
"Started up AMI '{}' for '{}' on tier '{}' with ip '{}'".
format(ami.id, service_info["name"], tier_config["tier"],
instance.private_ip_address))
else:
print "Instance was not created correctly"
sys.exit(1)
def _bake_command(args):
service_info = get_service_info()
tier_config = get_tier_config()
iam_conn = boto.iam.connect_to_region(tier_config["region"])
if args.ubuntu:
# Get all Ubuntu Trusty 14.04 images from the appropriate region and
# pick the most recent one.
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
print "Finding the latest AMI on AWS that matches", UBUNTU_RELEASE
ec2 = boto3.resource('ec2', region_name=tier_config["region"])
filters = [
{
'Name': 'name',
'Values': [UBUNTU_RELEASE]
},
]
amis = list(
ec2.images.filter(Owners=[AMI_OWNER_CANONICAL], Filters=filters))
if not amis:
print "No AMI found matching '{}'. Not sure what to do now.".format(
UBUNTU_RELEASE, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
else:
ec2 = boto3.resource('ec2', region_name=tier_config["region"])
filters = [
{
'Name': 'tag:service-name',
'Values': [UBUNTU_BASE_IMAGE_NAME]
},
{
'Name': 'tag:tier',
'Values': [tier_config["tier"]]
},
]
amis = list(ec2.images.filter(Owners=['self'], Filters=filters))
if not amis:
print "No '{}' AMI found for tier {}. Bake one using this command: {} ami bake --ubuntu".format(
UBUNTU_BASE_IMAGE_NAME, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creation_date
if args.ubuntu:
version = None
branch = ''
sha_commit = ''
deployment_manifest = create_deployment_manifest(
'bakeami') # Todo: Should be elsewhere or different
else:
cmd = "python setup.py sdist --formats=zip"
current_branch = get_branch()
if not args.tag:
# See if service is tagged to a specific version for this tier
for si in tier_config['deployables']:
if si['name'] == service_info['name']:
if 'release' in si:
text = "Error: As deployable '{}' for tier '{}' is pegged to a particular " \
"release, you must specify a release tag to which to bake from.\n" \
"Note that this is merely a safety measure.\n" \
"For reference, the current deployable for this tier is pegged at " \
"release tag '{}'."
print text.format(service_info['name'],
tier_config['tier'], si['release'])
sys.exit(1)
break
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
checkout(args.tag)
try:
deployment_manifest = create_deployment_manifest(
'bakeami') # Todo: Should be elsewhere or different
sha_commit = get_commit()
branch = get_branch()
version = get_git_version()
service_info = get_service_info()
if not args.preview:
os.system(cmd)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
if not version:
version = {'tag': 'untagged-branch'}
print "git version:", version
user = iam_conn.get_user() # The current IAM user running this command
# Need to generate a pre-signed url to the tiers root config file on S3
tiers_config = get_tiers_config()
tiers_config_url = '{}/{}.{}/{}'.format(tiers_config['region'],
tiers_config['bucket'],
tiers_config['domain'],
TIERS_CONFIG_FILENAME)
var = {
"service":
UBUNTU_BASE_IMAGE_NAME if args.ubuntu else service_info["name"],
"versionNumber": service_info["version"],
"region": tier_config["region"],
"source_ami": ami.id,
"branch": branch,
"commit": sha_commit,
"release": version['tag'],
"user_name": user.user_name,
"tier": tier_config["tier"],
"tier_url": tiers_config_url,
}
if args.ubuntu:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "ubuntu-packer.sh")
else:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "driftapp-packer.sh")
print "Using var:\n", pretty(var)
packer_cmd = "packer"
try:
result = subprocess.call(packer_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "%s was not found. Please install using the following method:" % packer_cmd
print " brew tap homebrew/binary\n brew install %s" % packer_cmd
sys.exit(1)
else:
print "Packer process returned", result
cmd = "%s build " % packer_cmd
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in var.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
# Dump deployment manifest into dist folder temporarily. The packer script
# will pick it up and bake it into the AMI.
deployment_manifest_filename = os.path.join("dist",
"deployment-manifest.json")
deployment_manifest_json = json.dumps(deployment_manifest, indent=4)
print "Deployment Manifest:\n", deployment_manifest_json
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
with open(deployment_manifest_filename, "w") as dif:
dif.write(deployment_manifest_json)
start_time = time.time()
try:
os.system(cmd)
finally:
os.remove(deployment_manifest_filename)
pkg_resources.cleanup_resources()
duration = time.time() - start_time
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' on tier '{}' in %.0f seconds".
format(service_info["name"], get_tier_name(), duration))
def run_command(args):
tier_config = get_tier_config()
service_info = get_service_info()
tier = tier_config["tier"]
region = tier_config["region"]
service_name = service_info["name"]
public = args.public
include_drift = args.drift
drift_filename = None
drift_fullpath = None
default_tenant = tier_config.get("default_tenant",
"default-{}".format(tier.lower()))
if args.tiername and args.tiername != tier:
print "Default tier is '{}' but you expected '{}'. Quitting now.".format(
tier, args.tiername)
return
is_protected_tier = _get_tier_protection(tier)
if is_protected_tier and tier != args.tiername:
print "You are quickdeploying to '{}' which is a protected tier.".format(
tier)
print "This is not recommended!"
print "If you must do this, and you know what you are doing, state the name of"
print "the tier using the --deploy-to-this-tier argument and run again."
return
# hack
if include_drift:
import drift
drift_path = os.path.split(os.path.split(drift.__file__)[0])[0]
build_fullpath = os.path.join(drift_path, "dist")
if os.path.exists(build_fullpath):
for filename in os.listdir(build_fullpath):
if filename.startswith("Drift-"):
os.remove(os.path.join(build_fullpath, filename))
drift_filename = None
print "Building Drift in {}...".format(build_fullpath)
cmd = [
"python",
os.path.join(drift_path, "setup.py"), "sdist", "--formats=zip"
]
p = subprocess.Popen(cmd,
cwd=drift_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, _ = p.communicate()
if p.returncode != 0:
print stdout
sys.exit(p.returncode)
drift_filename = None
for filename in os.listdir(build_fullpath):
if filename.startswith("Drift-"):
drift_filename = filename
if not drift_filename:
print "Error creating drift package: %s" % stdout
sys.exit(9)
print "Including drift package %s" % drift_filename
drift_fullpath = os.path.join(build_fullpath, drift_filename)
app_location = APP_LOCATION.format(service_name)
old_path = app_location + "_old"
for deployable in tier_config["deployables"]:
if deployable["name"] == service_name:
pem_file = deployable["ssh_key"]
break
else:
print "Service {} not found in tier config for {}".format(
service_name, tier)
sys.exit(1)
print "\n*** DEPLOYING service '{}' TO TIER '{}' IN REGION '{}'\n".format(
service_name, tier, region)
build_filename = "{}-{}.zip".format(service_info["name"],
service_info["version"])
build_fullpath = os.path.join("dist", build_filename)
try:
os.remove(build_fullpath)
except Exception as e:
if "No such file or directory" not in repr(e):
raise
print "Building {}...".format(build_fullpath)
p = subprocess.Popen(["python", "setup.py", "sdist", "--formats=zip"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, nothing = p.communicate()
if not os.path.exists(build_fullpath):
print "Build artefact not found at {}".format(build_fullpath)
print "Build output: %s" % stdout
sys.exit(1)
filters = {
'tag:service-name': service_name,
"instance-state-name": "running",
"tag:tier": tier,
}
print "Finding ec2 instances in region %s from filters: %s" % (region,
filters)
instances = get_ec2_instances(region, filters=filters)
if not instances:
print "Found no running ec2 instances with tag service-name={}".format(
service_name)
return
for ec2 in instances:
if not public:
ip_address = ec2.private_ip_address
else:
ip_address = ec2.ip_address
print "Deploying to {}...".format(ip_address)
env.host_string = ip_address
env.user = EC2_USERNAME
env.key_filename = '~/.ssh/{}'.format(pem_file)
with settings(warn_only=True):
run("rm -f {}".format(build_filename))
put(build_fullpath)
if drift_filename:
put(drift_fullpath)
temp_folder = os.path.splitext(build_filename)[0]
with settings(warn_only=True): # expect some commands to fail
run("sudo rm -f {}".format(UWSGI_LOGFILE))
run("rm -r -f {}".format(temp_folder))
with hide('output'):
run("unzip {}".format(build_filename))
run("sudo rm -r -f {}".format(old_path))
run("sudo mv {} {}".format(app_location, old_path))
deployment_manifest = create_deployment_manifest('quickdeploy')
if args.comment:
deployment_manifest['comment'] = args.comment
deployment_manifest_json = json.dumps(deployment_manifest,
indent=4)
cmd = "echo '{}' > {}/deployment-manifest.json".format(
deployment_manifest_json, temp_folder)
run(cmd)
run("sudo mv {} {}".format(temp_folder, app_location))
if not args.skiprequirements:
with hide('output'):
run("sudo pip install -U -r {}/requirements.txt".format(
app_location))
# unpack drift after we've installed requirements
if drift_filename:
with hide('output'):
run("unzip -o {}".format(drift_filename))
DRIFT_LOCATION = "/usr/local/lib/python2.7/dist-packages/drift"
run("sudo rm -rf {}/*".format(DRIFT_LOCATION))
run("sudo cp -r {}/drift/* {}".format(
drift_filename.replace(".zip", ""), DRIFT_LOCATION))
with hide('output'):
run("sudo service {} restart".format(service_name))
with settings(warn_only=True): # celery might not be present
run("sudo service {}-celery restart".format(service_name))
# make sure the service keeps running
sleep(1.0)
run("sudo service {} status".format(service_name))
# test the service endpoint
try:
with settings(warn_only=True):
with hide('output'):
out = run(
'curl http://127.0.0.1:{} -H "Accept: application/json" -H "Drift-Tenant: {}"'
.format(SERVICE_PORT, default_tenant))
d = json.loads(out)
if "endpoints" not in d:
raise Exception("service json is incorrect: %s" % out)
print "\nService {} is running on {}!".format(
service_name, ip_address)
slackbot.post_message(
"Successfully quick-deployed '{}' to tier '{}'".format(
service_name, tier))
except:
print "Unexpected response: %s" % out
error_report()
raise
def run_command(args):
service_info = get_service_info()
tier_config = get_tier_config()
ec2_conn = boto.ec2.connect_to_region(tier_config["region"])
iam_conn = boto.iam.connect_to_region(tier_config["region"])
if args.ubuntu:
# Get all Ubuntu Trusty 14.04 images from the appropriate region and
# pick the most recent one.
print "Finding the latest AMI on AWS that matches 'ubuntu-trusty-14.04*'"
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
amis = ec2_conn.get_all_images(
owners=['099720109477'],
filters={'name': 'ubuntu/images/hvm/ubuntu-trusty-14.04*'},
)
ami = max(amis, key=operator.attrgetter("creationDate"))
else:
amis = ec2_conn.get_all_images(
owners=['self'], # The current organization
filters={
'tag:service-name': UBUNTU_BASE_IMAGE_NAME,
'tag:tier': tier_config["tier"],
},
)
if not amis:
print "No '{}' AMI found for tier {}. Bake one using this command: {} bakeami --ubuntu".format(
UBUNTU_BASE_IMAGE_NAME, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creationDate"))
print "{} AMI(s) found.".format(len(amis))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creationDate
if args.ubuntu:
version = None
branch = ''
sha_commit = ''
else:
cmd = "python setup.py sdist --formats=zip"
current_branch = get_branch()
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
checkout(args.tag)
try:
sha_commit = get_commit()
branch = get_branch()
version = get_git_version()
if not args.preview:
os.system(cmd)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
if not version:
version = {'tag': 'untagged-branch'}
print "git version:", version
service_info = get_service_info()
user = iam_conn.get_user() # The current IAM user running this command
# Need to generate a pre-signed url to the tiers root config file on S3
tiers_config = get_tiers_config()
tiers_config_url = '{}/{}.{}/{}'.format(tiers_config['region'],
tiers_config['bucket'],
tiers_config['domain'],
TIERS_CONFIG_FILENAME)
var = {
"service":
UBUNTU_BASE_IMAGE_NAME if args.ubuntu else service_info["name"],
"versionNumber": service_info["version"],
"region": tier_config["region"],
"source_ami": str(ami.id),
"branch": branch,
"commit": sha_commit,
"release": version['tag'],
"user_name": str(user.user_name),
"tier": tier_config["tier"],
"tier_url": str(tiers_config_url),
}
if args.ubuntu:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "ubuntu-packer.sh")
else:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "driftapp-packer.sh")
print "Using var:\n", json.dumps({k: str(v)
for k, v in var.iteritems()},
indent=4)
packer_cmd = "packer"
try:
result = subprocess.call(packer_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "%s was not found. Please install using the following method:" % packer_cmd
print " brew tap homebrew/binary\n brew install %s" % packer_cmd
sys.exit(1)
else:
print "Packer process returned", result
cmd = "%s build " % packer_cmd
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in var.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
start_time = time.time()
# Dump deployment manifest into dist folder temporarily. The packer script
# will pick it up and bake it into the AMI.
deployment_manifest_filename = os.path.join("dist",
"deployment-manifest.json")
deployment_manifest_json = json.dumps(
create_deployment_manifest('bakeami'), indent=4)
print "Deployment Manifest:\n", deployment_manifest_json
with open(deployment_manifest_filename, "w") as dif:
dif.write(deployment_manifest_json)
try:
os.system(cmd)
finally:
os.remove(deployment_manifest_filename)
pkg_resources.cleanup_resources()
duration = time.time() - start_time
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' on tier '{}' in %.0f seconds".
format(service_info["name"], get_tier_name(), duration))