def create_deployment_manifest(method):
"""Returns a dict describing the current deployable."""
service_info = get_service_info()
commit = get_commit()
version = get_git_version()
info = {
'deployable': service_info['name'],
'method': method,
'username': getpass.getuser(),
'datetime': datetime.utcnow().isoformat(),
'branch': get_branch(),
'commit': commit,
'commit_url': get_repo_url() + "/commit/" + commit,
'release': version['tag'] if version else 'untagged-branch'
}
return info
def create_deployment_manifest(method, comment=None):
"""Returns a dict describing the current deployable."""
git_version = get_git_version()
git_commit = get_commit()
info = {
'method': method,
'deployable': get_app_name(),
'version': get_app_version(),
'username': getpass.getuser(),
'comment': comment,
'datetime': datetime.utcnow().isoformat(),
'git_branch': get_branch(),
'git_commit': git_commit,
'git_commit_url': get_repo_url() + "/commit/" + git_commit,
'git_release':
git_version['tag'] if git_version else 'untagged-branch',
}
return info
def _bake_command(args):
if args.ubuntu:
name = UBUNTU_BASE_IMAGE_NAME
else:
name = get_app_name()
name = get_app_name()
tier_name = get_tier_name()
conf = get_drift_config(tier_name=tier_name,
deployable_name=name,
drift_app=load_flask_config())
domain = conf.domain.get()
aws_region = domain['aws']['ami_baking_region']
ec2 = boto3.resource('ec2', region_name=aws_region)
print "DOMAIN:\n", json.dumps(domain, indent=4)
if not args.ubuntu:
print "DEPLOYABLE:", name
print "AWS REGION:", aws_region
# Create a list of all regions that are active
if args.ubuntu:
# Get all Ubuntu images from the appropriate region and pick the most recent one.
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
print "Finding the latest AMI on AWS that matches", UBUNTU_RELEASE
filters = [
{
'Name': 'name',
'Values': [UBUNTU_RELEASE]
},
]
amis = list(
ec2.images.filter(Owners=[AMI_OWNER_CANONICAL], Filters=filters))
if not amis:
print "No AMI found matching '{}'. Not sure what to do now.".format(
UBUNTU_RELEASE)
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
else:
filters = [
{
'Name': 'tag:service-name',
'Values': [UBUNTU_BASE_IMAGE_NAME]
},
{
'Name': 'tag:domain-name',
'Values': [domain['domain_name']]
},
]
amis = list(ec2.images.filter(Owners=['self'], Filters=filters))
if not amis:
criteria = {d['Name']: d['Values'][0] for d in filters}
print "No '{}' AMI found using the search criteria {}.".format(
UBUNTU_BASE_IMAGE_NAME, criteria)
print "Bake one using this command: {} ami bake --ubuntu".format(
sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creation_date
if args.ubuntu:
manifest = None
packer_vars = {
'setup_script':
pkg_resources.resource_filename(__name__, "ubuntu-packer.sh"),
'ubuntu_release':
UBUNTU_RELEASE,
}
else:
current_branch = get_branch()
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
# Wrap git branch modification in RAII.
checkout(args.tag)
try:
setup_script = ""
setup_script_custom = ""
with open(
pkg_resources.resource_filename(__name__,
"driftapp-packer.sh"),
'r') as f:
setup_script = f.read()
custom_script_name = os.path.join(conf.drift_app['app_root'],
'scripts', 'ami-bake.sh')
if os.path.exists(custom_script_name):
print "Using custom bake shell script", custom_script_name
setup_script_custom = "echo Executing custom bake shell script from {}\n".format(
custom_script_name)
setup_script_custom += open(custom_script_name, 'r').read()
setup_script_custom += "\necho Custom bake shell script completed\n"
else:
print "Note: No custom ami-bake.sh script found for this application."
# custom setup needs to happen first because we might be installing some requirements for the regular setup
setup_script = setup_script_custom + setup_script
tf = tempfile.NamedTemporaryFile(delete=False)
tf.write(setup_script)
tf.close()
setup_script_filename = tf.name
manifest = create_deployment_manifest('ami', comment=None)
packer_vars = {
'version': get_app_version(),
'setup_script': setup_script_filename,
}
if not args.preview:
cmd = ['python', 'setup.py', 'sdist', '--formats=zip']
ret = subprocess.call(cmd)
if ret != 0:
print "Failed to execute build command:", cmd
sys.exit(ret)
cmd = ["zip", "-r", "dist/aws.zip", "aws"]
ret = subprocess.call(cmd)
if ret != 0:
print "Failed to execute build command:", cmd
sys.exit(ret)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
user = boto.iam.connect_to_region(
aws_region).get_user() # The current IAM user running this command
packer_vars.update({
"service": name,
"region": aws_region,
"source_ami": ami.id,
"user_name": user.user_name,
"domain_name": domain['domain_name'],
})
print "Packer variables:\n", pretty(packer_vars)
# See if Packer is installed and generate sensible error code if something is off.
# This will also write the Packer version to the terminal which is useful info.
try:
subprocess.call(['packer', 'version'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "'packer version' command failed. Please install it if it's missing."
sys.exit(127)
cmd = "packer build "
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in packer_vars.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
start_time = time.time()
try:
# Execute Packer command and parse the output to find the ami id.
p = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
while True:
line = p.stdout.readline()
print line,
if line == '' and p.poll() is not None:
break
# The last lines from the packer execution look like this:
# ==> Builds finished. The artifacts of successful builds are:
# --> amazon-ebs: AMIs were created:
#
# eu-west-1: ami-0ee5eb68
if 'ami-' in line:
ami_id = line[line.rfind('ami-'):].strip()
ami = ec2.Image(ami_id)
print ""
print "AMI ID: %s" % ami.id
print ""
finally:
pkg_resources.cleanup_resources()
if p.returncode != 0:
print "Failed to execute packer command:", cmd
sys.exit(p.returncode)
duration = time.time() - start_time
if manifest:
print "Adding manifest tags to AMI:"
pretty(manifest)
prefix = "drift:manifest:"
tags = []
for k, v in manifest.iteritems():
tag_name = "{}{}".format(prefix, k)
tags.append({'Key': tag_name, 'Value': v or ''})
ami.create_tags(DryRun=False, Tags=tags)
if not args.skipcopy:
_copy_image(ami.id)
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' in %.0f seconds".format(
name, duration))
def _bake_command(args):
service_info = get_service_info()
tier_config = get_tier_config()
iam_conn = boto.iam.connect_to_region(tier_config["region"])
if args.ubuntu:
# Get all Ubuntu Trusty 14.04 images from the appropriate region and
# pick the most recent one.
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
print "Finding the latest AMI on AWS that matches", UBUNTU_RELEASE
ec2 = boto3.resource('ec2', region_name=tier_config["region"])
filters = [
{
'Name': 'name',
'Values': [UBUNTU_RELEASE]
},
]
amis = list(
ec2.images.filter(Owners=[AMI_OWNER_CANONICAL], Filters=filters))
if not amis:
print "No AMI found matching '{}'. Not sure what to do now.".format(
UBUNTU_RELEASE, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
else:
ec2 = boto3.resource('ec2', region_name=tier_config["region"])
filters = [
{
'Name': 'tag:service-name',
'Values': [UBUNTU_BASE_IMAGE_NAME]
},
{
'Name': 'tag:tier',
'Values': [tier_config["tier"]]
},
]
amis = list(ec2.images.filter(Owners=['self'], Filters=filters))
if not amis:
print "No '{}' AMI found for tier {}. Bake one using this command: {} ami bake --ubuntu".format(
UBUNTU_BASE_IMAGE_NAME, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creation_date"))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creation_date
if args.ubuntu:
version = None
branch = ''
sha_commit = ''
deployment_manifest = create_deployment_manifest(
'bakeami') # Todo: Should be elsewhere or different
else:
cmd = "python setup.py sdist --formats=zip"
current_branch = get_branch()
if not args.tag:
# See if service is tagged to a specific version for this tier
for si in tier_config['deployables']:
if si['name'] == service_info['name']:
if 'release' in si:
text = "Error: As deployable '{}' for tier '{}' is pegged to a particular " \
"release, you must specify a release tag to which to bake from.\n" \
"Note that this is merely a safety measure.\n" \
"For reference, the current deployable for this tier is pegged at " \
"release tag '{}'."
print text.format(service_info['name'],
tier_config['tier'], si['release'])
sys.exit(1)
break
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
checkout(args.tag)
try:
deployment_manifest = create_deployment_manifest(
'bakeami') # Todo: Should be elsewhere or different
sha_commit = get_commit()
branch = get_branch()
version = get_git_version()
service_info = get_service_info()
if not args.preview:
os.system(cmd)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
if not version:
version = {'tag': 'untagged-branch'}
print "git version:", version
user = iam_conn.get_user() # The current IAM user running this command
# Need to generate a pre-signed url to the tiers root config file on S3
tiers_config = get_tiers_config()
tiers_config_url = '{}/{}.{}/{}'.format(tiers_config['region'],
tiers_config['bucket'],
tiers_config['domain'],
TIERS_CONFIG_FILENAME)
var = {
"service":
UBUNTU_BASE_IMAGE_NAME if args.ubuntu else service_info["name"],
"versionNumber": service_info["version"],
"region": tier_config["region"],
"source_ami": ami.id,
"branch": branch,
"commit": sha_commit,
"release": version['tag'],
"user_name": user.user_name,
"tier": tier_config["tier"],
"tier_url": tiers_config_url,
}
if args.ubuntu:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "ubuntu-packer.sh")
else:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "driftapp-packer.sh")
print "Using var:\n", pretty(var)
packer_cmd = "packer"
try:
result = subprocess.call(packer_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "%s was not found. Please install using the following method:" % packer_cmd
print " brew tap homebrew/binary\n brew install %s" % packer_cmd
sys.exit(1)
else:
print "Packer process returned", result
cmd = "%s build " % packer_cmd
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in var.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
# Dump deployment manifest into dist folder temporarily. The packer script
# will pick it up and bake it into the AMI.
deployment_manifest_filename = os.path.join("dist",
"deployment-manifest.json")
deployment_manifest_json = json.dumps(deployment_manifest, indent=4)
print "Deployment Manifest:\n", deployment_manifest_json
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
with open(deployment_manifest_filename, "w") as dif:
dif.write(deployment_manifest_json)
start_time = time.time()
try:
os.system(cmd)
finally:
os.remove(deployment_manifest_filename)
pkg_resources.cleanup_resources()
duration = time.time() - start_time
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' on tier '{}' in %.0f seconds".
format(service_info["name"], get_tier_name(), duration))
def run_command(args):
service_info = get_service_info()
tier_config = get_tier_config()
ec2_conn = boto.ec2.connect_to_region(tier_config["region"])
iam_conn = boto.iam.connect_to_region(tier_config["region"])
if args.ubuntu:
# Get all Ubuntu Trusty 14.04 images from the appropriate region and
# pick the most recent one.
print "Finding the latest AMI on AWS that matches 'ubuntu-trusty-14.04*'"
# The 'Canonical' owner. This organization maintains the Ubuntu AMI's on AWS.
amis = ec2_conn.get_all_images(
owners=['099720109477'],
filters={'name': 'ubuntu/images/hvm/ubuntu-trusty-14.04*'},
)
ami = max(amis, key=operator.attrgetter("creationDate"))
else:
amis = ec2_conn.get_all_images(
owners=['self'], # The current organization
filters={
'tag:service-name': UBUNTU_BASE_IMAGE_NAME,
'tag:tier': tier_config["tier"],
},
)
if not amis:
print "No '{}' AMI found for tier {}. Bake one using this command: {} bakeami --ubuntu".format(
UBUNTU_BASE_IMAGE_NAME, tier_config["tier"], sys.argv[0])
sys.exit(1)
ami = max(amis, key=operator.attrgetter("creationDate"))
print "{} AMI(s) found.".format(len(amis))
print "Using source AMI:"
print "\tID:\t", ami.id
print "\tName:\t", ami.name
print "\tDate:\t", ami.creationDate
if args.ubuntu:
version = None
branch = ''
sha_commit = ''
else:
cmd = "python setup.py sdist --formats=zip"
current_branch = get_branch()
if not args.tag:
args.tag = current_branch
print "Using branch/tag", args.tag
checkout(args.tag)
try:
sha_commit = get_commit()
branch = get_branch()
version = get_git_version()
if not args.preview:
os.system(cmd)
finally:
print "Reverting to ", current_branch
checkout(current_branch)
if not version:
version = {'tag': 'untagged-branch'}
print "git version:", version
service_info = get_service_info()
user = iam_conn.get_user() # The current IAM user running this command
# Need to generate a pre-signed url to the tiers root config file on S3
tiers_config = get_tiers_config()
tiers_config_url = '{}/{}.{}/{}'.format(tiers_config['region'],
tiers_config['bucket'],
tiers_config['domain'],
TIERS_CONFIG_FILENAME)
var = {
"service":
UBUNTU_BASE_IMAGE_NAME if args.ubuntu else service_info["name"],
"versionNumber": service_info["version"],
"region": tier_config["region"],
"source_ami": str(ami.id),
"branch": branch,
"commit": sha_commit,
"release": version['tag'],
"user_name": str(user.user_name),
"tier": tier_config["tier"],
"tier_url": str(tiers_config_url),
}
if args.ubuntu:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "ubuntu-packer.sh")
else:
var['setup_script'] = pkg_resources.resource_filename(
__name__, "driftapp-packer.sh")
print "Using var:\n", json.dumps({k: str(v)
for k, v in var.iteritems()},
indent=4)
packer_cmd = "packer"
try:
result = subprocess.call(packer_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except Exception as e:
print "Error:", e
print "%s was not found. Please install using the following method:" % packer_cmd
print " brew tap homebrew/binary\n brew install %s" % packer_cmd
sys.exit(1)
else:
print "Packer process returned", result
cmd = "%s build " % packer_cmd
if args.debug:
cmd += "-debug "
cmd += "-only=amazon-ebs "
for k, v in var.iteritems():
cmd += "-var {}=\"{}\" ".format(k, v)
# Use generic packer script if project doesn't specify one
pkg_resources.cleanup_resources()
if args.ubuntu:
scriptfile = pkg_resources.resource_filename(__name__,
"ubuntu-packer.json")
cmd += scriptfile
elif os.path.exists("config/packer.json"):
cmd += "config/packer.json"
else:
scriptfile = pkg_resources.resource_filename(__name__,
"driftapp-packer.json")
cmd += scriptfile
print "Baking AMI with: {}".format(cmd)
if args.preview:
print "Not building or packaging because --preview is on. Exiting now."
return
start_time = time.time()
# Dump deployment manifest into dist folder temporarily. The packer script
# will pick it up and bake it into the AMI.
deployment_manifest_filename = os.path.join("dist",
"deployment-manifest.json")
deployment_manifest_json = json.dumps(
create_deployment_manifest('bakeami'), indent=4)
print "Deployment Manifest:\n", deployment_manifest_json
with open(deployment_manifest_filename, "w") as dif:
dif.write(deployment_manifest_json)
try:
os.system(cmd)
finally:
os.remove(deployment_manifest_filename)
pkg_resources.cleanup_resources()
duration = time.time() - start_time
print "Done after %.0f seconds" % (duration)
slackbot.post_message(
"Successfully baked a new AMI for '{}' on tier '{}' in %.0f seconds".
format(service_info["name"], get_tier_name(), duration))