def update_template(stackname):
"""Limited update of the Cloudformation template.
Resources can be added, but existing ones are immutable.
Moreover, we never add anything related to EC2 instances as they are
not supported anyway (they will come up as part of the template
but without any software being on it)
Moreover, EC2 instances must be running while this is executed or their
resources like PublicIP will be inaccessible"""
core_lifecycle.start(stackname)
(pname, _) = core.parse_stackname(stackname)
current_template = bootstrap.current_template(stackname)
cfngen.write_template(stackname, json.dumps(current_template))
more_context = cfngen.choose_config(stackname)
delta = cfngen.template_delta(pname, **more_context)
LOG.info("%s", pformat(delta))
utils.confirm('Confirming changes to the stack template?')
new_template = cfngen.merge_delta(stackname, delta)
bootstrap.update_template(stackname, new_template)
update(stackname)
def restart_all_running_ec2(statefile):
"restarts all running ec2 instances. multiple nodes are restarted serially and failures prevent the rest of the node from being restarted"
os.system("touch " + statefile)
results = core.active_stack_names(core.find_region())
u1404 = [
'api-gateway',
'journal',
'search',
'api-dummy',
'medium',
]
legacy = [
'elife-api'
]
dont_do = u1404 + legacy
# order not preserved
do_first = [
'master-server',
'bus',
'elife-alfred',
'elife-bot',
'iiif',
]
pname = lambda stackname: core.parse_stackname(stackname)[0]
todo = sorted(results, key=lambda stackname: pname(stackname) in do_first, reverse=True)
todo = filter(lambda stackname: pname(stackname) not in dont_do, todo)
with open(statefile, 'r') as fh:
done = fh.read().split('\n')
with open(statefile, 'a') as fh:
LOG.info('writing state to ' + fh.name)
for stackname in todo:
if stackname in done:
LOG.info('skipping ' + stackname)
continue
try:
LOG.info('restarting' + stackname)
# only restart instances that are currently running
# this will skip ci/end2end
lifecycle.restart(stackname, initial_states='running')
LOG.info('done' + stackname)
fh.write(stackname + "\n")
fh.flush()
except BaseException:
LOG.exception("unhandled exception restarting %s", stackname)
LOG.warn("%s is in an unknown state", stackname)
get_input('pausing, any key to continue, ctrl+c to quit')
print
print('wrote state to', fh.name)
def update_infrastructure(stackname, skip=None, start=['ec2']):
"""Limited update of the Cloudformation template and/or Terraform template.
Resources can be added, but most of the existing ones are immutable.
Some resources are updatable in place.
Moreover, we never add anything related to EC2 instances as they are
not supported anyway (they will come up as part of the template
but without any software being on it)
Moreover, EC2 instances must be running while this is executed or their
resources like PublicIP will be inaccessible.
Allows to skip EC2, SQS, S3 updates by passing `skip=ec2\\,sqs\\,s3`
By default starts EC2 instances but this can be avoid by passing `start=`"""
skip = skip.split(",") if skip else []
start = start.split(",") if isinstance(start, str) else start or []
(pname, _) = core.parse_stackname(stackname)
more_context = {}
context, delta, current_context = cfngen.regenerate_stack(
stackname, **more_context)
if _are_there_existing_servers(current_context) and 'ec2' in start:
core_lifecycle.start(stackname)
LOG.info("Create: %s", pformat(delta.plus))
LOG.info("Update: %s", pformat(delta.edit))
LOG.info("Delete: %s", pformat(delta.minus))
LOG.info("Terraform delta: %s", delta.terraform)
# see: `buildercore.config.BUILDER_NON_INTERACTIVE` for skipping confirmation prompts
utils.confirm(
'Confirming changes to CloudFormation and Terraform templates?')
context_handler.write_context(stackname, context)
cloudformation.update_template(stackname, delta.cloudformation)
terraform.update_template(stackname)
# TODO: move inside bootstrap.update_stack
# EC2
if _are_there_existing_servers(context) and not 'ec2' in skip:
# the /etc/buildvars.json file may need to be updated
buildvars.refresh(stackname, context)
update(stackname)
# SQS
if context.get('sqs', {}) and not 'sqs' in skip:
bootstrap.update_stack(stackname, service_list=['sqs'])
# S3
if context.get('s3', {}) and not 's3' in skip:
bootstrap.update_stack(stackname, service_list=['s3'])
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
def adhoc_instance(stackname):
"predicate, returns True if stackname is *not* in a known environment"
try:
iid = core.parse_stackname(stackname, all_bits=True,
idx=True)['instance_id']
return iid not in env_list
except (ValueError, AssertionError):
# thrown by `parse_stackname` when given value isn't a string or
# delimiter not found in string.
return True
def update_infrastructure(stackname, skip=None, start=['ec2']):
"""Limited update of the Cloudformation template and/or Terraform template.
Resources can be added, but most of the existing ones are immutable.
Some resources are updatable in place.
Moreover, we never add anything related to EC2 instances as they are
not supported anyway (they will come up as part of the template
but without any software being on it)
Moreover, EC2 instances must be running while this is executed or their
resources like PublicIP will be inaccessible.
Allows to skip EC2, SQS, S3 updates by passing `skip=ec2\\,sqs\\,s3`
By default starts EC2 instances but this can be avoid by passing `start=`"""
skip = skip.split(",") if skip else []
start = start.split(",") if isinstance(start, str) else start or []
(pname, _) = core.parse_stackname(stackname)
more_context = {}
context, delta, current_context = cfngen.regenerate_stack(stackname, **more_context)
if _are_there_existing_servers(current_context) and 'ec2' in start:
core_lifecycle.start(stackname)
LOG.info("Create: %s", pformat(delta.plus))
LOG.info("Update: %s", pformat(delta.edit))
LOG.info("Delete: %s", pformat(delta.minus))
LOG.info("Terraform delta: %s", delta.terraform)
utils.confirm('Confirming changes to CloudFormation and Terraform templates?')
context_handler.write_context(stackname, context)
cloudformation.update_template(stackname, delta.cloudformation)
terraform.update_template(stackname)
# TODO: move inside bootstrap.update_stack
# EC2
if _are_there_existing_servers(context) and not 'ec2' in skip:
# the /etc/buildvars.json file may need to be updated
buildvars.refresh(stackname, context)
update(stackname)
# SQS
if context.get('sqs', {}) and not 'sqs' in skip:
bootstrap.update_stack(stackname, service_list=['sqs'])
# S3
if context.get('s3', {}) and not 's3' in skip:
bootstrap.update_stack(stackname, service_list=['s3'])
def update_template(stackname):
"""Limited update of the Cloudformation template.
Resources can be added, but most of the existing ones are immutable.
Some resources are updatable in place.
Moreover, we never add anything related to EC2 instances as they are
not supported anyway (they will come up as part of the template
but without any software being on it)
Moreover, EC2 instances must be running while this is executed or their
resources like PublicIP will be inaccessible"""
(pname, _) = core.parse_stackname(stackname)
more_context = cfngen.choose_config(stackname)
context, delta_plus, delta_minus = cfngen.regenerate_stack(
pname, **more_context)
if context['ec2']:
core_lifecycle.start(stackname)
LOG.info("Create/update: %s", pformat(delta_plus))
LOG.info("Delete: %s", pformat(delta_minus))
utils.confirm(
'Confirming changes to the stack template? This will rewrite the context and the CloudFormation template'
)
context_handler.write_context(stackname, context)
if delta_plus['Resources'] or delta_plus['Outputs'] or delta_minus[
'Resources'] or delta_minus['Outputs']:
new_template = cfngen.merge_delta(stackname, delta_plus, delta_minus)
bootstrap.update_template(stackname, new_template)
# the /etc/buildvars.json file may need to be updated
buildvars.refresh(stackname, context)
else:
# attempting to apply an empty change set would result in an error
LOG.info("Nothing to update on CloudFormation")
update(stackname)
def test_template_delta_includes_cloudfront(self):
"we can add CDNs (that takes an hour or more) without downtime"
context = self._base_context()
stackname, environment_name = core.parse_stackname(context['stackname'])
context['full_hostname'] = "test--dummy1.example.org"
context['cloudfront'] = {
"subdomains": [
"test--cdn-dummy1.example.org"
],
"subdomains-without-dns": [],
"origins": {},
"compress": True,
"cookies": [],
"certificate_id": "AAAA...",
"headers": [],
"errors": None,
"default-ttl": 300,
"logging": False,
}
(delta_plus, delta_edit, delta_minus, cloudformation_delta, new_terraform_template_file) = cfngen.template_delta(context)
self.assertCountEqual(list(delta_plus['Resources'].keys()), ['CloudFrontCDN', 'CloudFrontCDNDNS1', 'ExtDNS'])
self.assertEqual(list(delta_plus['Outputs'].keys()), ['DomainName'])
def sort_by_env(name):
"""comparator. used when sorting a list of ec2 or cloudformation names.
basic alphabetical ordering if given `name` cannot be parsed."""
adhoc = 0 # adhoc/unrecognised names first
order = {
'continuumtest': 1,
'staging': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # prod last
}
try:
pname, env, node = core.parse_stackname(name, all_bits=True)
# groups results by project name, then a consistent ordering by env, then node
return "%s%s%s" % (pname, order.get(env, adhoc), node)
except (ValueError, AssertionError):
# thrown by `parse_stackname` when given value isn't a string or
# delimiter not found in string.
# by returning the given `name` here we get a basic alphabetical order
# for lists that don't contain an environment.
return name
def test_template_delta_includes_cloudfront(self):
"we can add CDNs (that takes an hour or more) without downtime"
context = self._base_context()
stackname, environment_name = core.parse_stackname(context['stackname'])
context['full_hostname'] = "test--dummy1.example.org"
context['cloudfront'] = {
"subdomains": [
"test--cdn-dummy1.example.org"
],
"subdomains-without-dns": [],
"origins": {},
"compress": True,
"cookies": [],
"certificate_id": "AAAA...",
"headers": [],
"errors": None,
"default-ttl": 300,
"logging": False,
}
(delta_plus, delta_edit, delta_minus, cloudformation_delta, new_terraform_template_file) = cfngen.template_delta(context)
self.assertCountEqual(list(delta_plus['Resources'].keys()), ['CloudFrontCDN', 'CloudFrontCDNDNS1', 'ExtDNS'])
self.assertEqual(list(delta_plus['Outputs'].keys()), ['DomainName'])
def remaster_all(*pname_list):
"calls `remaster` on *all* projects or just a subset of projects"
# there should only be one master-server instance at a time.
# multiple masters is bad news. assumptions break and it gets complicated quickly.
new_master_stackname = "master-server--2018-04-09-2"
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
# we can optionally pass in a list of projects to target
# this allows us to partition up the projects and have many of these
# remastering efforts happening concurrently
if pname_list:
more_ignore = [p for p in ec2stacks if p not in pname_list]
ec2stacks = exsubdict(ec2stacks, more_ignore)
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# TODO: skip any stacks without ec2 instances
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
# when would this ever be the case?
# `core.active_stack_names` doesn't discriminate against any list of projects
# it returns *all* steady stack names.
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
# print a reminder of which stack was just updated
print("\n(%s)\n" % stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
LOG.info("ctrl-c again to exit process entirely")
time.sleep(2)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open('remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s", stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn("failed to remaster %s, stopping further remasters to project %r", stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s", stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def _generate_context(self, stackname):
(pname, instance_id) = parse_stackname(stackname)
context = cfngen.build_context(pname, stackname=stackname)
self.contexts[stackname] = context
def _generate_context(self, stackname):
(pname, instance_id) = parse_stackname(stackname)
context = cfngen.build_context(pname, stackname=stackname)
self.contexts[stackname] = context
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
