def restart_all_running_ec2(statefile):
"restarts all running ec2 instances. multiple nodes are restarted serially and failures prevent the rest of the node from being restarted"
os.system("touch " + statefile)
results = core.active_stack_names(core.find_region())
u1404 = [
'api-gateway',
'journal',
'search',
'api-dummy',
'medium',
]
legacy = [
'elife-api'
]
dont_do = u1404 + legacy
# order not preserved
do_first = [
'master-server',
'bus',
'elife-alfred',
'elife-bot',
'iiif',
]
pname = lambda stackname: core.parse_stackname(stackname)[0]
todo = sorted(results, key=lambda stackname: pname(stackname) in do_first, reverse=True)
todo = filter(lambda stackname: pname(stackname) not in dont_do, todo)
with open(statefile, 'r') as fh:
done = fh.read().split('\n')
with open(statefile, 'a') as fh:
LOG.info('writing state to ' + fh.name)
for stackname in todo:
if stackname in done:
LOG.info('skipping ' + stackname)
continue
try:
LOG.info('restarting' + stackname)
# only restart instances that are currently running
# this will skip ci/end2end
lifecycle.restart(stackname, initial_states='running')
LOG.info('done' + stackname)
fh.write(stackname + "\n")
fh.flush()
except BaseException:
LOG.exception("unhandled exception restarting %s", stackname)
LOG.warn("%s is in an unknown state", stackname)
get_input('pausing, any key to continue, ctrl+c to quit')
print
print('wrote state to', fh.name)
def call(*args, **kwargs):
stackname = first(args) or os.environ.get('INSTANCE')
region = aws.find_region(stackname)
asl = core.active_stack_names(region)
if not asl:
raise RuntimeError(
'\nno AWS stacks *in an active state* exist, cannot continue.')
if not stackname or stackname not in asl:
stackname = utils._pick("stack",
asl,
default_file=deffile('.active-stack'))
args = args[1:]
return func(stackname, *args, **kwargs)
def call(*args, **kwargs):
stackname = first(args) or os.environ.get('INSTANCE')
region = utils.find_region(stackname)
if stackname:
args = args[1:]
return func(stackname, *args, **kwargs)
asl = core.active_stack_names(region)
if not asl:
raise RuntimeError('\nno AWS stacks *in an active state* exist, cannot continue.')
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl, default_file=deffile('.active-stack'))
args = args[1:]
return func(stackname, *args, **kwargs)
def _wrapper(stackname=None, *args, **kwargs):
region = utils.find_region(stackname)
asl = core.active_stack_names(region)
if not asl:
print('\nno AWS stacks exist, cannot continue.')
return
def pname_startswith(stack):
for pname in plist:
if stack.startswith(pname):
return stack
asl = lfilter(pname_startswith, asl)
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl)
return func(stackname, *args, **kwargs)
def _wrapper(stackname=None, *args, **kwargs):
region = aws.find_region(stackname)
asl = core.active_stack_names(region)
if not asl:
print '\nno AWS stacks exist, cannot continue.'
return
def pname_startswith(stack):
for pname in plist:
if stack.startswith(pname):
return stack
asl = filter(pname_startswith, asl)
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl)
return func(stackname, *args, **kwargs)
def deploy(pname, instance_id=None, branch='master', part_filter=None):
pdata = project.project_data(pname)
if not branch:
branch_list = utils.git_remote_branches(pdata['repo'])
branch_list = impose_ordering(branch_list)
branch = utils._pick('branch', branch_list, deffile('.branch'))
stackname = cfn.generate_stack_from_input(pname, instance_id)
region = pdata['aws']['region']
active_stacks = core.active_stack_names(region)
if stackname in active_stacks:
LOG.info("stack %r exists, skipping creation", stackname)
else:
LOG.info("stack %r doesn't exist, creating", stackname)
more_context = cfngen.choose_config(stackname)
more_context['branch'] = branch
cfngen.generate_stack(pname, **more_context)
bootstrap.create_update(stackname, part_filter)
setdefault('.active-stack', stackname)
def aws_stack_list():
"returns a list of realized stacks. does not include deleted stacks"
region = utils.find_region()
return core.active_stack_names(region)
def remaster_all(*pname_list):
"calls `remaster` on *all* projects or just a subset of projects"
# there should only be one master-server instance at a time.
# multiple masters is bad news. assumptions break and it gets complicated quickly.
new_master_stackname = "master-server--2018-04-09-2"
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
# we can optionally pass in a list of projects to target
# this allows us to partition up the projects and have many of these
# remastering efforts happening concurrently
if pname_list:
more_ignore = [p for p in ec2stacks if p not in pname_list]
ec2stacks = exsubdict(ec2stacks, more_ignore)
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# TODO: skip any stacks without ec2 instances
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
# when would this ever be the case?
# `core.active_stack_names` doesn't discriminate against any list of projects
# it returns *all* steady stack names.
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
# print a reminder of which stack was just updated
print("\n(%s)\n" % stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
LOG.info("ctrl-c again to exit process entirely")
time.sleep(2)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open('remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s", stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn("failed to remaster %s, stopping further remasters to project %r", stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s", stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def stack_list(region=None):
"returns a list of realized stacks. does not include deleted stacks"
if not region:
region = find_region()
return core.active_stack_names(region)
def aws_stack_list():
"returns a list of realized stacks. does not include deleted stacks"
region = utils.find_region()
return core.active_stack_names(region)
def remaster_minions():
map(remaster_minion, core.active_stack_names(aws.find_region()))
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def remaster_minions():
map(remaster_minion, core.active_stack_names(aws.find_region()))
