def restart_all_running_ec2(statefile): "restarts all running ec2 instances. multiple nodes are restarted serially and failures prevent the rest of the node from being restarted" os.system("touch " + statefile) results = core.active_stack_names(core.find_region()) u1404 = [ 'api-gateway', 'journal', 'search', 'api-dummy', 'medium', ] legacy = [ 'elife-api' ] dont_do = u1404 + legacy # order not preserved do_first = [ 'master-server', 'bus', 'elife-alfred', 'elife-bot', 'iiif', ] pname = lambda stackname: core.parse_stackname(stackname)[0] todo = sorted(results, key=lambda stackname: pname(stackname) in do_first, reverse=True) todo = filter(lambda stackname: pname(stackname) not in dont_do, todo) with open(statefile, 'r') as fh: done = fh.read().split('\n') with open(statefile, 'a') as fh: LOG.info('writing state to ' + fh.name) for stackname in todo: if stackname in done: LOG.info('skipping ' + stackname) continue try: LOG.info('restarting' + stackname) # only restart instances that are currently running # this will skip ci/end2end lifecycle.restart(stackname, initial_states='running') LOG.info('done' + stackname) fh.write(stackname + "\n") fh.flush() except BaseException: LOG.exception("unhandled exception restarting %s", stackname) LOG.warn("%s is in an unknown state", stackname) get_input('pausing, any key to continue, ctrl+c to quit') print print('wrote state to', fh.name)
def call(*args, **kwargs): stackname = first(args) or os.environ.get('INSTANCE') region = aws.find_region(stackname) asl = core.active_stack_names(region) if not asl: raise RuntimeError( '\nno AWS stacks *in an active state* exist, cannot continue.') if not stackname or stackname not in asl: stackname = utils._pick("stack", asl, default_file=deffile('.active-stack')) args = args[1:] return func(stackname, *args, **kwargs)
def call(*args, **kwargs): stackname = first(args) or os.environ.get('INSTANCE') region = utils.find_region(stackname) if stackname: args = args[1:] return func(stackname, *args, **kwargs) asl = core.active_stack_names(region) if not asl: raise RuntimeError('\nno AWS stacks *in an active state* exist, cannot continue.') if not stackname or stackname not in asl: stackname = utils._pick("stack", asl, default_file=deffile('.active-stack')) args = args[1:] return func(stackname, *args, **kwargs)
def _wrapper(stackname=None, *args, **kwargs): region = utils.find_region(stackname) asl = core.active_stack_names(region) if not asl: print('\nno AWS stacks exist, cannot continue.') return def pname_startswith(stack): for pname in plist: if stack.startswith(pname): return stack asl = lfilter(pname_startswith, asl) if not stackname or stackname not in asl: stackname = utils._pick("stack", asl) return func(stackname, *args, **kwargs)
def _wrapper(stackname=None, *args, **kwargs): region = aws.find_region(stackname) asl = core.active_stack_names(region) if not asl: print '\nno AWS stacks exist, cannot continue.' return def pname_startswith(stack): for pname in plist: if stack.startswith(pname): return stack asl = filter(pname_startswith, asl) if not stackname or stackname not in asl: stackname = utils._pick("stack", asl) return func(stackname, *args, **kwargs)
def deploy(pname, instance_id=None, branch='master', part_filter=None): pdata = project.project_data(pname) if not branch: branch_list = utils.git_remote_branches(pdata['repo']) branch_list = impose_ordering(branch_list) branch = utils._pick('branch', branch_list, deffile('.branch')) stackname = cfn.generate_stack_from_input(pname, instance_id) region = pdata['aws']['region'] active_stacks = core.active_stack_names(region) if stackname in active_stacks: LOG.info("stack %r exists, skipping creation", stackname) else: LOG.info("stack %r doesn't exist, creating", stackname) more_context = cfngen.choose_config(stackname) more_context['branch'] = branch cfngen.generate_stack(pname, **more_context) bootstrap.create_update(stackname, part_filter) setdefault('.active-stack', stackname)
def aws_stack_list(): "returns a list of realized stacks. does not include deleted stacks" region = utils.find_region() return core.active_stack_names(region)
def remaster_all(*pname_list): "calls `remaster` on *all* projects or just a subset of projects" # there should only be one master-server instance at a time. # multiple masters is bad news. assumptions break and it gets complicated quickly. new_master_stackname = "master-server--2018-04-09-2" LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) # we can optionally pass in a list of projects to target # this allows us to partition up the projects and have many of these # remastering efforts happening concurrently if pname_list: more_ignore = [p for p in ec2stacks if p not in pname_list] ec2stacks = exsubdict(ec2stacks, more_ignore) pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # TODO: skip any stacks without ec2 instances # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open( 'remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: # when would this ever be the case? # `core.active_stack_names` doesn't discriminate against any list of projects # it returns *all* steady stack names. if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn( "failed to remaster %s, stopping further remasters to project %r", stackname, pname) break # print a reminder of which stack was just updated print("\n(%s)\n" % stackname) open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) LOG.info("ctrl-c again to exit process entirely") time.sleep(2) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")
def remaster_all(new_master_stackname): LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) def sortbypname(n): unknown = 9 porder = { #'observer': 1, 'elife-metrics': 2, 'lax': 3, 'basebox': 4, 'containers': 5, 'elife-dashboard': 6, 'elife-ink': 7 } return porder.get(n, unknown) # pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open('remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) open('remastered.txt', 'a').write("%s\n" % stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn("failed to remaster %s, stopping further remasters to project %r", stackname, pname) break open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) time.sleep(1) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")
def stack_list(region=None): "returns a list of realized stacks. does not include deleted stacks" if not region: region = find_region() return core.active_stack_names(region)
def remaster_minions(): map(remaster_minion, core.active_stack_names(aws.find_region()))
def remaster_all(new_master_stackname): LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) def sortbypname(n): unknown = 9 porder = { #'observer': 1, 'elife-metrics': 2, 'lax': 3, 'basebox': 4, 'containers': 5, 'elife-dashboard': 6, 'elife-ink': 7 } return porder.get(n, unknown) # pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open( 'remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) open('remastered.txt', 'a').write("%s\n" % stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn( "failed to remaster %s, stopping further remasters to project %r", stackname, pname) break open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) time.sleep(1) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")