def update(master_stackname=None): "same as `cfn.update` but also removes any orphaned minion keys" master_stackname = master_stackname or core.find_master(utils.find_region()) bootstrap.update_stack(master_stackname, service_list=[ 'ec2' # master-server should be a self-contained EC2 instance ]) bootstrap.remove_all_orphaned_keys(master_stackname)
def split_macro_args(content): """ Splits macro content into different arguments, respecting quoted strings and interior macro calls. """ # TODO: Enable escaped tildes! # First, find quoted regions: qrs = list(utils.find_quoted_regions(content, qc='`').keys()) # Next, find macro calls: mrs = [] i = 0 while i < len(content): # Look for a macro-start: m = MACRO_START.search(content, i) if m: r = utils.find_region(qrs, m.start()) if r: i = r[1]+1 # skip to end of quoted region continue # Otherwise this try: mend = utils.matching_brace(content, m.start(), qc='`') mrs.append((m.start(), mend)) i = mend + 1 except utils.UnmatchedError: # malformed macro: skip to the end of the macro start i = m.end() + 1 else: # No more macros found break # Combine macro- and quote-excluded regions: exrs = sorted(qrs + mrs) # Finally, find non-excluded delimiters: result = [] i = 0 while i < len(content): # Compute first-non-excluded-delimiter index: try: di = content.index('~', i) except ValueError: di = len(content) for exr in exrs: if di >= len(content): break if exr[0] <= di and exr[1]>= di: try: di = content.index('~', exr[1]+1) except ValueError: di = len(content) result.append(content[i:di]) i = di + 1 return result
def detailed_stack_list(project=None): region = utils.find_region() results = core.active_aws_stacks(region, formatter=None) all_stacks = dict([(i.stack_name, vars(i)) for i in results]) if project: return {k: v for k, v in all_stacks.items() if k.startswith("%s-" % project)} return all_stacks
def detailed_stack_list(project=None): region = utils.find_region() results = core.active_aws_stacks(region, formatter=None) all_stacks = dict([(i.stack_name, vars(i)) for i in results]) if project: return { k: v for k, v in all_stacks.items() if k.startswith("%s-" % project) } return all_stacks
def call(*args, **kwargs): ss = core.steady_aws_stacks(utils.find_region()) keys = lmap(first, ss) idx = dict(zip(keys, ss)) helpfn = lambda pick: idx[pick][1] if not keys: print('\nno AWS stacks *in a steady state* exist, cannot continue.') return stackname = first(args) or os.environ.get('INSTANCE') if not stackname or stackname not in keys: stackname = utils._pick("stack", sorted(keys), helpfn=helpfn, default_file=deffile('.active-stack')) return func(stackname, *args[1:], **kwargs)
def call(*args, **kwargs): stackname = first(args) or os.environ.get('INSTANCE') region = utils.find_region(stackname) if stackname: args = args[1:] return func(stackname, *args, **kwargs) asl = core.active_stack_names(region) if not asl: raise RuntimeError('\nno AWS stacks *in an active state* exist, cannot continue.') if not stackname or stackname not in asl: stackname = utils._pick("stack", asl, default_file=deffile('.active-stack')) args = args[1:] return func(stackname, *args, **kwargs)
def parse_function(filename, regions, ext='png'): image_string = tf.read_file(filename) if ext == 'jpg': image_decoded = tf.image.decode_jpeg(image_string, channels=3) else: image_decoded = tf.image.decode_png(image_string, channels=3) image_decoded = tf.cast(image_decoded, tf.float32) / 255.0 output = {} for region_name in regions: output[region_name] = find_region(image_decoded, regions[region_name], is_tf=True) return output
def _wrapper(stackname=None, *args, **kwargs): region = utils.find_region(stackname) asl = core.active_stack_names(region) if not asl: print('\nno AWS stacks exist, cannot continue.') return def pname_startswith(stack): for pname in plist: if stack.startswith(pname): return stack asl = lfilter(pname_startswith, asl) if not stackname or stackname not in asl: stackname = utils._pick("stack", asl) return func(stackname, *args, **kwargs)
def call(*args, **kwargs): ss = core.steady_aws_stacks(utils.find_region()) keys = lmap(first, ss) idx = dict(zip(keys, ss)) helpfn = lambda pick: idx[pick][1] if not keys: print( '\nno AWS stacks *in a steady state* exist, cannot continue.') return stackname = first(args) or os.environ.get('INSTANCE') if not stackname or stackname not in keys: stackname = utils._pick("stack", sorted(keys), helpfn=helpfn, default_file=deffile('.active-stack')) return func(stackname, *args[1:], **kwargs)
def call(*args, **kwargs): stackname = first(args) or os.environ.get('INSTANCE') region = utils.find_region(stackname) if stackname: args = args[1:] return func(stackname, *args, **kwargs) asl = core.active_stack_names(region) if not asl: raise RuntimeError( '\nno AWS stacks *in an active state* exist, cannot continue.') if not stackname or stackname not in asl: stackname = utils._pick("stack", asl, default_file=deffile('.active-stack')) args = args[1:] return func(stackname, *args, **kwargs)
def aws_stack_list(): "returns a list of realized stacks. does not include deleted stacks" region = utils.find_region() return core.active_stack_names(region)
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'): "runs the given command on all aws instances. given command must escape double quotes" with stack_conn(core.find_master(utils.find_region())): remote_sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" % locals())
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'): "runs the given command on all aws instances. given command must escape double quotes" with stack_conn(core.find_master(utils.find_region())): sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" % locals())
regions = utils.search_region(model['signs2ineq'], model['signs2Ab'], signs) print(len(regions)) K = 100 xx = [np.linspace(-10, 10, K)] * Ds[0] xx = np.meshgrid(*xx) xx = np.vstack([x.flatten() for x in xx]).T yy = np.zeros((int(K ** Ds[0]), Ds[-1])) yy2 = np.zeros((int(K ** Ds[0]), Ds[-1])) As = np.array([regions[s]['Ab'][0] for s in regions]) Bs = np.array([regions[s]['Ab'][1] for s in regions]) w = utils.find_region(xx, regions, model['input2signs']) for i in range(len(regions)): yy[w == i] = np.dot(xx[w == i], As[i].T) + Bs[i] for k, x in enumerate(xx): yy2[k] = model['g'](x) error1.append(np.abs(yy - yy2).mean()) error2.append(((yy - yy2)**2).mean()) print(error1, error2) print(np.array(error1).reshape((3, 6))) print(np.array(error2).reshape((3, 6)))
def update_salt_master(region=None): "convenience. update the version of Salt installed on the master-server." region = region or utils.find_region() current_master_stackname = core.find_master(region) return update_salt(current_master_stackname)
def remaster_all(*pname_list): "calls `remaster` on *all* projects or just a subset of projects" # there should only be one master-server instance at a time. # multiple masters is bad news. assumptions break and it gets complicated quickly. new_master_stackname = "master-server--2018-04-09-2" LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) # we can optionally pass in a list of projects to target # this allows us to partition up the projects and have many of these # remastering efforts happening concurrently if pname_list: more_ignore = [p for p in ec2stacks if p not in pname_list] ec2stacks = exsubdict(ec2stacks, more_ignore) pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # TODO: skip any stacks without ec2 instances # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open( 'remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: # when would this ever be the case? # `core.active_stack_names` doesn't discriminate against any list of projects # it returns *all* steady stack names. if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn( "failed to remaster %s, stopping further remasters to project %r", stackname, pname) break # print a reminder of which stack was just updated print("\n(%s)\n" % stackname) open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) LOG.info("ctrl-c again to exit process entirely") time.sleep(2) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")
yy2 = np.zeros((K * K, 1)) allA, allB = [], [] for flips in regions: A_w, b_w = g(flips) allA.append(A_w) allB.append(b_w) allA = np.array(allA) allB = np.array(allB) flips = np.array(regions) w = utils.find_region(xx, flips, f) for i in range(len(regions)): yy[w == i] = np.dot(xx[w == i], allA[i].T) + allB[i] for k, x in enumerate(xx): yy2[k] = h(x) plt.figure(figsize=(12, 4)) plt.subplot(141) plt.imshow(w.reshape(K, K), aspect='auto', cmap='prism') plt.xticks([]) plt.yticks([]) plt.subplot(142)
def remaster_all(new_master_stackname): LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) def sortbypname(n): unknown = 9 porder = { #'observer': 1, 'elife-metrics': 2, 'lax': 3, 'basebox': 4, 'containers': 5, 'elife-dashboard': 6, 'elife-ink': 7 } return porder.get(n, unknown) # pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open('remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) open('remastered.txt', 'a').write("%s\n" % stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn("failed to remaster %s, stopping further remasters to project %r", stackname, pname) break open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) time.sleep(1) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")
def remaster_all(new_master_stackname): LOG.info('new master is: %s', new_master_stackname) ec2stacks = project.ec2_projects() ignore = [ 'master-server', 'jats4r', ] ec2stacks = exsubdict(ec2stacks, ignore) def sortbypname(n): unknown = 9 porder = { #'observer': 1, 'elife-metrics': 2, 'lax': 3, 'basebox': 4, 'containers': 5, 'elife-dashboard': 6, 'elife-ink': 7 } return porder.get(n, unknown) # pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically # only update ec2 instances in the same region as the new master region = utils.find_region(new_master_stackname) active_stacks = core.active_stack_names(region) stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks) def sortbyenv(n): adhoc = 0 # do these first order = { 'continuumtest': 1, 'ci': 2, 'end2end': 3, 'prod': 4, # update prod last } pname, iid = core.parse_stackname(n) return order.get(iid, adhoc) remastered_list = open( 'remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else [] for pname in pname_list: if pname not in stack_idx: continue project_stack_list = sorted(stack_idx[pname], key=sortbyenv) LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list))) try: for stackname in project_stack_list: try: if stackname in remastered_list: LOG.info("already updated, skipping stack: %s", stackname) open('remastered.txt', 'a').write("%s\n" % stackname) continue LOG.info("*" * 80) LOG.info("updating: %s" % stackname) utils.get_input('continue? ctrl-c to quit') if not remaster(stackname, new_master_stackname): LOG.warn( "failed to remaster %s, stopping further remasters to project %r", stackname, pname) break open('remastered.txt', 'a').write("%s\n" % stackname) except KeyboardInterrupt: LOG.warn("ctrl-c, skipping stack: %s", stackname) time.sleep(1) except BaseException: LOG.exception("unhandled exception updating stack: %s", stackname) except KeyboardInterrupt: LOG.warn("quitting") break LOG.info("wrote 'remastered.txt'")
def update_salt_master(region=None): "update the version of Salt installed on the master-server." region = region or utils.find_region() current_master_stackname = core.find_master(region) return remaster(current_master_stackname, current_master_stackname)