def update(master_stackname=None):
"same as `cfn.update` but also removes any orphaned minion keys"
master_stackname = master_stackname or core.find_master(utils.find_region())
bootstrap.update_stack(master_stackname, service_list=[
'ec2' # master-server should be a self-contained EC2 instance
])
bootstrap.remove_all_orphaned_keys(master_stackname)
def split_macro_args(content):
"""
Splits macro content into different arguments, respecting quoted strings and
interior macro calls.
"""
# TODO: Enable escaped tildes!
# First, find quoted regions:
qrs = list(utils.find_quoted_regions(content, qc='`').keys())
# Next, find macro calls:
mrs = []
i = 0
while i < len(content):
# Look for a macro-start:
m = MACRO_START.search(content, i)
if m:
r = utils.find_region(qrs, m.start())
if r:
i = r[1]+1 # skip to end of quoted region
continue
# Otherwise this
try:
mend = utils.matching_brace(content, m.start(), qc='`')
mrs.append((m.start(), mend))
i = mend + 1
except utils.UnmatchedError:
# malformed macro: skip to the end of the macro start
i = m.end() + 1
else:
# No more macros found
break
# Combine macro- and quote-excluded regions:
exrs = sorted(qrs + mrs)
# Finally, find non-excluded delimiters:
result = []
i = 0
while i < len(content):
# Compute first-non-excluded-delimiter index:
try:
di = content.index('~', i)
except ValueError:
di = len(content)
for exr in exrs:
if di >= len(content):
break
if exr[0] <= di and exr[1]>= di:
try:
di = content.index('~', exr[1]+1)
except ValueError:
di = len(content)
result.append(content[i:di])
i = di + 1
return result
def update(master_stackname=None):
"same as `cfn.update` but also removes any orphaned minion keys"
master_stackname = master_stackname or core.find_master(utils.find_region())
bootstrap.update_stack(master_stackname, service_list=[
'ec2' # master-server should be a self-contained EC2 instance
])
bootstrap.remove_all_orphaned_keys(master_stackname)
def detailed_stack_list(project=None):
region = utils.find_region()
results = core.active_aws_stacks(region, formatter=None)
all_stacks = dict([(i.stack_name, vars(i)) for i in results])
if project:
return {k: v for k, v in all_stacks.items() if k.startswith("%s-" % project)}
return all_stacks
def detailed_stack_list(project=None):
region = utils.find_region()
results = core.active_aws_stacks(region, formatter=None)
all_stacks = dict([(i.stack_name, vars(i)) for i in results])
if project:
return {
k: v
for k, v in all_stacks.items() if k.startswith("%s-" % project)
}
return all_stacks
def call(*args, **kwargs):
ss = core.steady_aws_stacks(utils.find_region())
keys = lmap(first, ss)
idx = dict(zip(keys, ss))
helpfn = lambda pick: idx[pick][1]
if not keys:
print('\nno AWS stacks *in a steady state* exist, cannot continue.')
return
stackname = first(args) or os.environ.get('INSTANCE')
if not stackname or stackname not in keys:
stackname = utils._pick("stack", sorted(keys), helpfn=helpfn, default_file=deffile('.active-stack'))
return func(stackname, *args[1:], **kwargs)
def call(*args, **kwargs):
stackname = first(args) or os.environ.get('INSTANCE')
region = utils.find_region(stackname)
if stackname:
args = args[1:]
return func(stackname, *args, **kwargs)
asl = core.active_stack_names(region)
if not asl:
raise RuntimeError('\nno AWS stacks *in an active state* exist, cannot continue.')
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl, default_file=deffile('.active-stack'))
args = args[1:]
return func(stackname, *args, **kwargs)
def parse_function(filename, regions, ext='png'):
image_string = tf.read_file(filename)
if ext == 'jpg':
image_decoded = tf.image.decode_jpeg(image_string, channels=3)
else:
image_decoded = tf.image.decode_png(image_string, channels=3)
image_decoded = tf.cast(image_decoded, tf.float32) / 255.0
output = {}
for region_name in regions:
output[region_name] = find_region(image_decoded,
regions[region_name],
is_tf=True)
return output
def _wrapper(stackname=None, *args, **kwargs):
region = utils.find_region(stackname)
asl = core.active_stack_names(region)
if not asl:
print('\nno AWS stacks exist, cannot continue.')
return
def pname_startswith(stack):
for pname in plist:
if stack.startswith(pname):
return stack
asl = lfilter(pname_startswith, asl)
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl)
return func(stackname, *args, **kwargs)
def _wrapper(stackname=None, *args, **kwargs):
region = utils.find_region(stackname)
asl = core.active_stack_names(region)
if not asl:
print('\nno AWS stacks exist, cannot continue.')
return
def pname_startswith(stack):
for pname in plist:
if stack.startswith(pname):
return stack
asl = lfilter(pname_startswith, asl)
if not stackname or stackname not in asl:
stackname = utils._pick("stack", asl)
return func(stackname, *args, **kwargs)
def call(*args, **kwargs):
ss = core.steady_aws_stacks(utils.find_region())
keys = lmap(first, ss)
idx = dict(zip(keys, ss))
helpfn = lambda pick: idx[pick][1]
if not keys:
print(
'\nno AWS stacks *in a steady state* exist, cannot continue.')
return
stackname = first(args) or os.environ.get('INSTANCE')
if not stackname or stackname not in keys:
stackname = utils._pick("stack",
sorted(keys),
helpfn=helpfn,
default_file=deffile('.active-stack'))
return func(stackname, *args[1:], **kwargs)
def call(*args, **kwargs):
stackname = first(args) or os.environ.get('INSTANCE')
region = utils.find_region(stackname)
if stackname:
args = args[1:]
return func(stackname, *args, **kwargs)
asl = core.active_stack_names(region)
if not asl:
raise RuntimeError(
'\nno AWS stacks *in an active state* exist, cannot continue.')
if not stackname or stackname not in asl:
stackname = utils._pick("stack",
asl,
default_file=deffile('.active-stack'))
args = args[1:]
return func(stackname, *args, **kwargs)
def aws_stack_list():
"returns a list of realized stacks. does not include deleted stacks"
region = utils.find_region()
return core.active_stack_names(region)
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'):
"runs the given command on all aws instances. given command must escape double quotes"
with stack_conn(core.find_master(utils.find_region())):
remote_sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" %
locals())
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'):
"runs the given command on all aws instances. given command must escape double quotes"
with stack_conn(core.find_master(utils.find_region())):
sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" % locals())
regions = utils.search_region(model['signs2ineq'], model['signs2Ab'],
signs)
print(len(regions))
K = 100
xx = [np.linspace(-10, 10, K)] * Ds[0]
xx = np.meshgrid(*xx)
xx = np.vstack([x.flatten() for x in xx]).T
yy = np.zeros((int(K ** Ds[0]), Ds[-1]))
yy2 = np.zeros((int(K ** Ds[0]), Ds[-1]))
As = np.array([regions[s]['Ab'][0] for s in regions])
Bs = np.array([regions[s]['Ab'][1] for s in regions])
w = utils.find_region(xx, regions, model['input2signs'])
for i in range(len(regions)):
yy[w == i] = np.dot(xx[w == i], As[i].T) + Bs[i]
for k, x in enumerate(xx):
yy2[k] = model['g'](x)
error1.append(np.abs(yy - yy2).mean())
error2.append(((yy - yy2)**2).mean())
print(error1, error2)
print(np.array(error1).reshape((3, 6)))
print(np.array(error2).reshape((3, 6)))
def update_salt_master(region=None):
"convenience. update the version of Salt installed on the master-server."
region = region or utils.find_region()
current_master_stackname = core.find_master(region)
return update_salt(current_master_stackname)
def remaster_all(*pname_list):
"calls `remaster` on *all* projects or just a subset of projects"
# there should only be one master-server instance at a time.
# multiple masters is bad news. assumptions break and it gets complicated quickly.
new_master_stackname = "master-server--2018-04-09-2"
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
# we can optionally pass in a list of projects to target
# this allows us to partition up the projects and have many of these
# remastering efforts happening concurrently
if pname_list:
more_ignore = [p for p in ec2stacks if p not in pname_list]
ec2stacks = exsubdict(ec2stacks, more_ignore)
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# TODO: skip any stacks without ec2 instances
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
# when would this ever be the case?
# `core.active_stack_names` doesn't discriminate against any list of projects
# it returns *all* steady stack names.
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
# print a reminder of which stack was just updated
print("\n(%s)\n" % stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
LOG.info("ctrl-c again to exit process entirely")
time.sleep(2)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
yy2 = np.zeros((K * K, 1))
allA, allB = [], []
for flips in regions:
A_w, b_w = g(flips)
allA.append(A_w)
allB.append(b_w)
allA = np.array(allA)
allB = np.array(allB)
flips = np.array(regions)
w = utils.find_region(xx, flips, f)
for i in range(len(regions)):
yy[w == i] = np.dot(xx[w == i], allA[i].T) + allB[i]
for k, x in enumerate(xx):
yy2[k] = h(x)
plt.figure(figsize=(12, 4))
plt.subplot(141)
plt.imshow(w.reshape(K, K), aspect='auto', cmap='prism')
plt.xticks([])
plt.yticks([])
plt.subplot(142)
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open('remastered.txt', 'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s", stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn("failed to remaster %s, stopping further remasters to project %r", stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s", stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def remaster_all(new_master_stackname):
LOG.info('new master is: %s', new_master_stackname)
ec2stacks = project.ec2_projects()
ignore = [
'master-server',
'jats4r',
]
ec2stacks = exsubdict(ec2stacks, ignore)
def sortbypname(n):
unknown = 9
porder = {
#'observer': 1,
'elife-metrics': 2,
'lax': 3,
'basebox': 4,
'containers': 5,
'elife-dashboard': 6,
'elife-ink': 7
}
return porder.get(n, unknown)
# pname_list = sorted(ec2stacks.keys(), key=sortbypname) # lets do this alphabetically
pname_list = sorted(ec2stacks.keys()) # lets do this alphabetically
# only update ec2 instances in the same region as the new master
region = utils.find_region(new_master_stackname)
active_stacks = core.active_stack_names(region)
stack_idx = mkidx(lambda v: core.parse_stackname(v)[0], active_stacks)
def sortbyenv(n):
adhoc = 0 # do these first
order = {
'continuumtest': 1,
'ci': 2,
'end2end': 3,
'prod': 4, # update prod last
}
pname, iid = core.parse_stackname(n)
return order.get(iid, adhoc)
remastered_list = open(
'remastered.txt',
'r').read().splitlines() if os.path.exists('remastered.txt') else []
for pname in pname_list:
if pname not in stack_idx:
continue
project_stack_list = sorted(stack_idx[pname], key=sortbyenv)
LOG.info("%r instances: %s" % (pname, ", ".join(project_stack_list)))
try:
for stackname in project_stack_list:
try:
if stackname in remastered_list:
LOG.info("already updated, skipping stack: %s",
stackname)
open('remastered.txt', 'a').write("%s\n" % stackname)
continue
LOG.info("*" * 80)
LOG.info("updating: %s" % stackname)
utils.get_input('continue? ctrl-c to quit')
if not remaster(stackname, new_master_stackname):
LOG.warn(
"failed to remaster %s, stopping further remasters to project %r",
stackname, pname)
break
open('remastered.txt', 'a').write("%s\n" % stackname)
except KeyboardInterrupt:
LOG.warn("ctrl-c, skipping stack: %s", stackname)
time.sleep(1)
except BaseException:
LOG.exception("unhandled exception updating stack: %s",
stackname)
except KeyboardInterrupt:
LOG.warn("quitting")
break
LOG.info("wrote 'remastered.txt'")
def update_salt_master(region=None):
"update the version of Salt installed on the master-server."
region = region or utils.find_region()
current_master_stackname = core.find_master(region)
return remaster(current_master_stackname, current_master_stackname)
def aws_stack_list():
"returns a list of realized stacks. does not include deleted stacks"
region = utils.find_region()
return core.active_stack_names(region)
