def _pick_node(instance_list, node):
instance_list = sorted(instance_list,
key=lambda n: tags2dict(n.tags)['Name'])
info = [n for n in instance_list] #
def helpfn(pick):
node = pick - 1
return "%s (%s, %s)" % (tags2dict(info[node].tags)['Name'],
info[node].id, info[node].public_ip_address)
num_instances = len(instance_list)
if num_instances > 1:
if not node:
node = utils._pick('node',
list(range(1, num_instances + 1)),
helpfn=helpfn)
node = int(node) - 1
instance = instance_list[int(node)]
else:
ensure(
node == 1 or node is None,
"You can't specify a node different from 1 for a single-instance stack"
)
instance = instance_list[0]
ensure(
instance.public_ip_address,
"Selected instance does not have a public ip address, are you sure it's running?"
)
return instance
def _retrieve_build_vars():
"""wrapper around `read_from_current_host` with integrity checks. returns buildvars for the current instance.
raises AssertionError on bad data."""
try:
buildvars = read_from_current_host()
LOG.debug('build vars: %s', buildvars)
# buildvars exist
ensure(
isinstance(buildvars, dict),
'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.'
% buildvars)
# nothing important is missing
missing_keys = core_utils.missingkeys(
buildvars, ['stackname', 'instance_id', 'branch', 'revision'])
ensure(
len(missing_keys) == 0,
'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.'
% missing_keys)
return buildvars
except (ValueError, AssertionError, JSONDecodeError) as ex:
LOG.exception(ex)
raise
def test_ensure(self):
utils.ensure(True, "True should allow ensure() to continue")
self.assertRaises(AssertionError, utils.ensure, False, "Error message")
class CustomException(Exception):
pass
self.assertRaises(CustomException, utils.ensure, False, "Error message", CustomException)
def wrapper(*args, **kwargs):
import master
ensure(
master.server_access(),
"this command requires access to the master server. you don't have it."
)
return fn(*args, **kwargs)
def test_ensure(self):
utils.ensure(True, "True should allow ensure() to continue")
self.assertRaises(AssertionError, utils.ensure, False, "Error message")
class CustomException(Exception):
pass
self.assertRaises(CustomException, utils.ensure, False,
"Error message", CustomException)
def context(pname, output_format=None):
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(), "unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(cfngen.build_context(pname, stackname=core.mk_stackname(pname, "test")))
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
ensure(output_format in [None, 'json', 'yaml'],
"unknown output format %r" % output_format)
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.ordered_dump,
None: lambda v: v
}
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(), "unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(),
"unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def context(pname, output_format=None):
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(),
"unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(
cfngen.build_context(pname, stackname=core.mk_stackname(pname,
"test")))
def _pick_node(instance_list, node):
num_instances = len(instance_list)
if num_instances > 1:
if not node:
node = utils._pick('node', range(1, num_instances + 1))
node = int(node) - 1
instance = instance_list[int(node)]
else:
instance = instance_list[0]
core_utils.ensure(
instance.ip_address is not None,
"Selected instance does not have a public ip address, are you sure it's running?"
)
return instance
def parse_validate_repolist(fdata, *repolist):
"returns a list of triples"
known_formulas = fdata.get('formula-dependencies', [])
known_formulas.extend([fdata['formula-repo'], fdata['private-repo']])
known_formula_map = OrderedDict(
zip(map(os.path.basename, known_formulas), known_formulas))
arglist = []
for user_string in repolist:
if '@' not in user_string:
print('skipping %r, no revision component' % user_string)
continue
repo, rev = user_string.split('@')
if not rev.strip():
print('skipping %r, empty revision component' % user_string)
continue
if repo not in known_formula_map:
print('skipping %r, unknown formula. known formulas: %s' %
(repo, ', '.join(known_formula_map.keys())))
continue
arglist.append((repo, known_formula_map[repo], rev))
# test given revisions actually exist in formulas
for name, _, revision in arglist:
path = join(config.PROJECT_PATH, "cloned-projects", name)
if not os.path.exists(path):
LOG.warn(
"couldn't find formula %r locally, revision check skipped",
path)
continue
with lcd(path), settings(warn_only=True):
ensure(
local("git fetch --quiet")['succeeded'],
"failed to fetch remote refs for %s" % path)
ensure(
local("git cat-file -e %s^{commit}" % revision)['succeeded'],
"failed to find ref %r in %s" % (revision, name))
return arglist
def generate_stack_from_input(pname, instance_id=None, alt_config=None):
"""creates a new CloudFormation file for the given project."""
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
checks.ensure_stack_does_not_exist(stackname)
more_context = {'stackname': stackname}
pdata = project.project_data(pname)
if alt_config:
ensure(
'aws-alt' in pdata,
"alternative configuration name given, but project has no alternate configurations"
)
# prompt user for alternate configurations
if pdata['aws-alt']:
default = 'skip'
def helpfn(altkey):
if altkey == default:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
if instance_id in pdata['aws-alt'].keys():
LOG.info(
"instance-id found in known alternative configurations. using configuration %r",
instance_id)
more_context['alt-config'] = instance_id
else:
alt_config_choices = [default] + list(pdata['aws-alt'].keys())
if not alt_config:
alt_config = utils._pick('alternative config',
alt_config_choices,
helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
# TODO: return the templates used here, so that they can be passed down to
# bootstrap.create_stack() without relying on them implicitly existing
# on the filesystem
cfngen.generate_stack(pname, **more_context)
return stackname
def _pick_node(instance_list, node):
instance_list = sorted(instance_list, key=lambda n: tags2dict(n.tags)['Name'])
info = [n for n in instance_list] #
def helpfn(pick):
node = pick - 1
return "%s (%s, %s)" % (tags2dict(info[node].tags)['Name'], info[node].id, info[node].public_ip_address)
num_instances = len(instance_list)
if num_instances > 1:
if not node:
node = utils._pick('node', list(range(1, num_instances + 1)), helpfn=helpfn)
node = int(node) - 1
instance = instance_list[int(node)]
else:
ensure(node == 1 or node is None, "You can't specify a node different from 1 for a single-instance stack")
instance = instance_list[0]
ensure(instance.public_ip_address, "Selected instance does not have a public ip address, are you sure it's running?")
return instance
def parse_validate_repolist(fdata, *repolist):
"returns a list of triples"
known_formulas = fdata.get('formula-dependencies', [])
known_formulas.extend([
fdata['formula-repo'],
fdata['private-repo']
])
known_formula_map = OrderedDict(zip(map(os.path.basename, known_formulas), known_formulas))
arglist = []
for user_string in repolist:
if '@' not in user_string:
print('skipping %r, no revision component' % user_string)
continue
repo, rev = user_string.split('@')
if not rev.strip():
print('skipping %r, empty revision component' % user_string)
continue
if repo not in known_formula_map:
print('skipping %r, unknown formula. known formulas: %s' % (repo, ', '.join(known_formula_map.keys())))
continue
arglist.append((repo, known_formula_map[repo], rev))
# test given revisions actually exist in formulas
for name, _, revision in arglist:
path = join(config.PROJECT_PATH, "cloned-projects", name)
if not os.path.exists(path):
LOG.warn("couldn't find formula %r locally, revision check skipped", path)
continue
with lcd(path), settings(warn_only=True):
ensure(local("git fetch --quiet").succeeded, "failed to fetch remote refs for %s" % path)
ensure(local("git cat-file -e %s^{commit}" % revision).succeeded, "failed to find ref %r in %s" % (revision, name))
return arglist
def launch(pname, instance_id=None, alt_config='standalone', *repolist):
stackname = cfn.generate_stack_from_input(pname, instance_id, alt_config)
pdata = core.project_data_for_stackname(stackname)
# ensure given alt config has masterless=True
# todo: can the choices presented to the user remove non-masterless alt-configs?
ensure(pdata['aws-alt'], "project has no alternate configurations")
ensure(alt_config in pdata['aws-alt'], "unknown alt-config %r" % alt_config)
ensure(pdata['aws-alt'][alt_config]['ec2']['masterless'], "alternative configuration %r has masterless=False" % alt_config)
formula_revisions = parse_validate_repolist(pdata, *repolist)
# todo: this is good UX but was simply debug output that got left in.
# a better summary of what is to be created could be printed out,
# preferably after the templates are printed out but before confirmation.
LOG.info('attempting to create masterless stack:')
LOG.info('stackname:\t' + stackname)
LOG.info('region:\t' + pdata['aws']['region'])
LOG.info('formula_revisions:\t%s' % pformat(formula_revisions))
if core.is_master_server_stack(stackname):
checks.ensure_can_access_builder_private(pname)
checks.ensure_stack_does_not_exist(stackname)
bootstrap.create_stack(stackname)
LOG.info('updating stack %s', stackname)
bootstrap.update_stack(stackname, service_list=['ec2', 'sqs', 's3'], formula_revisions=formula_revisions)
def launch(pname, instance_id=None, alt_config='standalone', *repolist):
stackname = cfn.generate_stack_from_input(pname, instance_id, alt_config)
pdata = core.project_data_for_stackname(stackname)
# ensure given alt config has masterless=True
ensure(pdata['aws-alt'], "project has no alternate configurations")
ensure(alt_config in pdata['aws-alt'],
"unknown alt-config %r" % alt_config)
ensure(pdata['aws-alt'][alt_config]['ec2']['masterless'],
"alternative configuration %r has masterless=False" % alt_config)
formula_revisions = parse_validate_repolist(pdata, *repolist)
LOG.info('attempting to create masterless stack:')
LOG.info('stackname:\t' + stackname)
LOG.info('region:\t' + pdata['aws']['region'])
LOG.info('formula_revisions:\t%s' % pformat(formula_revisions))
if core.is_master_server_stack(stackname):
checks.ensure_can_access_builder_private(pname)
checks.ensure_stack_does_not_exist(stackname)
bootstrap.create_stack(stackname)
LOG.info('updating stack %s', stackname)
bootstrap.update_stack(stackname,
service_list=['ec2', 'sqs', 's3'],
formula_revisions=formula_revisions)
def _validate():
"returns a pair of (type, build data) for the given instance. type is either 'old', 'abbrev' or 'full'"
try:
buildvars = read_from_current_host()
LOG.debug('build vars: %s', buildvars)
core_utils.ensure(
isinstance(buildvars, dict),
'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.',
buildvars)
missing_keys = core_utils.missingkeys(buildvars, [
'stackname', 'instance_id', 'branch', 'revision',
'is_prod_instance'
])
core_utils.ensure(
len(missing_keys) == 0,
'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.'
% missing_keys)
return buildvars
except (ValueError, AssertionError) as ex:
LOG.exception(ex)
raise
def _retrieve_build_vars():
"""wrapper around `read_from_current_host` with integrity checks. returns buildvars for the current instance.
raises AssertionError on bad data."""
try:
buildvars = read_from_current_host()
LOG.debug('build vars: %s', buildvars)
# buildvars exist
ensure(isinstance(buildvars, dict), 'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.' % buildvars)
# nothing important is missing
missing_keys = core_utils.missingkeys(buildvars, ['stackname', 'instance_id', 'branch', 'revision'])
ensure(
len(missing_keys) == 0,
'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.' % missing_keys
)
return buildvars
except (ValueError, AssertionError) as ex:
LOG.exception(ex)
raise
def generate_stack_from_input(pname, instance_id=None, alt_config=None):
"""creates a new CloudFormation file for the given project."""
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
checks.ensure_stack_does_not_exist(stackname)
more_context = {'stackname': stackname}
pdata = project.project_data(pname)
if alt_config:
ensure('aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations")
# prompt user for alternate configurations
if pdata['aws-alt']:
default = 'skip'
def helpfn(altkey):
if altkey == default:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
if instance_id in pdata['aws-alt'].keys():
LOG.info("instance-id found in known alternative configurations. using configuration %r", instance_id)
more_context['alt-config'] = instance_id
else:
alt_config_choices = [default] + list(pdata['aws-alt'].keys())
if not alt_config:
alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
# TODO: return the templates used here, so that they can be passed down to
# bootstrap.create_stack() without relying on them implicitly existing
# on the filesystem
cfngen.generate_stack(pname, **more_context)
return stackname
def _pick_node(instance_list, node):
info = [n for n in instance_list]
def helpfn(pick):
node = pick - 1
return info[node]
num_instances = len(instance_list)
if num_instances > 1:
if not node:
# TODO print some more info: ip address, instance id
node = utils._pick('node',
range(1, num_instances + 1),
helpfn=helpfn)
node = int(node) - 1
instance = instance_list[int(node)]
else:
assert node == 1 or node is None, "You can't specify a node different from 1 for a single-instance stack"
instance = instance_list[0]
core_utils.ensure(
instance.ip_address is not None,
"Selected instance does not have a public ip address, are you sure it's running?"
)
return instance
def test_ensure(self):
utils.ensure(True, "True should allow ensure() to continue")
self.assertRaises(AssertionError,
lambda: utils.ensure(False, "Error message"))
self.assertRaises(
AssertionError,
lambda: utils.ensure(False, "Error message: %s", "argument"))
class CustomException(Exception):
pass
self.assertRaises(
CustomException, lambda: utils.ensure(
False, "Error message", exception_class=CustomException))
self.assertRaises(
CustomException,
lambda: utils.ensure(False,
"Error message: %s",
"argument",
exception_class=CustomException))
self.assertRaises(
ValueError, lambda: utils.ensure(
False, "Error message", random_argument=CustomException))
def launch(pname, instance_id=None, alt_config='standalone', *repolist):
stackname = cfn.generate_stack_from_input(pname, instance_id, alt_config)
pdata = core.project_data_for_stackname(stackname)
# ensure given alt config has masterless=True
ensure(pdata['aws-alt'], "project has no alternate configurations")
ensure(alt_config in pdata['aws-alt'], "unknown alt-config %r" % alt_config)
ensure(pdata['aws-alt'][alt_config]['ec2']['masterless'], "alternative configuration %r has masterless=False" % alt_config)
formula_revisions = parse_validate_repolist(pdata, *repolist)
LOG.info('attempting to create masterless stack:')
LOG.info('stackname:\t' + stackname)
LOG.info('region:\t' + pdata['aws']['region'])
LOG.info('formula_revisions:\t%s' % pformat(formula_revisions))
if core.is_master_server_stack(stackname):
checks.ensure_can_access_builder_private(pname)
checks.ensure_stack_does_not_exist(stackname)
bootstrap.create_stack(stackname)
LOG.info('updating stack %s', stackname)
bootstrap.update_stack(stackname, service_list=['ec2', 'sqs', 's3'], formula_revisions=formula_revisions)
def wrapper(stackname=None, *args, **kwargs):
ctx = context_handler.load_context(stackname)
ensure(stackname and ctx['ec2']['masterless'], "this command requires a masterless instance.")
return fn(stackname, *args, **kwargs)
def wrapper(*args, **kwargs):
import master
ensure(master.server_access(), "this command requires access to the master server. you don't have it.")
return fn(*args, **kwargs)
def wrapper(stackname=None, *args, **kwargs):
ctx = context_handler.load_context(stackname)
ensure(stackname and ctx['ec2']['masterless'], "this command requires a masterless instance.")
return fn(stackname, *args, **kwargs)
def check_user_input(pname, instance_id=None, alt_config=None):
"marshals user input and checks it for correctness"
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
pdata = project.project_data(pname)
# alt-config given, die if it doesn't exist
if alt_config:
ensure(
'aws-alt' in pdata,
"alt-config %r given, but project has no alternate configurations"
% alt_config)
# if the requested instance-id matches a known alt-config, we'll use that alt-config. warn user.
if instance_id in pdata['aws-alt'].keys():
LOG.warn("instance-id %r found in alt-config list, using that.",
instance_id)
alt_config = instance_id
# no alt-config given but alt-config options exist, prompt user
if not alt_config and pdata['aws-alt']:
default_choice = 'skip'
def helpfn(altkey):
if altkey == default_choice:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
alt_config_choice_list = [default_choice] + list(
pdata['aws-alt'].keys())
alt_config_choice = utils._pick('alternative config',
alt_config_choice_list,
helpfn=helpfn)
if alt_config_choice != default_choice:
alt_config = alt_config_choice
# check the alt-config isn't unique and if it *is* unique, that an instance using it doesn't exist yet.
# note: it is *technically* possible that an instance is using a unique configuration but
# that its instance-id *is not* the name of the alt-config passed in.
# For example, if `journal--prod` didn't exist, I could create `journal--foo` using the `prod` config.
if alt_config and alt_config in pdata['aws-alt'] and pdata['aws-alt'][
alt_config]['unique']:
dealbreaker = core.mk_stackname(pname, alt_config)
# "project 'journal' config 'prod' is marked as unique!"
# "checking for any instance named 'journal--prod' ..."
print("project %r config %r is marked as unique!" %
(pname, alt_config))
print("checking for any instance named %r ..." % (dealbreaker, ))
try:
checks.ensure_stack_does_not_exist(dealbreaker)
except checks.StackAlreadyExistsProblem:
# "stack 'journal--prod' exists, cannot re-use unique configuration 'prod'"
msg = "stack %r exists, cannot re-use unique configuration %r." % (
dealbreaker, alt_config)
raise TaskExit(msg)
# check that the instance we want to create doesn't exist
try:
print("checking %r doesn't exist." % stackname)
checks.ensure_stack_does_not_exist(stackname)
except checks.StackAlreadyExistsProblem as e:
msg = 'stack %r already exists.' % e.stackname
raise TaskExit(msg)
more_context = {'stackname': stackname}
if alt_config:
more_context['alt-config'] = alt_config
return more_context
