def _pick_node(instance_list, node): instance_list = sorted(instance_list, key=lambda n: tags2dict(n.tags)['Name']) info = [n for n in instance_list] # def helpfn(pick): node = pick - 1 return "%s (%s, %s)" % (tags2dict(info[node].tags)['Name'], info[node].id, info[node].public_ip_address) num_instances = len(instance_list) if num_instances > 1: if not node: node = utils._pick('node', list(range(1, num_instances + 1)), helpfn=helpfn) node = int(node) - 1 instance = instance_list[int(node)] else: ensure( node == 1 or node is None, "You can't specify a node different from 1 for a single-instance stack" ) instance = instance_list[0] ensure( instance.public_ip_address, "Selected instance does not have a public ip address, are you sure it's running?" ) return instance
def _retrieve_build_vars(): """wrapper around `read_from_current_host` with integrity checks. returns buildvars for the current instance. raises AssertionError on bad data.""" try: buildvars = read_from_current_host() LOG.debug('build vars: %s', buildvars) # buildvars exist ensure( isinstance(buildvars, dict), 'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.' % buildvars) # nothing important is missing missing_keys = core_utils.missingkeys( buildvars, ['stackname', 'instance_id', 'branch', 'revision']) ensure( len(missing_keys) == 0, 'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.' % missing_keys) return buildvars except (ValueError, AssertionError, JSONDecodeError) as ex: LOG.exception(ex) raise
def test_ensure(self): utils.ensure(True, "True should allow ensure() to continue") self.assertRaises(AssertionError, utils.ensure, False, "Error message") class CustomException(Exception): pass self.assertRaises(CustomException, utils.ensure, False, "Error message", CustomException)
def wrapper(*args, **kwargs): import master ensure( master.server_access(), "this command requires access to the master server. you don't have it." ) return fn(*args, **kwargs)
def context(pname, output_format=None): formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.yaml_dumps, # None: core_utils.remove_ordereddict None: lambda v: v } ensure(output_format in formatters.keys(), "unknown output format %r" % output_format) formatter = formatters.get(output_format) return formatter(cfngen.build_context(pname, stackname=core.mk_stackname(pname, "test")))
def data(pname, output_format=None): "given a project name, returns the fully realized project description data." ensure(output_format in [None, 'json', 'yaml'], "unknown output format %r" % output_format) formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.ordered_dump, None: lambda v: v } formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def data(pname, output_format=None): "given a project name, returns the fully realized project description data." formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.yaml_dumps, # None: core_utils.remove_ordereddict None: lambda v: v } ensure(output_format in formatters.keys(), "unknown output format %r" % output_format) formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def context(pname, output_format=None): formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.yaml_dumps, # None: core_utils.remove_ordereddict None: lambda v: v } ensure(output_format in formatters.keys(), "unknown output format %r" % output_format) formatter = formatters.get(output_format) return formatter( cfngen.build_context(pname, stackname=core.mk_stackname(pname, "test")))
def _pick_node(instance_list, node): num_instances = len(instance_list) if num_instances > 1: if not node: node = utils._pick('node', range(1, num_instances + 1)) node = int(node) - 1 instance = instance_list[int(node)] else: instance = instance_list[0] core_utils.ensure( instance.ip_address is not None, "Selected instance does not have a public ip address, are you sure it's running?" ) return instance
def parse_validate_repolist(fdata, *repolist): "returns a list of triples" known_formulas = fdata.get('formula-dependencies', []) known_formulas.extend([fdata['formula-repo'], fdata['private-repo']]) known_formula_map = OrderedDict( zip(map(os.path.basename, known_formulas), known_formulas)) arglist = [] for user_string in repolist: if '@' not in user_string: print('skipping %r, no revision component' % user_string) continue repo, rev = user_string.split('@') if not rev.strip(): print('skipping %r, empty revision component' % user_string) continue if repo not in known_formula_map: print('skipping %r, unknown formula. known formulas: %s' % (repo, ', '.join(known_formula_map.keys()))) continue arglist.append((repo, known_formula_map[repo], rev)) # test given revisions actually exist in formulas for name, _, revision in arglist: path = join(config.PROJECT_PATH, "cloned-projects", name) if not os.path.exists(path): LOG.warn( "couldn't find formula %r locally, revision check skipped", path) continue with lcd(path), settings(warn_only=True): ensure( local("git fetch --quiet")['succeeded'], "failed to fetch remote refs for %s" % path) ensure( local("git cat-file -e %s^{commit}" % revision)['succeeded'], "failed to find ref %r in %s" % (revision, name)) return arglist
def generate_stack_from_input(pname, instance_id=None, alt_config=None): """creates a new CloudFormation file for the given project.""" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) checks.ensure_stack_does_not_exist(stackname) more_context = {'stackname': stackname} pdata = project.project_data(pname) if alt_config: ensure( 'aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations" ) # prompt user for alternate configurations if pdata['aws-alt']: default = 'skip' def helpfn(altkey): if altkey == default: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None if instance_id in pdata['aws-alt'].keys(): LOG.info( "instance-id found in known alternative configurations. using configuration %r", instance_id) more_context['alt-config'] = instance_id else: alt_config_choices = [default] + list(pdata['aws-alt'].keys()) if not alt_config: alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config # TODO: return the templates used here, so that they can be passed down to # bootstrap.create_stack() without relying on them implicitly existing # on the filesystem cfngen.generate_stack(pname, **more_context) return stackname
def _pick_node(instance_list, node): instance_list = sorted(instance_list, key=lambda n: tags2dict(n.tags)['Name']) info = [n for n in instance_list] # def helpfn(pick): node = pick - 1 return "%s (%s, %s)" % (tags2dict(info[node].tags)['Name'], info[node].id, info[node].public_ip_address) num_instances = len(instance_list) if num_instances > 1: if not node: node = utils._pick('node', list(range(1, num_instances + 1)), helpfn=helpfn) node = int(node) - 1 instance = instance_list[int(node)] else: ensure(node == 1 or node is None, "You can't specify a node different from 1 for a single-instance stack") instance = instance_list[0] ensure(instance.public_ip_address, "Selected instance does not have a public ip address, are you sure it's running?") return instance
def parse_validate_repolist(fdata, *repolist): "returns a list of triples" known_formulas = fdata.get('formula-dependencies', []) known_formulas.extend([ fdata['formula-repo'], fdata['private-repo'] ]) known_formula_map = OrderedDict(zip(map(os.path.basename, known_formulas), known_formulas)) arglist = [] for user_string in repolist: if '@' not in user_string: print('skipping %r, no revision component' % user_string) continue repo, rev = user_string.split('@') if not rev.strip(): print('skipping %r, empty revision component' % user_string) continue if repo not in known_formula_map: print('skipping %r, unknown formula. known formulas: %s' % (repo, ', '.join(known_formula_map.keys()))) continue arglist.append((repo, known_formula_map[repo], rev)) # test given revisions actually exist in formulas for name, _, revision in arglist: path = join(config.PROJECT_PATH, "cloned-projects", name) if not os.path.exists(path): LOG.warn("couldn't find formula %r locally, revision check skipped", path) continue with lcd(path), settings(warn_only=True): ensure(local("git fetch --quiet").succeeded, "failed to fetch remote refs for %s" % path) ensure(local("git cat-file -e %s^{commit}" % revision).succeeded, "failed to find ref %r in %s" % (revision, name)) return arglist
def launch(pname, instance_id=None, alt_config='standalone', *repolist): stackname = cfn.generate_stack_from_input(pname, instance_id, alt_config) pdata = core.project_data_for_stackname(stackname) # ensure given alt config has masterless=True # todo: can the choices presented to the user remove non-masterless alt-configs? ensure(pdata['aws-alt'], "project has no alternate configurations") ensure(alt_config in pdata['aws-alt'], "unknown alt-config %r" % alt_config) ensure(pdata['aws-alt'][alt_config]['ec2']['masterless'], "alternative configuration %r has masterless=False" % alt_config) formula_revisions = parse_validate_repolist(pdata, *repolist) # todo: this is good UX but was simply debug output that got left in. # a better summary of what is to be created could be printed out, # preferably after the templates are printed out but before confirmation. LOG.info('attempting to create masterless stack:') LOG.info('stackname:\t' + stackname) LOG.info('region:\t' + pdata['aws']['region']) LOG.info('formula_revisions:\t%s' % pformat(formula_revisions)) if core.is_master_server_stack(stackname): checks.ensure_can_access_builder_private(pname) checks.ensure_stack_does_not_exist(stackname) bootstrap.create_stack(stackname) LOG.info('updating stack %s', stackname) bootstrap.update_stack(stackname, service_list=['ec2', 'sqs', 's3'], formula_revisions=formula_revisions)
def launch(pname, instance_id=None, alt_config='standalone', *repolist): stackname = cfn.generate_stack_from_input(pname, instance_id, alt_config) pdata = core.project_data_for_stackname(stackname) # ensure given alt config has masterless=True ensure(pdata['aws-alt'], "project has no alternate configurations") ensure(alt_config in pdata['aws-alt'], "unknown alt-config %r" % alt_config) ensure(pdata['aws-alt'][alt_config]['ec2']['masterless'], "alternative configuration %r has masterless=False" % alt_config) formula_revisions = parse_validate_repolist(pdata, *repolist) LOG.info('attempting to create masterless stack:') LOG.info('stackname:\t' + stackname) LOG.info('region:\t' + pdata['aws']['region']) LOG.info('formula_revisions:\t%s' % pformat(formula_revisions)) if core.is_master_server_stack(stackname): checks.ensure_can_access_builder_private(pname) checks.ensure_stack_does_not_exist(stackname) bootstrap.create_stack(stackname) LOG.info('updating stack %s', stackname) bootstrap.update_stack(stackname, service_list=['ec2', 'sqs', 's3'], formula_revisions=formula_revisions)
def _validate(): "returns a pair of (type, build data) for the given instance. type is either 'old', 'abbrev' or 'full'" try: buildvars = read_from_current_host() LOG.debug('build vars: %s', buildvars) core_utils.ensure( isinstance(buildvars, dict), 'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.', buildvars) missing_keys = core_utils.missingkeys(buildvars, [ 'stackname', 'instance_id', 'branch', 'revision', 'is_prod_instance' ]) core_utils.ensure( len(missing_keys) == 0, 'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.' % missing_keys) return buildvars except (ValueError, AssertionError) as ex: LOG.exception(ex) raise
def _retrieve_build_vars(): """wrapper around `read_from_current_host` with integrity checks. returns buildvars for the current instance. raises AssertionError on bad data.""" try: buildvars = read_from_current_host() LOG.debug('build vars: %s', buildvars) # buildvars exist ensure(isinstance(buildvars, dict), 'build vars not found (%s). use `./bldr buildvars.fix` to attempt to fix this.' % buildvars) # nothing important is missing missing_keys = core_utils.missingkeys(buildvars, ['stackname', 'instance_id', 'branch', 'revision']) ensure( len(missing_keys) == 0, 'build vars are not valid: missing keys %s. use `./bldr buildvars.fix` to attempt to fix this.' % missing_keys ) return buildvars except (ValueError, AssertionError) as ex: LOG.exception(ex) raise
def generate_stack_from_input(pname, instance_id=None, alt_config=None): """creates a new CloudFormation file for the given project.""" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) checks.ensure_stack_does_not_exist(stackname) more_context = {'stackname': stackname} pdata = project.project_data(pname) if alt_config: ensure('aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations") # prompt user for alternate configurations if pdata['aws-alt']: default = 'skip' def helpfn(altkey): if altkey == default: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None if instance_id in pdata['aws-alt'].keys(): LOG.info("instance-id found in known alternative configurations. using configuration %r", instance_id) more_context['alt-config'] = instance_id else: alt_config_choices = [default] + list(pdata['aws-alt'].keys()) if not alt_config: alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config # TODO: return the templates used here, so that they can be passed down to # bootstrap.create_stack() without relying on them implicitly existing # on the filesystem cfngen.generate_stack(pname, **more_context) return stackname
def _pick_node(instance_list, node): info = [n for n in instance_list] def helpfn(pick): node = pick - 1 return info[node] num_instances = len(instance_list) if num_instances > 1: if not node: # TODO print some more info: ip address, instance id node = utils._pick('node', range(1, num_instances + 1), helpfn=helpfn) node = int(node) - 1 instance = instance_list[int(node)] else: assert node == 1 or node is None, "You can't specify a node different from 1 for a single-instance stack" instance = instance_list[0] core_utils.ensure( instance.ip_address is not None, "Selected instance does not have a public ip address, are you sure it's running?" ) return instance
def test_ensure(self): utils.ensure(True, "True should allow ensure() to continue") self.assertRaises(AssertionError, lambda: utils.ensure(False, "Error message")) self.assertRaises( AssertionError, lambda: utils.ensure(False, "Error message: %s", "argument")) class CustomException(Exception): pass self.assertRaises( CustomException, lambda: utils.ensure( False, "Error message", exception_class=CustomException)) self.assertRaises( CustomException, lambda: utils.ensure(False, "Error message: %s", "argument", exception_class=CustomException)) self.assertRaises( ValueError, lambda: utils.ensure( False, "Error message", random_argument=CustomException))
def wrapper(stackname=None, *args, **kwargs): ctx = context_handler.load_context(stackname) ensure(stackname and ctx['ec2']['masterless'], "this command requires a masterless instance.") return fn(stackname, *args, **kwargs)
def wrapper(*args, **kwargs): import master ensure(master.server_access(), "this command requires access to the master server. you don't have it.") return fn(*args, **kwargs)
def check_user_input(pname, instance_id=None, alt_config=None): "marshals user input and checks it for correctness" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) pdata = project.project_data(pname) # alt-config given, die if it doesn't exist if alt_config: ensure( 'aws-alt' in pdata, "alt-config %r given, but project has no alternate configurations" % alt_config) # if the requested instance-id matches a known alt-config, we'll use that alt-config. warn user. if instance_id in pdata['aws-alt'].keys(): LOG.warn("instance-id %r found in alt-config list, using that.", instance_id) alt_config = instance_id # no alt-config given but alt-config options exist, prompt user if not alt_config and pdata['aws-alt']: default_choice = 'skip' def helpfn(altkey): if altkey == default_choice: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None alt_config_choice_list = [default_choice] + list( pdata['aws-alt'].keys()) alt_config_choice = utils._pick('alternative config', alt_config_choice_list, helpfn=helpfn) if alt_config_choice != default_choice: alt_config = alt_config_choice # check the alt-config isn't unique and if it *is* unique, that an instance using it doesn't exist yet. # note: it is *technically* possible that an instance is using a unique configuration but # that its instance-id *is not* the name of the alt-config passed in. # For example, if `journal--prod` didn't exist, I could create `journal--foo` using the `prod` config. if alt_config and alt_config in pdata['aws-alt'] and pdata['aws-alt'][ alt_config]['unique']: dealbreaker = core.mk_stackname(pname, alt_config) # "project 'journal' config 'prod' is marked as unique!" # "checking for any instance named 'journal--prod' ..." print("project %r config %r is marked as unique!" % (pname, alt_config)) print("checking for any instance named %r ..." % (dealbreaker, )) try: checks.ensure_stack_does_not_exist(dealbreaker) except checks.StackAlreadyExistsProblem: # "stack 'journal--prod' exists, cannot re-use unique configuration 'prod'" msg = "stack %r exists, cannot re-use unique configuration %r." % ( dealbreaker, alt_config) raise TaskExit(msg) # check that the instance we want to create doesn't exist try: print("checking %r doesn't exist." % stackname) checks.ensure_stack_does_not_exist(stackname) except checks.StackAlreadyExistsProblem as e: msg = 'stack %r already exists.' % e.stackname raise TaskExit(msg) more_context = {'stackname': stackname} if alt_config: more_context['alt-config'] = alt_config return more_context