def generate_stack_from_input(pname, instance_id=None): """creates a new CloudFormation file for the given project.""" if not instance_id: default_instance_id = core_utils.ymd() instance_id = utils.uin("instance id", default_instance_id) stackname = core.mk_stackname(pname, instance_id) more_context = {'stackname': stackname} # prompt user for alternate configurations pdata = project.project_data(pname) if 'aws-alt' in pdata: def helpfn(altkey): try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None default = 'skip this step' alt_config = [default] + pdata['aws-alt'].keys() alt_config = utils._pick('alternative config', alt_config, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config cfngen.generate_stack(pname, **more_context) return stackname
def install_update_formula_deps(): pdata = project.project_data('master-server') for dep in pdata.get('formula-dependencies', []): name = os.path.basename( dep ) # ll: 'some-formula' in 'https://github.com/elifesciences/some-formula install_formula(name, dep)
def test_merge_default_snippet(self): "merging a snippet into the defaults ensures all projects get that new default" # all projects now get 999 cpus. perfectly sane requirement. project_data = project.project_data('dummy1') project_data = utils.remove_ordereddict(project_data) expected_data = json.load(open(self.dummy1_config, 'r')) expected_data['vagrant']['cpus'] = 999 self.assertEqual(project_data, expected_data)
def test_merge_default_snippet(self): "merging a snippet into the defaults ensures all projects get that new default" # all projects now get 999 cpus. perfectly sane requirement. project_data = project.project_data('dummy1') project_data = utils.remove_ordereddict(project_data) expected_data = json.load(open(self.dummy1_config, 'r')) expected_data['vagrant']['cpus'] = 999 self.assertEqual(project_data, expected_data)
def data(pname, output_format=None): assert output_format in [None, 'json', 'yaml'], "unknown output format %r" % output_format formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.ordered_dump, None: lambda v: v } formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def data(pname, output_format=None): assert output_format in [None, 'json', 'yaml' ], "unknown output format %r" % output_format formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.ordered_dump, None: lambda v: v } formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def data(pname, output_format=None): "given a project name, returns the fully realized project description data." formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.yaml_dumps, # None: core_utils.remove_ordereddict None: lambda v: v } ensure(output_format in formatters.keys(), "unknown output format %r" % output_format) formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def data(pname, output_format=None): "given a project name, returns the fully realized project description data." ensure(output_format in [None, 'json', 'yaml'], "unknown output format %r" % output_format) formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.ordered_dump, None: lambda v: v } formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def test_configurations(self): expected = [ ('dummy1', self.dummy1_config), ('dummy2', self.dummy2_config), ('dummy3', self.dummy3_config), ] for pname, expected_path in expected: expected_data = json.load(open(expected_path, 'r')) project_data = project.project_data(pname) project_data = utils.remove_ordereddict(project_data) self.assertEqual(expected_data, project_data)
def test_configurations(self): expected = [ ('dummy1', self.dummy1_config), ('dummy2', self.dummy2_config), ('dummy3', self.dummy3_config), ] for pname, expected_path in expected: expected_data = json.load(open(expected_path, 'r')) project_data = project.project_data(pname) project_data = utils.remove_ordereddict(project_data) self.assertEqual(expected_data, project_data)
def data(pname, output_format=None): "given a project name, returns the fully realized project description data." formatters = { 'json': core_utils.json_dumps, 'yaml': core_utils.yaml_dumps, # None: core_utils.remove_ordereddict None: lambda v: v } ensure(output_format in formatters.keys(), "unknown output format %r" % output_format) formatter = formatters.get(output_format) return formatter(project.project_data(pname))
def test_merge_multiple_default_snippets(self): """merging multiple overlapping snippets into the defaults ensures all projects get the new defaults""" # all projects now get 999 cpus. perfectly sane requirement. project_data = project.project_data('dummy1') project_data = utils.remove_ordereddict(project_data) expected_data = json.load(open(self.dummy1_config, 'r')) expected_data['vagrant']['cpus'] = 999 expected_data['vagrant']['cpucap'] = 111 self.assertEqual(project_data, expected_data)
def test_merge_multiple_default_snippets(self): """merging multiple overlapping snippets into the defaults ensures all projects get the new defaults""" # all projects now get 999 cpus. perfectly sane requirement. project_data = project.project_data('dummy1') project_data = utils.remove_ordereddict(project_data) expected_data = json.load(open(self.dummy1_config, 'r')) expected_data['vagrant']['cpus'] = 999 expected_data['vagrant']['cpucap'] = 111 self.assertEqual(project_data, expected_data)
def generate_stack_from_input(pname, instance_id=None, alt_config=None): """creates a new CloudFormation file for the given project.""" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) checks.ensure_stack_does_not_exist(stackname) more_context = {'stackname': stackname} pdata = project.project_data(pname) if alt_config: ensure( 'aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations" ) # prompt user for alternate configurations if pdata['aws-alt']: default = 'skip' def helpfn(altkey): if altkey == default: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None if instance_id in pdata['aws-alt'].keys(): LOG.info( "instance-id found in known alternative configurations. using configuration %r", instance_id) more_context['alt-config'] = instance_id else: alt_config_choices = [default] + list(pdata['aws-alt'].keys()) if not alt_config: alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config # TODO: return the templates used here, so that they can be passed down to # bootstrap.create_stack() without relying on them implicitly existing # on the filesystem cfngen.generate_stack(pname, **more_context) return stackname
def deploy(pname, instance_id=None, branch='master', part_filter=None): pdata = project.project_data(pname) if not branch: branch_list = utils.git_remote_branches(pdata['repo']) branch_list = impose_ordering(branch_list) branch = utils._pick('branch', branch_list, deffile('.branch')) stackname = cfn.generate_stack_from_input(pname, instance_id) region = pdata['aws']['region'] active_stacks = core.active_stack_names(region) if stackname in active_stacks: LOG.info("stack %r exists, skipping creation", stackname) else: LOG.info("stack %r doesn't exist, creating", stackname) more_context = cfngen.choose_config(stackname) more_context['branch'] = branch cfngen.generate_stack(pname, **more_context) bootstrap.create_update(stackname, part_filter) setdefault('.active-stack', stackname)
def create_stack(pname): """creates a new CloudFormation template for the given project.""" default_instance_id = core_utils.ymd() inst_id = utils.uin("instance id", default_instance_id) stackname = core.mk_stackname(pname, inst_id) more_context = {'instance_id': stackname} # prompt user for alternate configurations pdata = project.project_data(pname) if pdata.has_key('aws-alt'): def helpfn(altkey): try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None default = 'skip this step' alt_config = [default] + pdata['aws-alt'].keys() alt_config = utils._pick('alternative config', alt_config, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config cfngen.generate_stack(pname, **more_context) return stackname
def generate_stack_from_input(pname, instance_id=None, alt_config=None): """creates a new CloudFormation file for the given project.""" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) checks.ensure_stack_does_not_exist(stackname) more_context = {'stackname': stackname} pdata = project.project_data(pname) if alt_config: ensure('aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations") # prompt user for alternate configurations if pdata['aws-alt']: default = 'skip' def helpfn(altkey): if altkey == default: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None if instance_id in pdata['aws-alt'].keys(): LOG.info("instance-id found in known alternative configurations. using configuration %r", instance_id) more_context['alt-config'] = instance_id else: alt_config_choices = [default] + list(pdata['aws-alt'].keys()) if not alt_config: alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn) if alt_config != default: more_context['alt-config'] = alt_config # TODO: return the templates used here, so that they can be passed down to # bootstrap.create_stack() without relying on them implicitly existing # on the filesystem cfngen.generate_stack(pname, **more_context) return stackname
def install_update_all_project_formulas(): for pname in project.projects_with_formulas(): pdata = project.project_data(pname) formula_url = pdata['formula-repo'] install_formula(pname, formula_url)
def project_config(pname): return core_utils.remove_ordereddict(project.project_data(pname))
def install_update_all_project_formulas(): for pname in project.projects_with_formulas(): pdata = project.project_data(pname) formula_url = pdata['formula-repo'] install_formula(pname, formula_url)
import logging logging.disable(logging.CRITICAL) # import buildercore src_dir = os.path.abspath('src') sys.path.insert(0, src_dir) from buildercore import project, utils, bootstrap output = None # specific project, specific task if args.pname: pname = args.pname[0] # multiple projects in future? if args.task == 'project-data': output = project.project_data(pname) elif pname == 'master-server' and args.task == 'salt-master-config': master_configuration_template = open('etc-salt-master.template', 'r') output = bootstrap.expand_master_configuration(master_configuration_template) # many projects else: if args.formula: # only project formulas output = project.known_formulas() elif args.env: # vagrant/aws # only projects that use given environment output = list(project.filtered_projects(lambda pname, pdata: args.env in pdata).keys()) else: # all projects
def prj(pname, path): return core_utils.lookup(project.project_data(pname), path)
def check_user_input(pname, instance_id=None, alt_config=None): "marshals user input and checks it for correctness" instance_id = instance_id or utils.uin("instance id", core_utils.ymd()) stackname = core.mk_stackname(pname, instance_id) pdata = project.project_data(pname) # alt-config given, die if it doesn't exist if alt_config: ensure( 'aws-alt' in pdata, "alt-config %r given, but project has no alternate configurations" % alt_config) # if the requested instance-id matches a known alt-config, we'll use that alt-config. warn user. if instance_id in pdata['aws-alt'].keys(): LOG.warn("instance-id %r found in alt-config list, using that.", instance_id) alt_config = instance_id # no alt-config given but alt-config options exist, prompt user if not alt_config and pdata['aws-alt']: default_choice = 'skip' def helpfn(altkey): if altkey == default_choice: return 'uses the default configuration' try: return pdata['aws-alt'][altkey]['description'] except KeyError: return None alt_config_choice_list = [default_choice] + list( pdata['aws-alt'].keys()) alt_config_choice = utils._pick('alternative config', alt_config_choice_list, helpfn=helpfn) if alt_config_choice != default_choice: alt_config = alt_config_choice # check the alt-config isn't unique and if it *is* unique, that an instance using it doesn't exist yet. # note: it is *technically* possible that an instance is using a unique configuration but # that its instance-id *is not* the name of the alt-config passed in. # For example, if `journal--prod` didn't exist, I could create `journal--foo` using the `prod` config. if alt_config and alt_config in pdata['aws-alt'] and pdata['aws-alt'][ alt_config]['unique']: dealbreaker = core.mk_stackname(pname, alt_config) # "project 'journal' config 'prod' is marked as unique!" # "checking for any instance named 'journal--prod' ..." print("project %r config %r is marked as unique!" % (pname, alt_config)) print("checking for any instance named %r ..." % (dealbreaker, )) try: checks.ensure_stack_does_not_exist(dealbreaker) except checks.StackAlreadyExistsProblem: # "stack 'journal--prod' exists, cannot re-use unique configuration 'prod'" msg = "stack %r exists, cannot re-use unique configuration %r." % ( dealbreaker, alt_config) raise TaskExit(msg) # check that the instance we want to create doesn't exist try: print("checking %r doesn't exist." % stackname) checks.ensure_stack_does_not_exist(stackname) except checks.StackAlreadyExistsProblem as e: msg = 'stack %r already exists.' % e.stackname raise TaskExit(msg) more_context = {'stackname': stackname} if alt_config: more_context['alt-config'] = alt_config return more_context
def fetch_cert(stackname): # NOTE: this was ported from the old builder and won't work with new instances # this isn't a problem because new instances shouldn't be using letsencrypt if # they can avoid it. try: # replicates some logic in builder core pname = core.project_name_from_stackname(stackname) project_data = project.project_data(pname) assert 'subdomain' in project_data, "project subdomain not found. quitting" instance_id = stackname[len(pname + "-"):] is_prod = instance_id in ['master', 'production'] # we still have some instances that are the production/master # instances but don't adhere to the naming yet. old_prods = [ 'elife-ci-2015-11-04', ] if not is_prod and stackname in old_prods: is_prod = True hostname_data = core.hostname_struct(stackname) domain_names = [hostname_data['full_hostname']] if is_prod: project_hostname = hostname_data['project_hostname'] if acme_enabled(project_hostname): domain_names.append(project_hostname) else: print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname print '\nthese hosts will be targeted:' print '* ' + '\n* '.join(domain_names) #pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR) # server = { # 'staging': pillar_data['sys']['webserver']['acme_staging_server'], # 'live': pillar_data['sys']['webserver']['acme_server'], #} server = { 'staging': "https://acme-staging.api.letsencrypt.org/directory", 'live': "https://acme-v01.api.letsencrypt.org/directory", } certtype = utils._pick("certificate type", ['staging', 'live']) cmds = [ "cd /opt/letsencrypt/", "./fetch-ssl-certs.sh -d %s --server %s" % (" -d ".join(domain_names), server[certtype]), "sudo service nginx reload", ] print print 'the following commands will be run:' print ' * ' + '\n * '.join(cmds) print if raw_input('enter to continue, ctrl-c to quit') == '': with stack_conn(stackname): return run(" && ".join(cmds)) except AssertionError as ex: print print "* " + str(ex) print exit(1)
def fetch_cert(stackname): try: # replicates some logic in builder core pname = core.project_name_from_stackname(stackname) project_data = project.project_data(pname) assert project_data.has_key('subdomain'), "project subdomain not found. quitting" instance_id = stackname[len(pname + "-"):] is_prod = instance_id in ['master', 'production'] # we still have some instances that are the production/master # instances but don't adhere to the naming yet. old_prods = [ 'elife-ci-2015-11-04', 'elife-jira-2015-06-02' ] if not is_prod and stackname in old_prods: is_prod = True hostname_data = core.hostname_struct(stackname) domain_names = [hostname_data['full_hostname']] if is_prod: project_hostname = hostname_data['project_hostname'] if acme_enabled(project_hostname): domain_names.append(project_hostname) else: print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname print '\nthese hosts will be targeted:' print '* ' + '\n* '.join(domain_names) #pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR) #server = { # 'staging': pillar_data['sys']['webserver']['acme_staging_server'], # 'live': pillar_data['sys']['webserver']['acme_server'], #} server = { 'staging': "https://acme-staging.api.letsencrypt.org/directory", 'live': "https://acme-v01.api.letsencrypt.org/directory", } certtype = utils._pick("certificate type", ['staging', 'live']) cmds = [ "cd /opt/letsencrypt/", "./fetch-ssl-certs.sh -d %s --server %s" % (" -d ".join(domain_names), server[certtype]), "sudo service nginx reload", ] print print 'the following commands will be run:' print ' * ' + '\n * '.join(cmds) print if raw_input('enter to continue, ctrl-c to quit') == '': with stack_conn(stackname): return run(" && ".join(cmds)) except AssertionError, ex: print print "* " + str(ex) print exit(1)
parser.add_argument('--format', default='yaml') args = parser.parse_args() # hide any unimportant logging import logging logging.disable(logging.CRITICAL) # import buildercore src_dir = os.path.abspath('src') sys.path.insert(0, src_dir) from buildercore import project, utils # project data if args.pname: pname = args.pname[0] # multiple projects in future? output = project.project_data(pname) # project list else: if args.env: # vagrant/aws output = project.filtered_projects( lambda pname, pdata: args.env in pdata).keys() else: output = project.project_list() output.sort() formats = { 'yaml': utils.ordered_dump, 'json': partial(json.dumps, indent=4), } if args.format not in formats:
def install_update_formula_deps(): pdata = project.project_data('master-server') for dep in pdata.get('formula-dependencies', []): name = os.path.basename(dep) # ll: 'some-formula' in 'https://github.com/elifesciences/some-formula install_formula(name, dep)
def project_config(pname): return core_utils.remove_ordereddict(project.project_data(pname))