def generate_stack_from_input(pname, instance_id=None):
"""creates a new CloudFormation file for the given project."""
if not instance_id:
default_instance_id = core_utils.ymd()
instance_id = utils.uin("instance id", default_instance_id)
stackname = core.mk_stackname(pname, instance_id)
more_context = {'stackname': stackname}
# prompt user for alternate configurations
pdata = project.project_data(pname)
if 'aws-alt' in pdata:
def helpfn(altkey):
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
default = 'skip this step'
alt_config = [default] + pdata['aws-alt'].keys()
alt_config = utils._pick('alternative config',
alt_config,
helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
cfngen.generate_stack(pname, **more_context)
return stackname
def install_update_formula_deps():
pdata = project.project_data('master-server')
for dep in pdata.get('formula-dependencies', []):
name = os.path.basename(
dep
) # ll: 'some-formula' in 'https://github.com/elifesciences/some-formula
install_formula(name, dep)
def test_merge_default_snippet(self):
"merging a snippet into the defaults ensures all projects get that new default"
# all projects now get 999 cpus. perfectly sane requirement.
project_data = project.project_data('dummy1')
project_data = utils.remove_ordereddict(project_data)
expected_data = json.load(open(self.dummy1_config, 'r'))
expected_data['vagrant']['cpus'] = 999
self.assertEqual(project_data, expected_data)
def test_merge_default_snippet(self):
"merging a snippet into the defaults ensures all projects get that new default"
# all projects now get 999 cpus. perfectly sane requirement.
project_data = project.project_data('dummy1')
project_data = utils.remove_ordereddict(project_data)
expected_data = json.load(open(self.dummy1_config, 'r'))
expected_data['vagrant']['cpus'] = 999
self.assertEqual(project_data, expected_data)
def data(pname, output_format=None):
assert output_format in [None, 'json', 'yaml'], "unknown output format %r" % output_format
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.ordered_dump,
None: lambda v: v
}
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def data(pname, output_format=None):
assert output_format in [None, 'json', 'yaml'
], "unknown output format %r" % output_format
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.ordered_dump,
None: lambda v: v
}
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(), "unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
ensure(output_format in [None, 'json', 'yaml'],
"unknown output format %r" % output_format)
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.ordered_dump,
None: lambda v: v
}
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def test_configurations(self):
expected = [
('dummy1', self.dummy1_config),
('dummy2', self.dummy2_config),
('dummy3', self.dummy3_config),
]
for pname, expected_path in expected:
expected_data = json.load(open(expected_path, 'r'))
project_data = project.project_data(pname)
project_data = utils.remove_ordereddict(project_data)
self.assertEqual(expected_data, project_data)
def test_configurations(self):
expected = [
('dummy1', self.dummy1_config),
('dummy2', self.dummy2_config),
('dummy3', self.dummy3_config),
]
for pname, expected_path in expected:
expected_data = json.load(open(expected_path, 'r'))
project_data = project.project_data(pname)
project_data = utils.remove_ordereddict(project_data)
self.assertEqual(expected_data, project_data)
def data(pname, output_format=None):
"given a project name, returns the fully realized project description data."
formatters = {
'json': core_utils.json_dumps,
'yaml': core_utils.yaml_dumps,
# None: core_utils.remove_ordereddict
None: lambda v: v
}
ensure(output_format in formatters.keys(),
"unknown output format %r" % output_format)
formatter = formatters.get(output_format)
return formatter(project.project_data(pname))
def test_merge_multiple_default_snippets(self):
"""merging multiple overlapping snippets into the defaults
ensures all projects get the new defaults"""
# all projects now get 999 cpus. perfectly sane requirement.
project_data = project.project_data('dummy1')
project_data = utils.remove_ordereddict(project_data)
expected_data = json.load(open(self.dummy1_config, 'r'))
expected_data['vagrant']['cpus'] = 999
expected_data['vagrant']['cpucap'] = 111
self.assertEqual(project_data, expected_data)
def test_merge_multiple_default_snippets(self):
"""merging multiple overlapping snippets into the defaults
ensures all projects get the new defaults"""
# all projects now get 999 cpus. perfectly sane requirement.
project_data = project.project_data('dummy1')
project_data = utils.remove_ordereddict(project_data)
expected_data = json.load(open(self.dummy1_config, 'r'))
expected_data['vagrant']['cpus'] = 999
expected_data['vagrant']['cpucap'] = 111
self.assertEqual(project_data, expected_data)
def generate_stack_from_input(pname, instance_id=None, alt_config=None):
"""creates a new CloudFormation file for the given project."""
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
checks.ensure_stack_does_not_exist(stackname)
more_context = {'stackname': stackname}
pdata = project.project_data(pname)
if alt_config:
ensure(
'aws-alt' in pdata,
"alternative configuration name given, but project has no alternate configurations"
)
# prompt user for alternate configurations
if pdata['aws-alt']:
default = 'skip'
def helpfn(altkey):
if altkey == default:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
if instance_id in pdata['aws-alt'].keys():
LOG.info(
"instance-id found in known alternative configurations. using configuration %r",
instance_id)
more_context['alt-config'] = instance_id
else:
alt_config_choices = [default] + list(pdata['aws-alt'].keys())
if not alt_config:
alt_config = utils._pick('alternative config',
alt_config_choices,
helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
# TODO: return the templates used here, so that they can be passed down to
# bootstrap.create_stack() without relying on them implicitly existing
# on the filesystem
cfngen.generate_stack(pname, **more_context)
return stackname
def deploy(pname, instance_id=None, branch='master', part_filter=None):
pdata = project.project_data(pname)
if not branch:
branch_list = utils.git_remote_branches(pdata['repo'])
branch_list = impose_ordering(branch_list)
branch = utils._pick('branch', branch_list, deffile('.branch'))
stackname = cfn.generate_stack_from_input(pname, instance_id)
region = pdata['aws']['region']
active_stacks = core.active_stack_names(region)
if stackname in active_stacks:
LOG.info("stack %r exists, skipping creation", stackname)
else:
LOG.info("stack %r doesn't exist, creating", stackname)
more_context = cfngen.choose_config(stackname)
more_context['branch'] = branch
cfngen.generate_stack(pname, **more_context)
bootstrap.create_update(stackname, part_filter)
setdefault('.active-stack', stackname)
def create_stack(pname):
"""creates a new CloudFormation template for the given project."""
default_instance_id = core_utils.ymd()
inst_id = utils.uin("instance id", default_instance_id)
stackname = core.mk_stackname(pname, inst_id)
more_context = {'instance_id': stackname}
# prompt user for alternate configurations
pdata = project.project_data(pname)
if pdata.has_key('aws-alt'):
def helpfn(altkey):
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
default = 'skip this step'
alt_config = [default] + pdata['aws-alt'].keys()
alt_config = utils._pick('alternative config', alt_config, helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
cfngen.generate_stack(pname, **more_context)
return stackname
def generate_stack_from_input(pname, instance_id=None, alt_config=None):
"""creates a new CloudFormation file for the given project."""
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
checks.ensure_stack_does_not_exist(stackname)
more_context = {'stackname': stackname}
pdata = project.project_data(pname)
if alt_config:
ensure('aws-alt' in pdata, "alternative configuration name given, but project has no alternate configurations")
# prompt user for alternate configurations
if pdata['aws-alt']:
default = 'skip'
def helpfn(altkey):
if altkey == default:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
if instance_id in pdata['aws-alt'].keys():
LOG.info("instance-id found in known alternative configurations. using configuration %r", instance_id)
more_context['alt-config'] = instance_id
else:
alt_config_choices = [default] + list(pdata['aws-alt'].keys())
if not alt_config:
alt_config = utils._pick('alternative config', alt_config_choices, helpfn=helpfn)
if alt_config != default:
more_context['alt-config'] = alt_config
# TODO: return the templates used here, so that they can be passed down to
# bootstrap.create_stack() without relying on them implicitly existing
# on the filesystem
cfngen.generate_stack(pname, **more_context)
return stackname
def install_update_all_project_formulas():
for pname in project.projects_with_formulas():
pdata = project.project_data(pname)
formula_url = pdata['formula-repo']
install_formula(pname, formula_url)
def project_config(pname):
return core_utils.remove_ordereddict(project.project_data(pname))
def install_update_all_project_formulas():
for pname in project.projects_with_formulas():
pdata = project.project_data(pname)
formula_url = pdata['formula-repo']
install_formula(pname, formula_url)
import logging
logging.disable(logging.CRITICAL)
# import buildercore
src_dir = os.path.abspath('src')
sys.path.insert(0, src_dir)
from buildercore import project, utils, bootstrap
output = None
# specific project, specific task
if args.pname:
pname = args.pname[0] # multiple projects in future?
if args.task == 'project-data':
output = project.project_data(pname)
elif pname == 'master-server' and args.task == 'salt-master-config':
master_configuration_template = open('etc-salt-master.template', 'r')
output = bootstrap.expand_master_configuration(master_configuration_template)
# many projects
else:
if args.formula:
# only project formulas
output = project.known_formulas()
elif args.env: # vagrant/aws
# only projects that use given environment
output = list(project.filtered_projects(lambda pname, pdata: args.env in pdata).keys())
else:
# all projects
def prj(pname, path):
return core_utils.lookup(project.project_data(pname), path)
def check_user_input(pname, instance_id=None, alt_config=None):
"marshals user input and checks it for correctness"
instance_id = instance_id or utils.uin("instance id", core_utils.ymd())
stackname = core.mk_stackname(pname, instance_id)
pdata = project.project_data(pname)
# alt-config given, die if it doesn't exist
if alt_config:
ensure(
'aws-alt' in pdata,
"alt-config %r given, but project has no alternate configurations"
% alt_config)
# if the requested instance-id matches a known alt-config, we'll use that alt-config. warn user.
if instance_id in pdata['aws-alt'].keys():
LOG.warn("instance-id %r found in alt-config list, using that.",
instance_id)
alt_config = instance_id
# no alt-config given but alt-config options exist, prompt user
if not alt_config and pdata['aws-alt']:
default_choice = 'skip'
def helpfn(altkey):
if altkey == default_choice:
return 'uses the default configuration'
try:
return pdata['aws-alt'][altkey]['description']
except KeyError:
return None
alt_config_choice_list = [default_choice] + list(
pdata['aws-alt'].keys())
alt_config_choice = utils._pick('alternative config',
alt_config_choice_list,
helpfn=helpfn)
if alt_config_choice != default_choice:
alt_config = alt_config_choice
# check the alt-config isn't unique and if it *is* unique, that an instance using it doesn't exist yet.
# note: it is *technically* possible that an instance is using a unique configuration but
# that its instance-id *is not* the name of the alt-config passed in.
# For example, if `journal--prod` didn't exist, I could create `journal--foo` using the `prod` config.
if alt_config and alt_config in pdata['aws-alt'] and pdata['aws-alt'][
alt_config]['unique']:
dealbreaker = core.mk_stackname(pname, alt_config)
# "project 'journal' config 'prod' is marked as unique!"
# "checking for any instance named 'journal--prod' ..."
print("project %r config %r is marked as unique!" %
(pname, alt_config))
print("checking for any instance named %r ..." % (dealbreaker, ))
try:
checks.ensure_stack_does_not_exist(dealbreaker)
except checks.StackAlreadyExistsProblem:
# "stack 'journal--prod' exists, cannot re-use unique configuration 'prod'"
msg = "stack %r exists, cannot re-use unique configuration %r." % (
dealbreaker, alt_config)
raise TaskExit(msg)
# check that the instance we want to create doesn't exist
try:
print("checking %r doesn't exist." % stackname)
checks.ensure_stack_does_not_exist(stackname)
except checks.StackAlreadyExistsProblem as e:
msg = 'stack %r already exists.' % e.stackname
raise TaskExit(msg)
more_context = {'stackname': stackname}
if alt_config:
more_context['alt-config'] = alt_config
return more_context
def fetch_cert(stackname):
# NOTE: this was ported from the old builder and won't work with new instances
# this isn't a problem because new instances shouldn't be using letsencrypt if
# they can avoid it.
try:
# replicates some logic in builder core
pname = core.project_name_from_stackname(stackname)
project_data = project.project_data(pname)
assert 'subdomain' in project_data, "project subdomain not found. quitting"
instance_id = stackname[len(pname + "-"):]
is_prod = instance_id in ['master', 'production']
# we still have some instances that are the production/master
# instances but don't adhere to the naming yet.
old_prods = [
'elife-ci-2015-11-04',
]
if not is_prod and stackname in old_prods:
is_prod = True
hostname_data = core.hostname_struct(stackname)
domain_names = [hostname_data['full_hostname']]
if is_prod:
project_hostname = hostname_data['project_hostname']
if acme_enabled(project_hostname):
domain_names.append(project_hostname)
else:
print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname
print '\nthese hosts will be targeted:'
print '* ' + '\n* '.join(domain_names)
#pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR)
# server = {
# 'staging': pillar_data['sys']['webserver']['acme_staging_server'],
# 'live': pillar_data['sys']['webserver']['acme_server'],
#}
server = {
'staging': "https://acme-staging.api.letsencrypt.org/directory",
'live': "https://acme-v01.api.letsencrypt.org/directory",
}
certtype = utils._pick("certificate type", ['staging', 'live'])
cmds = [
"cd /opt/letsencrypt/",
"./fetch-ssl-certs.sh -d %s --server %s" %
(" -d ".join(domain_names), server[certtype]),
"sudo service nginx reload",
]
print
print 'the following commands will be run:'
print ' * ' + '\n * '.join(cmds)
print
if raw_input('enter to continue, ctrl-c to quit') == '':
with stack_conn(stackname):
return run(" && ".join(cmds))
except AssertionError as ex:
print
print "* " + str(ex)
print
exit(1)
def fetch_cert(stackname):
try:
# replicates some logic in builder core
pname = core.project_name_from_stackname(stackname)
project_data = project.project_data(pname)
assert project_data.has_key('subdomain'), "project subdomain not found. quitting"
instance_id = stackname[len(pname + "-"):]
is_prod = instance_id in ['master', 'production']
# we still have some instances that are the production/master
# instances but don't adhere to the naming yet.
old_prods = [
'elife-ci-2015-11-04',
'elife-jira-2015-06-02'
]
if not is_prod and stackname in old_prods:
is_prod = True
hostname_data = core.hostname_struct(stackname)
domain_names = [hostname_data['full_hostname']]
if is_prod:
project_hostname = hostname_data['project_hostname']
if acme_enabled(project_hostname):
domain_names.append(project_hostname)
else:
print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname
print '\nthese hosts will be targeted:'
print '* ' + '\n* '.join(domain_names)
#pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR)
#server = {
# 'staging': pillar_data['sys']['webserver']['acme_staging_server'],
# 'live': pillar_data['sys']['webserver']['acme_server'],
#}
server = {
'staging': "https://acme-staging.api.letsencrypt.org/directory",
'live': "https://acme-v01.api.letsencrypt.org/directory",
}
certtype = utils._pick("certificate type", ['staging', 'live'])
cmds = [
"cd /opt/letsencrypt/",
"./fetch-ssl-certs.sh -d %s --server %s" % (" -d ".join(domain_names), server[certtype]),
"sudo service nginx reload",
]
print
print 'the following commands will be run:'
print ' * ' + '\n * '.join(cmds)
print
if raw_input('enter to continue, ctrl-c to quit') == '':
with stack_conn(stackname):
return run(" && ".join(cmds))
except AssertionError, ex:
print
print "* " + str(ex)
print
exit(1)
parser.add_argument('--format', default='yaml')
args = parser.parse_args()
# hide any unimportant logging
import logging
logging.disable(logging.CRITICAL)
# import buildercore
src_dir = os.path.abspath('src')
sys.path.insert(0, src_dir)
from buildercore import project, utils
# project data
if args.pname:
pname = args.pname[0] # multiple projects in future?
output = project.project_data(pname)
# project list
else:
if args.env: # vagrant/aws
output = project.filtered_projects(
lambda pname, pdata: args.env in pdata).keys()
else:
output = project.project_list()
output.sort()
formats = {
'yaml': utils.ordered_dump,
'json': partial(json.dumps, indent=4),
}
if args.format not in formats:
def install_update_formula_deps():
pdata = project.project_data('master-server')
for dep in pdata.get('formula-dependencies', []):
name = os.path.basename(dep) # ll: 'some-formula' in 'https://github.com/elifesciences/some-formula
install_formula(name, dep)
def project_config(pname):
return core_utils.remove_ordereddict(project.project_data(pname))
