def _download(path, destination):
with stack_conn(
stackname,
username=BOOTSTRAP_USER if use_bootstrap_user else DEPLOY_USER,
node=node):
if allow_missing and not remote_file_exists(path):
return # skip download
download(path, destination, use_sudo=True)
def dumpdata(stackname):
with stack_conn(stackname):
cmds = [
'cd /srv/lax/ && ./manage.sh dumpdata | gzip -9 - > /tmp/lax-db.json.gz',
]
map(run, cmds)
get('/tmp/lax-db.json.gz', 'public/lax-db.json.gz')
local('gunzip public/lax-db.json.gz')
print '\ndata dumped to `public/lax-db.json`\n'
def upload_file(stackname, local_path, remote_path, overwrite=False):
with stack_conn(stackname):
print 'stack:', stackname
print 'local:', local_path
print 'remote:', remote_path
print 'overwrite:', overwrite
raw_input('continue?')
if files.exists(remote_path) and not overwrite:
print 'remote file exists, not overwriting'
exit(1)
put(local_path, remote_path)
def upload_file(stackname, local_path, remote_path, overwrite=False):
with stack_conn(stackname):
print 'stack:',stackname
print 'local:',local_path
print 'remote:',remote_path
print 'overwrite:',overwrite
raw_input('continue?')
if files.exists(remote_path) and not overwrite:
print 'remote file exists, not overwriting'
exit(1)
put(local_path, remote_path)
def read(stackname):
"returns the unencoded build variables found on given instance"
with stack_conn(stackname):
# due to a typo we now have two types of file naming in existence
# prefer hyphenated over underscores
for fname in ['build-vars.json.b64', 'build_vars.json.b64']:
try:
fd = StringIO()
get(join('/etc/', fname), fd)
return _decode_bvars(fd.getvalue())
except FabricException, ex:
# file not found
continue
def remaster_minion(stackname):
"""tell minion who their new master is.
deletes the master's key on the minion
updates the minion, which re-writes the minion config and eventually calls highstate
* assumes you don't have ssh access to the minion
* assumes writing keypairs to S3 is turned on"""
print 're-mastering',stackname
expected_key = core.stack_pem(stackname)
if not os.path.exists(expected_key):
download_keypair(stackname)
with core.stack_conn(stackname, username=config.BOOTSTRAP_USER):
sudo("rm -f /etc/salt/pki/minion/minion_master.pub") # destroy the old master key we have
bootstrap.update_stack(stackname)
def _update_remote_bvars(stackname, bvars):
LOG.info('updating %r with new vars %r',stackname, bvars)
assert core_utils.hasallkeys(bvars, ['branch']) #, 'revision']) # we don't use 'revision'
with stack_conn(stackname):
encoded = _encode_bvars(bvars)
fid = core_utils.ymd(fmt='%Y%m%d%H%M%S')
cmds = [
# make a backup
'if [ -f /etc/build-vars.json.b64 ]; then cp /etc/build-vars.json.b64 /tmp/build-vars.json.b64.%s; fi;' % fid,
# purge any mention of build vars
'rm -f /etc/build*vars.*',
]
map(sudo, cmds)
put(StringIO(encoded), "/etc/build-vars.json.b64", use_sudo=True)
LOG.info("%r updated", stackname)
def upload_file(stackname, local_path, remote_path=None, overwrite=False, confirm=False, node=1):
remote_path = remote_path or os.path.join("/tmp", os.path.basename(local_path))
overwrite = str(overwrite).lower() == "true"
confirm = str(confirm).lower() == "true"
node = int(node)
with stack_conn(stackname, node=node):
print('stack:', stackname, 'node', node)
print('local:', local_path)
print('remote:', remote_path)
print('overwrite:', overwrite)
if not confirm:
utils.get_input('continue?')
if remote_file_exists(remote_path) and not overwrite:
print('remote file exists, not overwriting')
exit(1)
upload(local_path, remote_path)
def remaster_minion(stackname):
"""tell minion who their new master is.
deletes the master's key on the minion
updates the minion, which re-writes the minion config and eventually calls highstate
* assumes you don't have ssh access to the minion
* assumes writing keypairs to S3 is turned on"""
print 're-mastering', stackname
expected_key = core.stack_pem(stackname)
if not os.path.exists(expected_key):
download_keypair(stackname)
with core.stack_conn(stackname, username=config.BOOTSTRAP_USER):
sudo("rm -f /etc/salt/pki/minion/minion_master.pub"
) # destroy the old master key we have
bootstrap.update_stack(stackname)
def upload_file(stackname, local_path, remote_path=None, overwrite=False, confirm=False, node=1):
remote_path = remote_path or os.path.join("/tmp", os.path.basename(local_path))
overwrite = str(overwrite).lower() == "true"
confirm = str(confirm).lower() == "true"
node = int(node)
with stack_conn(stackname, node=node):
print('stack:', stackname, 'node', node)
print('local:', local_path)
print('remote:', remote_path)
print('overwrite:', overwrite)
if not confirm:
utils.get_input('continue?')
if files.exists(remote_path) and not overwrite:
print('remote file exists, not overwriting')
exit(1)
put(local_path, remote_path)
def create_ami(stackname, name=None):
"creates an AMI from the running stack"
with core.stack_conn(stackname, username=config.BOOTSTRAP_USER):
bootstrap.clean_stack_for_ami()
ec2 = core.find_ec2_instances(stackname)[0]
kwargs = {
'Name': name or ami_name(stackname),
'NoReboot': True,
#'DryRun': True
}
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Instance.create_image
ami = ec2.create_image(**kwargs)
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Image
ami.wait_until_exists(Filters=[{
'Name': 'state',
'Values': ['available'],
}])
return str(ami.id) # this should be used to update the stack templates
def create_ami(stackname, name=None):
"creates an AMI from the running stack"
with core.stack_conn(stackname, username=config.BOOTSTRAP_USER):
bootstrap.clean_stack_for_ami()
ec2 = core.find_ec2_instances(stackname)[0]
kwargs = {
'Name': name or ami_name(stackname),
'NoReboot': True,
#'DryRun': True
}
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Instance.create_image
ami = ec2.create_image(**kwargs)
# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.Image
ami.wait_until_exists(Filters=[{
'Name': 'state',
'Values': ['available'],
}])
return str(ami.id) # this should be used to update the stack templates
def download_file(stackname,
path,
destination=None,
allow_missing="False",
use_bootstrap_user="******"):
"""
Downloads `path` from `stackname` putting it into the `destination` folder, or the `destination` file if it exists and it is a file.
If `allow_missing` is "True", a not existing `path` will be skipped without errors.
If `use_bootstrap_user` is "True", the owner_ssh user will be used for connecting instead of the standard deploy user.
Boolean arguments are expressed as strings as this is the idiomatic way of passing them from the command line.
"""
if not destination:
destination = '.'
with stack_conn(stackname, username=_user(use_bootstrap_user)):
if _should_be_skipped(path, allow_missing):
return
get(path, destination, use_sudo=True)
def create_ami(stackname):
"creates an AMI from the running stack"
with core.stack_conn(stackname):
bootstrap.prep_stack()
ec2 = core.find_ec2_instance(stackname)[0]
kwargs = {
'instance_id': ec2.id,
'name': core.ami_name(stackname),
'no_reboot': True,
#'dry_run': True
}
conn = core.connect_aws_with_stack(stackname, 'ec2')
ami_id = conn.create_image(**kwargs)
# image.__dict__ == {'root_device_type': u'ebs', 'ramdisk_id': None, 'id': u'ami-6bc99d0e', 'owner_alias': None, 'billing_products': [], 'tags': {}, 'platform': None, 'state': u'pending', 'location': u'512686554592/elife-lax.2015-10-15', 'type': u'machine', 'virtualization_type': u'hvm', 'sriov_net_support': u'simple', 'architecture': u'x86_64', 'description': None, 'block_device_mapping': {}, 'kernel_id': None, 'owner_id': u'512686554592', 'is_public': False, 'instance_lifecycle': None, 'creationDate': u'2015-10-15T16:07:21.000Z', 'name': u'elife-lax.2015-10-15', 'hypervisor': u'xen', 'region': RegionInfo:us-east-1, 'item': u'\n ', 'connection': EC2Connection:ec2.us-east-1.amazonaws.com, 'root_device_name': None, 'ownerId': u'512686554592', 'product_codes': []}
def is_pending():
image = conn.get_all_images(image_ids=[ami_id])[0]
return image.state == 'pending'
utils.call_while(is_pending, update_msg="Waiting for AWS to bake AMI %s ... " % ami_id)
return str(ami_id) # this should be used to update the stack templates
def create_ami(stackname):
"creates an AMI from the running stack"
with core.stack_conn(stackname, username=config.BOOTSTRAP_USER):
bootstrap.prep_ec2_instance()
ec2 = core.find_ec2_instances(stackname)[0]
kwargs = {
'instance_id': ec2.id,
'name': core.ami_name(stackname),
'no_reboot': True,
#'dry_run': True
}
conn = core.connect_aws_with_stack(stackname, 'ec2')
ami_id = conn.create_image(**kwargs)
# image.__dict__ == {'root_device_type': u'ebs', 'ramdisk_id': None, 'id': u'ami-6bc99d0e', 'owner_alias': None, 'billing_products': [], 'tags': {}, 'platform': None, 'state': u'pending', 'location': u'512686554592/elife-lax.2015-10-15', 'type': u'machine', 'virtualization_type': u'hvm', 'sriov_net_support': u'simple', 'architecture': u'x86_64', 'description': None, 'block_device_mapping': {}, 'kernel_id': None, 'owner_id': u'512686554592', 'is_public': False, 'instance_lifecycle': None, 'creationDate': u'2015-10-15T16:07:21.000Z', 'name': u'elife-lax.2015-10-15', 'hypervisor': u'xen', 'region': RegionInfo:us-east-1, 'item': u'\n ', 'connection': EC2Connection:ec2.us-east-1.amazonaws.com, 'root_device_name': None, 'ownerId': u'512686554592', 'product_codes': []}
def is_pending():
image = conn.get_all_images(image_ids=[ami_id])[0]
return image.state == 'pending'
utils.call_while(is_pending,
update_msg="Waiting for AWS to bake AMI %s ... " % ami_id)
return str(ami_id) # this should be used to update the stack templates
def download_db_fixtures(stackname):
"""downloads a dump of the lax database as *django fixtures* to the
current directory as 'stackname-yyyymmddhhmmss'.json.gz"""
dtstamp = core_utils.utcnow().isoformat().rsplit('.',
1)[0].replace(':', '-')
local_path = "./%s.%s.json.gz" % (stackname, dtstamp)
remote_path = '/tmp/db.json.gz'
with stack_conn(stackname), cd('/srv/lax/'):
# dump fixtures
run('./manage.sh dumpdata --exclude=contenttypes --natural-foreign --natural-primary --indent=4 | '
'gzip -9 - > ' + remote_path)
# download
get(remote_path, local_path)
# remove remote dump
run('rm ' + remote_path)
print '\ndata dumped to `%s`\n' % local_path
return local_path
def aws_update_many_projects(pname_list):
minions = ' or '.join(map(lambda pname: pname + "-*", pname_list))
region = aws.find_region()
with core.stack_conn(core.find_master(region)):
sudo("salt -C '%s' state.highstate --retcode-passthrough" % minions)
def _update_syslog(stackname):
with stack_conn(stackname):
cmd = "salt-call state.sls_id syslog-ng-hook base.syslog-ng test=True"
return sudo(cmd)
def cmd(stackname, command=None):
if command is None:
abort("Please specify a command e.g. ./bldr cmd:%s,ls" % stackname)
with stack_conn(stackname):
with settings(abort_on_prompts=True):
run(command)
def _download(path, destination):
with stack_conn(stackname, username=BOOTSTRAP_USER if use_bootstrap_user else DEPLOY_USER, node=node):
if allow_missing and not files.exists(path):
return # skip download
get(path, destination, use_sudo=True)
def highstate(stackname):
"a fast update with many caveats. prefer `update` instead"
with stack_conn(stackname, username=BOOTSTRAP_USER):
bootstrap.run_script('highstate.sh')
def repair_cfn_info(stackname):
with stack_conn(stackname):
bootstrap.write_environment_info(stackname, overwrite=True)
def test_core_listfiles_remote(self):
with core.stack_conn(self.stackname, username=BOOTSTRAP_USER):
results = command.remote_listfiles('/')
sublist = ['/tmp', '/bin', '/boot', '/var'] # /etc
self.assertTrue(set(results) >= set(sublist))
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'):
"runs the given command on all aws instances. given command must escape double quotes"
with stack_conn(core.find_master(utils.find_region())):
remote_sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" %
locals())
def kick():
stackname = core.find_master(core.find_region())
with core.stack_conn(stackname, user=config.BOOTSTRAP_USER):
bootstrap.run_script('kick-master.sh')
def fetch_cert(stackname):
try:
# replicates some logic in builder core
pname = core.project_name_from_stackname(stackname)
project_data = project.project_data(pname)
assert project_data.has_key('subdomain'), "project subdomain not found. quitting"
instance_id = stackname[len(pname + "-"):]
is_prod = instance_id in ['master', 'production']
# we still have some instances that are the production/master
# instances but don't adhere to the naming yet.
old_prods = [
'elife-ci-2015-11-04',
'elife-jira-2015-06-02'
]
if not is_prod and stackname in old_prods:
is_prod = True
hostname_data = core.hostname_struct(stackname)
domain_names = [hostname_data['full_hostname']]
if is_prod:
project_hostname = hostname_data['project_hostname']
if acme_enabled(project_hostname):
domain_names.append(project_hostname)
else:
print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname
print '\nthese hosts will be targeted:'
print '* ' + '\n* '.join(domain_names)
#pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR)
#server = {
# 'staging': pillar_data['sys']['webserver']['acme_staging_server'],
# 'live': pillar_data['sys']['webserver']['acme_server'],
#}
server = {
'staging': "https://acme-staging.api.letsencrypt.org/directory",
'live': "https://acme-v01.api.letsencrypt.org/directory",
}
certtype = utils._pick("certificate type", ['staging', 'live'])
cmds = [
"cd /opt/letsencrypt/",
"./fetch-ssl-certs.sh -d %s --server %s" % (" -d ".join(domain_names), server[certtype]),
"sudo service nginx reload",
]
print
print 'the following commands will be run:'
print ' * ' + '\n * '.join(cmds)
print
if raw_input('enter to continue, ctrl-c to quit') == '':
with stack_conn(stackname):
return run(" && ".join(cmds))
except AssertionError, ex:
print
print "* " + str(ex)
print
exit(1)
def aws_update_many_projects(pname_list):
minions = ' or '.join(map(lambda pname: pname + "-*", pname_list))
region = aws.find_region()
with core.stack_conn(core.find_master(region)):
sudo("salt -C '%s' state.highstate --retcode-passthrough" % minions)
def switch_revision_update_instance(stackname, revision=None):
switch_revision(stackname, revision)
with stack_conn(stackname):
return bootstrap.run_script('highstate.sh')
def fetch_cert(stackname):
# NOTE: this was ported from the old builder and won't work with new instances
# this isn't a problem because new instances shouldn't be using letsencrypt if
# they can avoid it.
try:
# replicates some logic in builder core
pname = core.project_name_from_stackname(stackname)
project_data = project.project_data(pname)
assert 'subdomain' in project_data, "project subdomain not found. quitting"
instance_id = stackname[len(pname + "-"):]
is_prod = instance_id in ['master', 'production']
# we still have some instances that are the production/master
# instances but don't adhere to the naming yet.
old_prods = [
'elife-ci-2015-11-04',
]
if not is_prod and stackname in old_prods:
is_prod = True
hostname_data = core.hostname_struct(stackname)
domain_names = [hostname_data['full_hostname']]
if is_prod:
project_hostname = hostname_data['project_hostname']
if acme_enabled(project_hostname):
domain_names.append(project_hostname)
else:
print '* project hostname (%s) doesnt appear to have letsencrypt enabled, ignore' % project_hostname
print '\nthese hosts will be targeted:'
print '* ' + '\n* '.join(domain_names)
#pillar_data = cfngen.salt_pillar_data(config.PILLAR_DIR)
# server = {
# 'staging': pillar_data['sys']['webserver']['acme_staging_server'],
# 'live': pillar_data['sys']['webserver']['acme_server'],
#}
server = {
'staging': "https://acme-staging.api.letsencrypt.org/directory",
'live': "https://acme-v01.api.letsencrypt.org/directory",
}
certtype = utils._pick("certificate type", ['staging', 'live'])
cmds = [
"cd /opt/letsencrypt/",
"./fetch-ssl-certs.sh -d %s --server %s" %
(" -d ".join(domain_names), server[certtype]),
"sudo service nginx reload",
]
print
print 'the following commands will be run:'
print ' * ' + '\n * '.join(cmds)
print
if raw_input('enter to continue, ctrl-c to quit') == '':
with stack_conn(stackname):
return run(" && ".join(cmds))
except AssertionError as ex:
print
print "* " + str(ex)
print
exit(1)
def kick():
stackname = core.find_master(core.find_region())
with core.stack_conn(stackname, user=config.BOOTSTRAP_USER):
bootstrap.run_script('kick-master.sh')
def repair_cfn_info(stackname):
with stack_conn(stackname):
bootstrap.write_environment_info(stackname, overwrite=True)
def regenerate_results(stackname):
with stack_conn(stackname):
with cd("/srv/elife-metrics/"):
run('./import-all-metrics.sh')
def _update_syslog(stackname):
with stack_conn(stackname):
cmd = "salt-call state.sls_id syslog-ng-hook base.syslog-ng test=True"
return sudo(cmd)
def salt_master_cmd(cmd, module='cmd.run', minions=r'\*'):
"runs the given command on all aws instances. given command must escape double quotes"
with stack_conn(core.find_master(utils.find_region())):
sudo("salt %(minions)s %(module)s %(cmd)s --timeout=30" % locals())
def highstate(stackname):
"a fast update with many caveats. prefer `update` instead"
with stack_conn(stackname, username=BOOTSTRAP_USER):
bootstrap.run_script('highstate.sh')
def pillar(stackname):
"returns the pillar data a minion is using"
with stack_conn(stackname, username=BOOTSTRAP_USER):
sudo('salt-call pillar.items')
def pillar(stackname):
"returns the pillar data a minion is using"
with stack_conn(stackname, username=BOOTSTRAP_USER):
remote_sudo('salt-call pillar.items')
def test_bootstrap_run_script(self):
with core.stack_conn(self.stackname, username=BOOTSTRAP_USER):
bootstrap.run_script('test.sh')
def download_file(stackname, path, destination):
fname = os.path.basename(path)
utils.mkdirp(destination)
with stack_conn(stackname):
get(path, destination, use_sudo=True)
def regenerate_results(stackname):
with stack_conn(stackname):
with cd("/srv/elife-metrics/"):
run('./import-all-metrics.sh')
def test_bootstrap_write_environment_info(self):
with core.stack_conn(self.stackname, username=BOOTSTRAP_USER):
bootstrap.write_environment_info(self.stackname, overwrite=False)
bootstrap.write_environment_info(self.stackname, overwrite=True)
