def ssh_config(host_string=None):
"""
Return ssh configuration dict for current env.host_string host value.
Memoizes the loaded SSH config file, but not the specific per-host results.
This function performs the necessary "is SSH config enabled?" checks and
will simply return an empty dict if not. If SSH config *is* enabled and the
value of env.ssh_config_path is not a valid file, it will abort.
May give an explicit host string as ``host_string``.
"""
from fabric.state import env
if not env.use_ssh_config:
return {}
if '_ssh_config' not in env:
try:
conf = ssh.SSHConfig()
path = os.path.expanduser(env.ssh_config_path)
with open(path) as fd:
conf.parse(fd)
env._ssh_config = conf
except IOError:
abort("Unable to load SSH config file '%s'" % path)
host = parse_host_string(host_string or env.host_string)['host']
return env._ssh_config.lookup(host)
def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force):
"""Do a build on the build server."""
import ssh
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not options.resetserver:
print "Checking for valid existing build server"
if (os.path.exists(os.path.join('builder', 'Vagrantfile'))
and os.path.exists(os.path.join('builder', '.vagrant'))):
print "...VM is present"
p = subprocess.Popen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') != -1:
print "...snapshot exists - resetting build server to clean state"
p = subprocess.Popen(['vagrant', 'status'],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('running') != -1:
print "...suspending"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'restore', 'fdroidclean'
],
cwd='builder') == 0:
print "...reset to snapshot - server is valid"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
vm_ok = True
else:
print "...failed to reset to snapshot"
else:
print "...snapshot doesn't exist - vagrant snap said:\n" + output
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
print "Removing broken/incomplete/unwanted build server"
subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant::Config.run do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write(
'config.vm.customize ["modifyvm", :id, "--memory", "768"]\n')
vf.write('end\n')
print "Starting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
print "Connecting to virtual machine..."
if subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder',
shell=True) != 0:
raise BuildException("Error getting ssh config")
vagranthost = 'default' # Host in ssh config file
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'],
username=sshconfig['user'],
port=int(sshconfig['port']),
timeout=300,
look_for_keys=False,
key_filename=idfile)
sshs.close()
print "Saving clean state of new build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'take', 'fdroidclean'
],
cwd='builder') != 0:
raise BuildException("Failed to take snapshot")
print "Restarting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Make sure it worked...
p = subprocess.Popen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') == -1:
raise BuildException("Failed to take snapshot.")
try:
# Get SSH configuration settings for us to connect...
print "Getting ssh configuration..."
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder',
shell=True)
vagranthost = 'default' # Host in ssh config file
# Load and parse the SSH config...
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
print "Connecting to virtual machine..."
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'],
username=sshconfig['user'],
port=int(sshconfig['port']),
timeout=300,
look_for_keys=False,
key_filename=idfile)
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
if not os.path.islink(os.path.join(root, rr, ff)):
ftp.put(os.path.join(root, rr, ff), ff)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
print "Preparing server for build..."
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'),
'config.py')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir('/home/vagrant')
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy the main app source code
if os.path.exists(build_dir):
send_dir(build_dir)
# Copy any extlibs that are required...
if 'extlibs' in thisbuild:
ftp.chdir('/home/vagrant/build/extlib')
for lib in thisbuild['extlibs'].split(';'):
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(os.path.join('build/extlib', lib), lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if 'srclibs' in thisbuild:
for lib in thisbuild['srclibs'].split(';'):
name, _ = lib.split('@')
if options.verbose:
print "Processing srclib '" + name + "'"
srclibpaths.append((name,
common.getsrclib(lib,
'build/srclib',
sdk_path,
basepath=True,
prepare=False)))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, lib in srclibpaths:
print "Sending srclib '" + lib + "'"
ftp.chdir('/home/vagrant/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
send_dir(lib)
# Copy the metadata file too...
ftp.chdir('/home/vagrant/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt')
# Execute the build script...
print "Starting build..."
chan = sshs.get_transport().open_session()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode']
chan.exec_command(cmdline)
output = ''
error = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
print "...getting exit status"
returncode = chan.recv_exit_status()
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
if returncode != 0:
raise BuildException(
"Build.py failed on server for %s:%s" %
(app['id'], thisbuild['version']), output.strip(),
error.strip())
# Retrieve the built files...
print "Retrieving build output..."
if force:
ftp.chdir('/home/vagrant/tmp')
else:
ftp.chdir('/home/vagrant/unsigned')
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
output.strip(), error.strip())
ftp.close()
finally:
# Suspend the build server.
print "Suspending build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def main(args=None):
parser = argparse.ArgumentParser(add_help=False,
description='''
Command line utility for packaging and installing vagrant boxes built by
executing fabric tasks. %(prog)s is implemented as a thin wrapper around
fabric's 'fab' command. It creates a temporary vagrant environment, configures
fabric to connect to it, then hands off execution to fabric, which runs its
tasks against the environment. When finished, it packages and/or installs the
built box according to the arguments provided.''')
meta = parser.add_argument_group(title='optional arguments')
meta.add_argument(
'-h',
'--help',
action='store_true',
help='''Show this help message and exit. For help with fabric options,
run 'fab --help'.''')
meta.add_argument('-V',
'--version',
action='store_true',
help='''show program's version number and exit''')
meta.add_argument('--log-level',
help='Level for logging program output',
default='warning',
type=log_level)
# Primary arguments for basebox command.
main = parser.add_argument_group(
title='build arguments',
description='''Arguments to configure building, installation, and
packaging''')
main.add_argument(
'--base',
help='''Vagrant base box to build with. Defaults to vagrant's
precise64 box, also available at
https://files.vagrantup.com/precise64.box''',
default='https://files.vagrantup.com/precise64.box')
main.add_argument(
'--vagrantfile-template',
# help='Jinja template for rendering Vagrantfile to build with',
help=argparse.SUPPRESS,
default=TEMPLATE_ENV.get_template('Vagrantfile.default'),
type=TEMPLATE_ENV.get_template)
main.add_argument('--package-as',
metavar='PACKAGE_FILE',
help='Package file to write the build result to.')
main.add_argument(
'--package-vagrantfile',
help='''Specify how/whether to set the output package Vagrantfile. The
default is 'inherit', which packages with the Vagrantfile of the
base box if it exists. Specifying 'none' will create the package
without a Vagrantfile. If a file path or raw string is provided,
its contents will be used as the output package Vagrantfile.
''',
default='inherit',
type=package_vagrantfile,
metavar='(VFILE_PATH|VFILE_STRING|inherit|none)')
main.add_argument('--install-as',
help='Install the built box to vagrant as BOXNAME',
metavar='BOXNAME')
# Arguments shared with fabric
shared = parser.add_argument_group(
title='shared arguments',
description='''These arguments are arguments to fabric, but
%(prog)s also uses them to configure its environment. They are
passed through to fabric when it executes.''')
shared.add_argument(
'-i',
metavar='PATH',
help='''Path to SSH private key file. May be repeated. Added to the
generated Vagrantfile as the 'config.ssh.private_key_path' option.
'''),
shared.add_argument(
'-u',
'--user',
help='''Username to use when connecting to vagrant hosts. Added to
the generated Vagrantfile as the 'config.ssh.username' option.
'''),
shared.add_argument(
'--port',
help='''SSH connection port. Added to the generated Vagrantfile as the
'config.ssh.port' option''')
# OK, this one doesn't do anything extra, but we still want to sanity-check
# the fabfile before executing.
shared.add_argument('-f', '--fabfile', help=argparse.SUPPRESS)
# The 'hosts' parameter works a bit differently in basebox - each hostname
# must be a valid identifier name and can be assigned roles by listing them
# in the format 'host:role1,role2'.
#
# >>> basebox -H box1:web box2:db,cache
main.add_argument('-H',
'--hosts',
help='',
nargs='+',
type=host_with_roles,
action='append')
args, fab_args = parser.parse_known_args(args=args)
# Set log level so everything after this can emit proper logs
LOG.level = args.log_level
# Remove the separator from the fabric arguments
if '--' in fab_args:
fab_args.remove('--')
# Flatten host-role entries and map bidirectionally
host_roles = {}
roledefs = {}
for host, roles in [tpl for sublist in args.hosts for tpl in sublist]:
if roles:
LOG.info('Host <%s> specified with roles: %s' % (host, roles))
else:
LOG.info('Host <%s> specified with no roles.' % host)
host_roles[host] = roles
for role in roles:
roledefs.setdefault(role, []).append(host)
# Add hosts to fabric args
fab_args[:0] = ['--hosts', ','.join(host_roles.keys())]
# Duplicate shared args back into the fabric argument list
for action in shared._group_actions:
if getattr(args, action.dest, None):
flag = ('-%s' if len(action.dest) == 1 else '--%s') % action.dest
fab_args[:0] = [flag, getattr(args, action.dest)]
LOG.info('Checking fabric parameters: %s' % fab_args)
if args.help:
parser.print_help()
elif args.version:
print_version()
else:
if not (args.install_as or args.package_as):
print 'No action specified (you should use --install-as or --package-as).'
else:
# Replace sys.argv to simulate calling fabric as a CLI script
argv_original = sys.argv
stderr_original = sys.stderr
sys.argv = ['fab'] + fab_args
sys.stderr = StringIO.StringIO()
# Sanity-check fabric arguments before doing anything heavy
try:
fabric.main.parse_options()
except SystemExit:
print(
'An error was encountered while trying to parse the '
'arguments to fabric:')
print ''
print os.linesep.join(
'\t%s' % line
for line in sys.stderr.getvalue().splitlines()[2:])
print ''
print 'Please check the syntax and try again.'
raise
finally:
sys.stderr = stderr_original
# Check fabfile resolution
fabfile_original = fabric.state.env.fabfile
if args.fabfile:
fabric.state.env.fabfile = args.fabfile
if not fabric.main.find_fabfile():
print(
"Fabric couldn't find any fabfiles! (You may want to "
"change directories or specify the -f option)")
raise SystemExit
fabric.state.env.fabfile = fabfile_original
# Render input Vagrantfile with appropriate context
vfile_ctx = {
'base': args.base,
'hosts': host_roles.keys(),
'ssh': {
'username': args.user,
'private_key_path': args.i,
'port': args.port
}
}
mode_local()
with tempbox(base=args.base,
vfile_template=args.vagrantfile_template,
vfile_template_context=vfile_ctx) as default_box:
context = default_box.context
# Fabricate an SSH config for the vagrant environment and add
# it to fabric.state.env
ssh_conf = ssh.SSHConfig()
ssh_configs = ssh_conf._config
# Add a host entry to SSH config for each vagrant box
for box_name in context.list_boxes():
box = context[box_name]
box.up()
ssh_settings = box.ssh_config()
ssh_settings.update({
'host': box_name,
'stricthostkeychecking': 'no'
})
ssh_configs.append(ssh_settings)
fabric.api.settings.use_ssh_config = True
fabric.api.env._ssh_config = ssh_conf
# Configure roledefs
fabric.state.env.roledefs = roledefs
# Hand execution over to fabric
with mode_remote():
try:
fabric.main.main()
except SystemExit as e: # Raised when fabric finishes.
if e.code is not 0:
LOG.error('Fabric exited with error code %s' %
e.code)
raise
pass
vfile = resolve_package_vagrantfile(args.package_vagrantfile)
context.package(vagrantfile=vfile,
install_as=args.install_as,
output=args.package_as)
sys.argv = argv_original