def build_server(app, thisbuild, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
paramiko
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
logging.getLogger("paramiko").setLevel(logging.DEBUG)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
sshinfo = get_clean_vm()
try:
# Open SSH connection...
logging.info("Connecting to virtual machine...")
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'],
username=sshinfo['user'],
port=sshinfo['port'],
timeout=300,
look_for_keys=False,
key_filename=sshinfo['idfile'])
homedir = '/home/' + sshinfo['user']
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir(homedir)
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
lfile = os.path.join(root, rr, ff)
if not os.path.islink(lfile):
ftp.put(lfile, ff)
ftp.chmod(ff, os.stat(lfile).st_mode)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py')
ftp.put(
os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
subprocess.call('git rev-parse HEAD >' +
os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'),
shell=True,
cwd=serverpath)
ftp.put('tmp/fdroidserverid', 'fdroidserverid')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if thisbuild['extlibs']:
ftp.chdir(homedir + '/build/extlib')
for lib in thisbuild['extlibs']:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib {0}".format(libsrc))
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(libsrc, lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if thisbuild['srclibs']:
for lib in thisbuild['srclibs']:
srclibpaths.append(
common.getsrclib(lib,
'build/srclib',
srclibpaths,
basepath=True,
prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
logging.info("Sending srclib '%s'" % lib)
ftp.chdir(homedir + '/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
fv = '.fdroidvcs-' + name
ftp.put(os.path.join('build/srclib', fv), fv)
send_dir(lib)
# Copy the metadata file too...
ftp.chdir(homedir + '/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt')
# Copy the main app source code
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
fv = '.fdroidvcs-' + app['id']
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
# Execute the build script...
logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app['id'], thisbuild['vercode'])
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
time.sleep(0.1)
logging.info("...getting exit status")
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
if len(get) == 0:
break
output += get
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app['id'], thisbuild['version']), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
if force:
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, thisbuild)
tarball = common.getsrcname(app, thisbuild)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app['id'], thisbuild['version']), output)
ftp.close()
finally:
# Suspend the build server.
release_vm()
def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force):
"""Do a build on the build server."""
import ssh
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not options.resetserver:
print "Checking for valid existing build server"
if (os.path.exists(os.path.join('builder', 'Vagrantfile')) and
os.path.exists(os.path.join('builder', '.vagrant'))):
print "...VM is present"
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
cwd='builder', stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') != -1:
print "...snapshot exists - resetting build server to clean state"
p = subprocess.Popen(['vagrant', 'status'],
cwd='builder', stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('running') != -1:
print "...suspending"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'],
cwd='builder') == 0:
print "...reset to snapshot - server is valid"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
vm_ok = True
else:
print "...failed to reset to snapshot"
else:
print "...snapshot doesn't exist - vagrant snap said:\n" + output
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
print "Removing broken/incomplete/unwanted build server"
subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant::Config.run do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('config.vm.customize ["modifyvm", :id, "--memory", "768"]\n')
vf.write('end\n')
print "Starting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
print "Connecting to virtual machine..."
if subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True) != 0:
raise BuildException("Error getting ssh config")
vagranthost = 'default' # Host in ssh config file
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
port=int(sshconfig['port']), timeout=300, look_for_keys=False,
key_filename=idfile)
sshs.close()
print "Saving clean state of new build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'],
cwd='builder') != 0:
raise BuildException("Failed to take snapshot")
print "Restarting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Make sure it worked...
p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'],
cwd='builder', stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') == -1:
raise BuildException("Failed to take snapshot.")
try:
# Get SSH configuration settings for us to connect...
print "Getting ssh configuration..."
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder', shell=True)
vagranthost = 'default' # Host in ssh config file
# Load and parse the SSH config...
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
print "Connecting to virtual machine..."
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'], username=sshconfig['user'],
port=int(sshconfig['port']), timeout=300, look_for_keys=False,
key_filename=idfile)
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
if not os.path.islink(os.path.join(root, rr, ff)):
ftp.put(os.path.join(root, rr, ff), ff)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
print "Preparing server for build..."
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir('/home/vagrant')
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy the main app source code
if os.path.exists(build_dir):
send_dir(build_dir)
# Copy any extlibs that are required...
if 'extlibs' in thisbuild:
ftp.chdir('/home/vagrant/build/extlib')
for lib in thisbuild['extlibs'].split(';'):
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(os.path.join('build/extlib', lib), lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if 'srclibs' in thisbuild:
for lib in thisbuild['srclibs'].split(';'):
name, _ = lib.split('@')
if options.verbose:
print "Processing srclib '" + name + "'"
srclibpaths.append((name, common.getsrclib(lib, 'build/srclib', sdk_path, basepath=True, prepare=False)))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, lib in srclibpaths:
print "Sending srclib '" + lib + "'"
ftp.chdir('/home/vagrant/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
send_dir(lib)
# Copy the metadata file too...
ftp.chdir('/home/vagrant/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'),
name + '.txt')
# Execute the build script...
print "Starting build..."
chan = sshs.get_transport().open_session()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode']
chan.exec_command(cmdline)
output = ''
error = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
print "...getting exit status"
returncode = chan.recv_exit_status()
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
if returncode != 0:
raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
# Retrieve the built files...
print "Retrieving build output..."
if force:
ftp.chdir('/home/vagrant/tmp')
else:
ftp.chdir('/home/vagrant/unsigned')
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip())
ftp.close()
finally:
# Suspend the build server.
print "Suspending build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force):
"""Do a build on the build server."""
import ssh
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not options.resetserver:
print "Checking for valid existing build server"
if (os.path.exists(os.path.join('builder', 'Vagrantfile'))
and os.path.exists(os.path.join('builder', '.vagrant'))):
print "...VM is present"
p = subprocess.Popen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') != -1:
print "...snapshot exists - resetting build server to clean state"
p = subprocess.Popen(['vagrant', 'status'],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('running') != -1:
print "...suspending"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'restore', 'fdroidclean'
],
cwd='builder') == 0:
print "...reset to snapshot - server is valid"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
vm_ok = True
else:
print "...failed to reset to snapshot"
else:
print "...snapshot doesn't exist - vagrant snap said:\n" + output
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
print "Removing broken/incomplete/unwanted build server"
subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant::Config.run do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write(
'config.vm.customize ["modifyvm", :id, "--memory", "768"]\n')
vf.write('end\n')
print "Starting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
print "Connecting to virtual machine..."
if subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder',
shell=True) != 0:
raise BuildException("Error getting ssh config")
vagranthost = 'default' # Host in ssh config file
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'],
username=sshconfig['user'],
port=int(sshconfig['port']),
timeout=300,
look_for_keys=False,
key_filename=idfile)
sshs.close()
print "Saving clean state of new build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
if subprocess.call([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'take', 'fdroidclean'
],
cwd='builder') != 0:
raise BuildException("Failed to take snapshot")
print "Restarting new build server"
if subprocess.call(['vagrant', 'up'], cwd='builder') != 0:
raise BuildException("Failed to start build server")
# Make sure it worked...
p = subprocess.Popen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder',
stdout=subprocess.PIPE)
output = p.communicate()[0]
if output.find('fdroidclean') == -1:
raise BuildException("Failed to take snapshot.")
try:
# Get SSH configuration settings for us to connect...
print "Getting ssh configuration..."
subprocess.call('vagrant ssh-config >sshconfig',
cwd='builder',
shell=True)
vagranthost = 'default' # Host in ssh config file
# Load and parse the SSH config...
sshconfig = ssh.SSHConfig()
sshf = open('builder/sshconfig', 'r')
sshconfig.parse(sshf)
sshf.close()
sshconfig = sshconfig.lookup(vagranthost)
# Open SSH connection...
print "Connecting to virtual machine..."
sshs = ssh.SSHClient()
sshs.set_missing_host_key_policy(ssh.AutoAddPolicy())
idfile = sshconfig['identityfile']
if idfile.startswith('"') and idfile.endswith('"'):
idfile = idfile[1:-1]
sshs.connect(sshconfig['hostname'],
username=sshconfig['user'],
port=int(sshconfig['port']),
timeout=300,
look_for_keys=False,
key_filename=idfile)
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir('/home/vagrant')
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
if not os.path.islink(os.path.join(root, rr, ff)):
ftp.put(os.path.join(root, rr, ff), ff)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
print "Preparing server for build..."
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.put(os.path.join(serverpath, 'build.py'), 'build.py')
ftp.put(os.path.join(serverpath, 'common.py'), 'common.py')
ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'),
'config.py')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app['id'] + '.txt'),
app['id'] + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app['id'])):
send_dir(os.path.join('metadata', app['id']))
ftp.chdir('/home/vagrant')
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy the main app source code
if os.path.exists(build_dir):
send_dir(build_dir)
# Copy any extlibs that are required...
if 'extlibs' in thisbuild:
ftp.chdir('/home/vagrant/build/extlib')
for lib in thisbuild['extlibs'].split(';'):
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(os.path.join('build/extlib', lib), lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if 'srclibs' in thisbuild:
for lib in thisbuild['srclibs'].split(';'):
name, _ = lib.split('@')
if options.verbose:
print "Processing srclib '" + name + "'"
srclibpaths.append((name,
common.getsrclib(lib,
'build/srclib',
sdk_path,
basepath=True,
prepare=False)))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, lib in srclibpaths:
print "Sending srclib '" + lib + "'"
ftp.chdir('/home/vagrant/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
send_dir(lib)
# Copy the metadata file too...
ftp.chdir('/home/vagrant/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt')
# Execute the build script...
print "Starting build..."
chan = sshs.get_transport().open_session()
cmdline = 'python build.py --on-server'
if force:
cmdline += ' --force --test'
cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode']
chan.exec_command(cmdline)
output = ''
error = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
print "...getting exit status"
returncode = chan.recv_exit_status()
while chan.recv_ready():
output += chan.recv(1024)
while chan.recv_stderr_ready():
error += chan.recv_stderr(1024)
if returncode != 0:
raise BuildException(
"Build.py failed on server for %s:%s" %
(app['id'], thisbuild['version']), output.strip(),
error.strip())
# Retrieve the built files...
print "Retrieving build output..."
if force:
ftp.chdir('/home/vagrant/tmp')
else:
ftp.chdir('/home/vagrant/unsigned')
apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk'
tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz'
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
output.strip(), error.strip())
ftp.close()
finally:
# Suspend the build server.
print "Suspending build server"
subprocess.call(['vagrant', 'suspend'], cwd='builder')
def build_server(app, build, vcs, build_dir, output_dir, force):
"""Do a build on the build server."""
try:
paramiko
except NameError:
raise BuildException("Paramiko is required to use the buildserver")
if options.verbose:
logging.getLogger("paramiko").setLevel(logging.INFO)
else:
logging.getLogger("paramiko").setLevel(logging.WARN)
sshinfo = get_clean_vm()
try:
# Open SSH connection...
logging.info("Connecting to virtual machine...")
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
port=sshinfo['port'], timeout=300,
look_for_keys=False, key_filename=sshinfo['idfile'])
homedir = '/home/' + sshinfo['user']
# Get an SFTP connection...
ftp = sshs.open_sftp()
ftp.get_channel().settimeout(15)
# Put all the necessary files in place...
ftp.chdir(homedir)
# Helper to copy the contents of a directory to the server...
def send_dir(path):
root = os.path.dirname(path)
main = os.path.basename(path)
ftp.mkdir(main)
for r, d, f in os.walk(path):
rr = os.path.relpath(r, root)
ftp.chdir(rr)
for dd in d:
ftp.mkdir(dd)
for ff in f:
lfile = os.path.join(root, rr, ff)
if not os.path.islink(lfile):
ftp.put(lfile, ff)
ftp.chmod(ff, os.stat(lfile).st_mode)
for i in range(len(rr.split('/'))):
ftp.chdir('..')
ftp.chdir('..')
logging.info("Preparing server for build...")
serverpath = os.path.abspath(os.path.dirname(__file__))
ftp.mkdir('fdroidserver')
ftp.chdir('fdroidserver')
ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid')
ftp.chmod('fdroid', 0o755)
send_dir(os.path.join(serverpath))
ftp.chdir(homedir)
ftp.put(os.path.join(serverpath, '..', 'buildserver',
'config.buildserver.py'), 'config.py')
ftp.chmod('config.py', 0o600)
# Copy over the ID (head commit hash) of the fdroidserver in use...
subprocess.call('git rev-parse HEAD >' +
os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'),
shell=True, cwd=serverpath)
ftp.put('tmp/fdroidserverid', 'fdroidserverid')
# Copy the metadata - just the file for this app...
ftp.mkdir('metadata')
ftp.mkdir('srclibs')
ftp.chdir('metadata')
ftp.put(os.path.join('metadata', app.id + '.txt'),
app.id + '.txt')
# And patches if there are any...
if os.path.exists(os.path.join('metadata', app.id)):
send_dir(os.path.join('metadata', app.id))
ftp.chdir(homedir)
# Create the build directory...
ftp.mkdir('build')
ftp.chdir('build')
ftp.mkdir('extlib')
ftp.mkdir('srclib')
# Copy any extlibs that are required...
if build.extlibs:
ftp.chdir(homedir + '/build/extlib')
for lib in build.extlibs:
lib = lib.strip()
libsrc = os.path.join('build/extlib', lib)
if not os.path.exists(libsrc):
raise BuildException("Missing extlib {0}".format(libsrc))
lp = lib.split('/')
for d in lp[:-1]:
if d not in ftp.listdir():
ftp.mkdir(d)
ftp.chdir(d)
ftp.put(libsrc, lp[-1])
for _ in lp[:-1]:
ftp.chdir('..')
# Copy any srclibs that are required...
srclibpaths = []
if build.srclibs:
for lib in build.srclibs:
srclibpaths.append(
common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False))
# If one was used for the main source, add that too.
basesrclib = vcs.getsrclib()
if basesrclib:
srclibpaths.append(basesrclib)
for name, number, lib in srclibpaths:
logging.info("Sending srclib '%s'" % lib)
ftp.chdir(homedir + '/build/srclib')
if not os.path.exists(lib):
raise BuildException("Missing srclib directory '" + lib + "'")
fv = '.fdroidvcs-' + name
ftp.put(os.path.join('build/srclib', fv), fv)
send_dir(lib)
# Copy the metadata file too...
ftp.chdir(homedir + '/srclibs')
ftp.put(os.path.join('srclibs', name + '.txt'),
name + '.txt')
# Copy the main app source code
# (no need if it's a srclib)
if (not basesrclib) and os.path.exists(build_dir):
ftp.chdir(homedir + '/build')
fv = '.fdroidvcs-' + app.id
ftp.put(os.path.join('build', fv), fv)
send_dir(build_dir)
# Execute the build script...
logging.info("Starting build...")
chan = sshs.get_transport().open_session()
chan.get_pty()
cmdline = os.path.join(homedir, 'fdroidserver', 'fdroid')
cmdline += ' build --on-server'
if force:
cmdline += ' --force --test'
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app.id, build.vercode)
chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"')
output = ''
while not chan.exit_status_ready():
while chan.recv_ready():
output += chan.recv(1024)
time.sleep(0.1)
logging.info("...getting exit status")
returncode = chan.recv_exit_status()
while True:
get = chan.recv(1024)
if len(get) == 0:
break
output += get
if returncode != 0:
raise BuildException(
"Build.py failed on server for {0}:{1}".format(
app.id, build.version), output)
# Retrieve the built files...
logging.info("Retrieving build output...")
if force:
ftp.chdir(homedir + '/tmp')
else:
ftp.chdir(homedir + '/unsigned')
apkfile = common.getapkname(app, build)
tarball = common.getsrcname(app, build)
try:
ftp.get(apkfile, os.path.join(output_dir, apkfile))
if not options.notarball:
ftp.get(tarball, os.path.join(output_dir, tarball))
except:
raise BuildException(
"Build failed for %s:%s - missing output files".format(
app.id, build.version), output)
ftp.close()
finally:
# Suspend the build server.
release_vm()
