def build_server(app, thisbuild, vcs, build_dir, output_dir, force): """Do a build on the build server.""" try: paramiko except NameError: raise BuildException("Paramiko is required to use the buildserver") if options.verbose: logging.getLogger("paramiko").setLevel(logging.DEBUG) else: logging.getLogger("paramiko").setLevel(logging.WARN) sshinfo = get_clean_vm() try: # Open SSH connection... logging.info("Connecting to virtual machine...") sshs = paramiko.SSHClient() sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) homedir = '/home/' + sshinfo['user'] # Get an SFTP connection... ftp = sshs.open_sftp() ftp.get_channel().settimeout(15) # Put all the necessary files in place... ftp.chdir(homedir) # Helper to copy the contents of a directory to the server... def send_dir(path): root = os.path.dirname(path) main = os.path.basename(path) ftp.mkdir(main) for r, d, f in os.walk(path): rr = os.path.relpath(r, root) ftp.chdir(rr) for dd in d: ftp.mkdir(dd) for ff in f: lfile = os.path.join(root, rr, ff) if not os.path.islink(lfile): ftp.put(lfile, ff) ftp.chmod(ff, os.stat(lfile).st_mode) for i in range(len(rr.split('/'))): ftp.chdir('..') ftp.chdir('..') logging.info("Preparing server for build...") serverpath = os.path.abspath(os.path.dirname(__file__)) ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') ftp.put(os.path.join(serverpath, 'metadata.py'), 'metadata.py') ftp.put( os.path.join(serverpath, '..', 'buildserver', 'config.buildserver.py'), 'config.py') ftp.chmod('config.py', 0o600) # Copy over the ID (head commit hash) of the fdroidserver in use... subprocess.call('git rev-parse HEAD >' + os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'), shell=True, cwd=serverpath) ftp.put('tmp/fdroidserverid', 'fdroidserverid') # Copy the metadata - just the file for this app... ftp.mkdir('metadata') ftp.mkdir('srclibs') ftp.chdir('metadata') ftp.put(os.path.join('metadata', app['id'] + '.txt'), app['id'] + '.txt') # And patches if there are any... if os.path.exists(os.path.join('metadata', app['id'])): send_dir(os.path.join('metadata', app['id'])) ftp.chdir(homedir) # Create the build directory... ftp.mkdir('build') ftp.chdir('build') ftp.mkdir('extlib') ftp.mkdir('srclib') # Copy any extlibs that are required... if thisbuild['extlibs']: ftp.chdir(homedir + '/build/extlib') for lib in thisbuild['extlibs']: lib = lib.strip() libsrc = os.path.join('build/extlib', lib) if not os.path.exists(libsrc): raise BuildException("Missing extlib {0}".format(libsrc)) lp = lib.split('/') for d in lp[:-1]: if d not in ftp.listdir(): ftp.mkdir(d) ftp.chdir(d) ftp.put(libsrc, lp[-1]) for _ in lp[:-1]: ftp.chdir('..') # Copy any srclibs that are required... srclibpaths = [] if thisbuild['srclibs']: for lib in thisbuild['srclibs']: srclibpaths.append( common.getsrclib(lib, 'build/srclib', srclibpaths, basepath=True, prepare=False)) # If one was used for the main source, add that too. basesrclib = vcs.getsrclib() if basesrclib: srclibpaths.append(basesrclib) for name, number, lib in srclibpaths: logging.info("Sending srclib '%s'" % lib) ftp.chdir(homedir + '/build/srclib') if not os.path.exists(lib): raise BuildException("Missing srclib directory '" + lib + "'") fv = '.fdroidvcs-' + name ftp.put(os.path.join('build/srclib', fv), fv) send_dir(lib) # Copy the metadata file too... ftp.chdir(homedir + '/srclibs') ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt') # Copy the main app source code # (no need if it's a srclib) if (not basesrclib) and os.path.exists(build_dir): ftp.chdir(homedir + '/build') fv = '.fdroidvcs-' + app['id'] ftp.put(os.path.join('build', fv), fv) send_dir(build_dir) # Execute the build script... logging.info("Starting build...") chan = sshs.get_transport().open_session() chan.get_pty() cmdline = 'python build.py --on-server' if force: cmdline += ' --force --test' if options.verbose: cmdline += ' --verbose' cmdline += " %s:%s" % (app['id'], thisbuild['vercode']) chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"') output = '' while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) time.sleep(0.1) logging.info("...getting exit status") returncode = chan.recv_exit_status() while True: get = chan.recv(1024) if len(get) == 0: break output += get if returncode != 0: raise BuildException( "Build.py failed on server for {0}:{1}".format( app['id'], thisbuild['version']), output) # Retrieve the built files... logging.info("Retrieving build output...") if force: ftp.chdir(homedir + '/tmp') else: ftp.chdir(homedir + '/unsigned') apkfile = common.getapkname(app, thisbuild) tarball = common.getsrcname(app, thisbuild) try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) if not options.notarball: ftp.get(tarball, os.path.join(output_dir, tarball)) except: raise BuildException( "Build failed for %s:%s - missing output files".format( app['id'], thisbuild['version']), output) ftp.close() finally: # Suspend the build server. release_vm()
def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force): """Do a build on the build server.""" import ssh # Reset existing builder machine to a clean state if possible. vm_ok = False if not options.resetserver: print "Checking for valid existing build server" if (os.path.exists(os.path.join('builder', 'Vagrantfile')) and os.path.exists(os.path.join('builder', '.vagrant'))): print "...VM is present" p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('fdroidclean') != -1: print "...snapshot exists - resetting build server to clean state" p = subprocess.Popen(['vagrant', 'status'], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('running') != -1: print "...suspending" subprocess.call(['vagrant', 'suspend'], cwd='builder') if subprocess.call(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'], cwd='builder') == 0: print "...reset to snapshot - server is valid" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") vm_ok = True else: print "...failed to reset to snapshot" else: print "...snapshot doesn't exist - vagrant snap said:\n" + output # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: if os.path.exists('builder'): print "Removing broken/incomplete/unwanted build server" subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') shutil.rmtree('builder') os.mkdir('builder') with open('builder/Vagrantfile', 'w') as vf: vf.write('Vagrant::Config.run do |config|\n') vf.write('config.vm.box = "buildserver"\n') vf.write('config.vm.customize ["modifyvm", :id, "--memory", "768"]\n') vf.write('end\n') print "Starting new build server" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") # Open SSH connection to make sure it's working and ready... print "Connecting to virtual machine..." if subprocess.call('vagrant ssh-config >sshconfig', cwd='builder', shell=True) != 0: raise BuildException("Error getting ssh config") vagranthost = 'default' # Host in ssh config file sshconfig = ssh.SSHConfig() sshf = open('builder/sshconfig', 'r') sshconfig.parse(sshf) sshf.close() sshconfig = sshconfig.lookup(vagranthost) sshs = ssh.SSHClient() sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) idfile = sshconfig['identityfile'] if idfile.startswith('"') and idfile.endswith('"'): idfile = idfile[1:-1] sshs.connect(sshconfig['hostname'], username=sshconfig['user'], port=int(sshconfig['port']), timeout=300, look_for_keys=False, key_filename=idfile) sshs.close() print "Saving clean state of new build server" subprocess.call(['vagrant', 'suspend'], cwd='builder') if subprocess.call(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'], cwd='builder') != 0: raise BuildException("Failed to take snapshot") print "Restarting new build server" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") # Make sure it worked... p = subprocess.Popen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('fdroidclean') == -1: raise BuildException("Failed to take snapshot.") try: # Get SSH configuration settings for us to connect... print "Getting ssh configuration..." subprocess.call('vagrant ssh-config >sshconfig', cwd='builder', shell=True) vagranthost = 'default' # Host in ssh config file # Load and parse the SSH config... sshconfig = ssh.SSHConfig() sshf = open('builder/sshconfig', 'r') sshconfig.parse(sshf) sshf.close() sshconfig = sshconfig.lookup(vagranthost) # Open SSH connection... print "Connecting to virtual machine..." sshs = ssh.SSHClient() sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) idfile = sshconfig['identityfile'] if idfile.startswith('"') and idfile.endswith('"'): idfile = idfile[1:-1] sshs.connect(sshconfig['hostname'], username=sshconfig['user'], port=int(sshconfig['port']), timeout=300, look_for_keys=False, key_filename=idfile) # Get an SFTP connection... ftp = sshs.open_sftp() ftp.get_channel().settimeout(15) # Put all the necessary files in place... ftp.chdir('/home/vagrant') # Helper to copy the contents of a directory to the server... def send_dir(path): root = os.path.dirname(path) main = os.path.basename(path) ftp.mkdir(main) for r, d, f in os.walk(path): rr = os.path.relpath(r, root) ftp.chdir(rr) for dd in d: ftp.mkdir(dd) for ff in f: if not os.path.islink(os.path.join(root, rr, ff)): ftp.put(os.path.join(root, rr, ff), ff) for i in range(len(rr.split('/'))): ftp.chdir('..') ftp.chdir('..') print "Preparing server for build..." serverpath = os.path.abspath(os.path.dirname(__file__)) ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py') # Copy the metadata - just the file for this app... ftp.mkdir('metadata') ftp.mkdir('srclibs') ftp.chdir('metadata') ftp.put(os.path.join('metadata', app['id'] + '.txt'), app['id'] + '.txt') # And patches if there are any... if os.path.exists(os.path.join('metadata', app['id'])): send_dir(os.path.join('metadata', app['id'])) ftp.chdir('/home/vagrant') # Create the build directory... ftp.mkdir('build') ftp.chdir('build') ftp.mkdir('extlib') ftp.mkdir('srclib') # Copy the main app source code if os.path.exists(build_dir): send_dir(build_dir) # Copy any extlibs that are required... if 'extlibs' in thisbuild: ftp.chdir('/home/vagrant/build/extlib') for lib in thisbuild['extlibs'].split(';'): lp = lib.split('/') for d in lp[:-1]: if d not in ftp.listdir(): ftp.mkdir(d) ftp.chdir(d) ftp.put(os.path.join('build/extlib', lib), lp[-1]) for _ in lp[:-1]: ftp.chdir('..') # Copy any srclibs that are required... srclibpaths = [] if 'srclibs' in thisbuild: for lib in thisbuild['srclibs'].split(';'): name, _ = lib.split('@') if options.verbose: print "Processing srclib '" + name + "'" srclibpaths.append((name, common.getsrclib(lib, 'build/srclib', sdk_path, basepath=True, prepare=False))) # If one was used for the main source, add that too. basesrclib = vcs.getsrclib() if basesrclib: srclibpaths.append(basesrclib) for name, lib in srclibpaths: print "Sending srclib '" + lib + "'" ftp.chdir('/home/vagrant/build/srclib') if not os.path.exists(lib): raise BuildException("Missing srclib directory '" + lib + "'") send_dir(lib) # Copy the metadata file too... ftp.chdir('/home/vagrant/srclibs') ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt') # Execute the build script... print "Starting build..." chan = sshs.get_transport().open_session() cmdline = 'python build.py --on-server' if force: cmdline += ' --force --test' cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode'] chan.exec_command(cmdline) output = '' error = '' while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) while chan.recv_stderr_ready(): error += chan.recv_stderr(1024) print "...getting exit status" returncode = chan.recv_exit_status() while chan.recv_ready(): output += chan.recv(1024) while chan.recv_stderr_ready(): error += chan.recv_stderr(1024) if returncode != 0: raise BuildException("Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) # Retrieve the built files... print "Retrieving build output..." if force: ftp.chdir('/home/vagrant/tmp') else: ftp.chdir('/home/vagrant/unsigned') apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) ftp.get(tarball, os.path.join(output_dir, tarball)) except: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) ftp.close() finally: # Suspend the build server. print "Suspending build server" subprocess.call(['vagrant', 'suspend'], cwd='builder')
def build_server(app, thisbuild, vcs, build_dir, output_dir, sdk_path, force): """Do a build on the build server.""" import ssh # Reset existing builder machine to a clean state if possible. vm_ok = False if not options.resetserver: print "Checking for valid existing build server" if (os.path.exists(os.path.join('builder', 'Vagrantfile')) and os.path.exists(os.path.join('builder', '.vagrant'))): print "...VM is present" p = subprocess.Popen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details' ], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('fdroidclean') != -1: print "...snapshot exists - resetting build server to clean state" p = subprocess.Popen(['vagrant', 'status'], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('running') != -1: print "...suspending" subprocess.call(['vagrant', 'suspend'], cwd='builder') if subprocess.call([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean' ], cwd='builder') == 0: print "...reset to snapshot - server is valid" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") vm_ok = True else: print "...failed to reset to snapshot" else: print "...snapshot doesn't exist - vagrant snap said:\n" + output # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: if os.path.exists('builder'): print "Removing broken/incomplete/unwanted build server" subprocess.call(['vagrant', 'destroy', '-f'], cwd='builder') shutil.rmtree('builder') os.mkdir('builder') with open('builder/Vagrantfile', 'w') as vf: vf.write('Vagrant::Config.run do |config|\n') vf.write('config.vm.box = "buildserver"\n') vf.write( 'config.vm.customize ["modifyvm", :id, "--memory", "768"]\n') vf.write('end\n') print "Starting new build server" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") # Open SSH connection to make sure it's working and ready... print "Connecting to virtual machine..." if subprocess.call('vagrant ssh-config >sshconfig', cwd='builder', shell=True) != 0: raise BuildException("Error getting ssh config") vagranthost = 'default' # Host in ssh config file sshconfig = ssh.SSHConfig() sshf = open('builder/sshconfig', 'r') sshconfig.parse(sshf) sshf.close() sshconfig = sshconfig.lookup(vagranthost) sshs = ssh.SSHClient() sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) idfile = sshconfig['identityfile'] if idfile.startswith('"') and idfile.endswith('"'): idfile = idfile[1:-1] sshs.connect(sshconfig['hostname'], username=sshconfig['user'], port=int(sshconfig['port']), timeout=300, look_for_keys=False, key_filename=idfile) sshs.close() print "Saving clean state of new build server" subprocess.call(['vagrant', 'suspend'], cwd='builder') if subprocess.call([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean' ], cwd='builder') != 0: raise BuildException("Failed to take snapshot") print "Restarting new build server" if subprocess.call(['vagrant', 'up'], cwd='builder') != 0: raise BuildException("Failed to start build server") # Make sure it worked... p = subprocess.Popen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details' ], cwd='builder', stdout=subprocess.PIPE) output = p.communicate()[0] if output.find('fdroidclean') == -1: raise BuildException("Failed to take snapshot.") try: # Get SSH configuration settings for us to connect... print "Getting ssh configuration..." subprocess.call('vagrant ssh-config >sshconfig', cwd='builder', shell=True) vagranthost = 'default' # Host in ssh config file # Load and parse the SSH config... sshconfig = ssh.SSHConfig() sshf = open('builder/sshconfig', 'r') sshconfig.parse(sshf) sshf.close() sshconfig = sshconfig.lookup(vagranthost) # Open SSH connection... print "Connecting to virtual machine..." sshs = ssh.SSHClient() sshs.set_missing_host_key_policy(ssh.AutoAddPolicy()) idfile = sshconfig['identityfile'] if idfile.startswith('"') and idfile.endswith('"'): idfile = idfile[1:-1] sshs.connect(sshconfig['hostname'], username=sshconfig['user'], port=int(sshconfig['port']), timeout=300, look_for_keys=False, key_filename=idfile) # Get an SFTP connection... ftp = sshs.open_sftp() ftp.get_channel().settimeout(15) # Put all the necessary files in place... ftp.chdir('/home/vagrant') # Helper to copy the contents of a directory to the server... def send_dir(path): root = os.path.dirname(path) main = os.path.basename(path) ftp.mkdir(main) for r, d, f in os.walk(path): rr = os.path.relpath(r, root) ftp.chdir(rr) for dd in d: ftp.mkdir(dd) for ff in f: if not os.path.islink(os.path.join(root, rr, ff)): ftp.put(os.path.join(root, rr, ff), ff) for i in range(len(rr.split('/'))): ftp.chdir('..') ftp.chdir('..') print "Preparing server for build..." serverpath = os.path.abspath(os.path.dirname(__file__)) ftp.put(os.path.join(serverpath, 'build.py'), 'build.py') ftp.put(os.path.join(serverpath, 'common.py'), 'common.py') ftp.put(os.path.join(serverpath, '..', 'config.buildserver.py'), 'config.py') # Copy the metadata - just the file for this app... ftp.mkdir('metadata') ftp.mkdir('srclibs') ftp.chdir('metadata') ftp.put(os.path.join('metadata', app['id'] + '.txt'), app['id'] + '.txt') # And patches if there are any... if os.path.exists(os.path.join('metadata', app['id'])): send_dir(os.path.join('metadata', app['id'])) ftp.chdir('/home/vagrant') # Create the build directory... ftp.mkdir('build') ftp.chdir('build') ftp.mkdir('extlib') ftp.mkdir('srclib') # Copy the main app source code if os.path.exists(build_dir): send_dir(build_dir) # Copy any extlibs that are required... if 'extlibs' in thisbuild: ftp.chdir('/home/vagrant/build/extlib') for lib in thisbuild['extlibs'].split(';'): lp = lib.split('/') for d in lp[:-1]: if d not in ftp.listdir(): ftp.mkdir(d) ftp.chdir(d) ftp.put(os.path.join('build/extlib', lib), lp[-1]) for _ in lp[:-1]: ftp.chdir('..') # Copy any srclibs that are required... srclibpaths = [] if 'srclibs' in thisbuild: for lib in thisbuild['srclibs'].split(';'): name, _ = lib.split('@') if options.verbose: print "Processing srclib '" + name + "'" srclibpaths.append((name, common.getsrclib(lib, 'build/srclib', sdk_path, basepath=True, prepare=False))) # If one was used for the main source, add that too. basesrclib = vcs.getsrclib() if basesrclib: srclibpaths.append(basesrclib) for name, lib in srclibpaths: print "Sending srclib '" + lib + "'" ftp.chdir('/home/vagrant/build/srclib') if not os.path.exists(lib): raise BuildException("Missing srclib directory '" + lib + "'") send_dir(lib) # Copy the metadata file too... ftp.chdir('/home/vagrant/srclibs') ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt') # Execute the build script... print "Starting build..." chan = sshs.get_transport().open_session() cmdline = 'python build.py --on-server' if force: cmdline += ' --force --test' cmdline += ' -p ' + app['id'] + ' --vercode ' + thisbuild['vercode'] chan.exec_command(cmdline) output = '' error = '' while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) while chan.recv_stderr_ready(): error += chan.recv_stderr(1024) print "...getting exit status" returncode = chan.recv_exit_status() while chan.recv_ready(): output += chan.recv(1024) while chan.recv_stderr_ready(): error += chan.recv_stderr(1024) if returncode != 0: raise BuildException( "Build.py failed on server for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) # Retrieve the built files... print "Retrieving build output..." if force: ftp.chdir('/home/vagrant/tmp') else: ftp.chdir('/home/vagrant/unsigned') apkfile = app['id'] + '_' + thisbuild['vercode'] + '.apk' tarball = app['id'] + '_' + thisbuild['vercode'] + '_src' + '.tar.gz' try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) ftp.get(tarball, os.path.join(output_dir, tarball)) except: raise BuildException( "Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) ftp.close() finally: # Suspend the build server. print "Suspending build server" subprocess.call(['vagrant', 'suspend'], cwd='builder')
def build_server(app, build, vcs, build_dir, output_dir, force): """Do a build on the build server.""" try: paramiko except NameError: raise BuildException("Paramiko is required to use the buildserver") if options.verbose: logging.getLogger("paramiko").setLevel(logging.INFO) else: logging.getLogger("paramiko").setLevel(logging.WARN) sshinfo = get_clean_vm() try: # Open SSH connection... logging.info("Connecting to virtual machine...") sshs = paramiko.SSHClient() sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) homedir = '/home/' + sshinfo['user'] # Get an SFTP connection... ftp = sshs.open_sftp() ftp.get_channel().settimeout(15) # Put all the necessary files in place... ftp.chdir(homedir) # Helper to copy the contents of a directory to the server... def send_dir(path): root = os.path.dirname(path) main = os.path.basename(path) ftp.mkdir(main) for r, d, f in os.walk(path): rr = os.path.relpath(r, root) ftp.chdir(rr) for dd in d: ftp.mkdir(dd) for ff in f: lfile = os.path.join(root, rr, ff) if not os.path.islink(lfile): ftp.put(lfile, ff) ftp.chmod(ff, os.stat(lfile).st_mode) for i in range(len(rr.split('/'))): ftp.chdir('..') ftp.chdir('..') logging.info("Preparing server for build...") serverpath = os.path.abspath(os.path.dirname(__file__)) ftp.mkdir('fdroidserver') ftp.chdir('fdroidserver') ftp.put(os.path.join(serverpath, '..', 'fdroid'), 'fdroid') ftp.chmod('fdroid', 0o755) send_dir(os.path.join(serverpath)) ftp.chdir(homedir) ftp.put(os.path.join(serverpath, '..', 'buildserver', 'config.buildserver.py'), 'config.py') ftp.chmod('config.py', 0o600) # Copy over the ID (head commit hash) of the fdroidserver in use... subprocess.call('git rev-parse HEAD >' + os.path.join(os.getcwd(), 'tmp', 'fdroidserverid'), shell=True, cwd=serverpath) ftp.put('tmp/fdroidserverid', 'fdroidserverid') # Copy the metadata - just the file for this app... ftp.mkdir('metadata') ftp.mkdir('srclibs') ftp.chdir('metadata') ftp.put(os.path.join('metadata', app.id + '.txt'), app.id + '.txt') # And patches if there are any... if os.path.exists(os.path.join('metadata', app.id)): send_dir(os.path.join('metadata', app.id)) ftp.chdir(homedir) # Create the build directory... ftp.mkdir('build') ftp.chdir('build') ftp.mkdir('extlib') ftp.mkdir('srclib') # Copy any extlibs that are required... if build.extlibs: ftp.chdir(homedir + '/build/extlib') for lib in build.extlibs: lib = lib.strip() libsrc = os.path.join('build/extlib', lib) if not os.path.exists(libsrc): raise BuildException("Missing extlib {0}".format(libsrc)) lp = lib.split('/') for d in lp[:-1]: if d not in ftp.listdir(): ftp.mkdir(d) ftp.chdir(d) ftp.put(libsrc, lp[-1]) for _ in lp[:-1]: ftp.chdir('..') # Copy any srclibs that are required... srclibpaths = [] if build.srclibs: for lib in build.srclibs: srclibpaths.append( common.getsrclib(lib, 'build/srclib', basepath=True, prepare=False)) # If one was used for the main source, add that too. basesrclib = vcs.getsrclib() if basesrclib: srclibpaths.append(basesrclib) for name, number, lib in srclibpaths: logging.info("Sending srclib '%s'" % lib) ftp.chdir(homedir + '/build/srclib') if not os.path.exists(lib): raise BuildException("Missing srclib directory '" + lib + "'") fv = '.fdroidvcs-' + name ftp.put(os.path.join('build/srclib', fv), fv) send_dir(lib) # Copy the metadata file too... ftp.chdir(homedir + '/srclibs') ftp.put(os.path.join('srclibs', name + '.txt'), name + '.txt') # Copy the main app source code # (no need if it's a srclib) if (not basesrclib) and os.path.exists(build_dir): ftp.chdir(homedir + '/build') fv = '.fdroidvcs-' + app.id ftp.put(os.path.join('build', fv), fv) send_dir(build_dir) # Execute the build script... logging.info("Starting build...") chan = sshs.get_transport().open_session() chan.get_pty() cmdline = os.path.join(homedir, 'fdroidserver', 'fdroid') cmdline += ' build --on-server' if force: cmdline += ' --force --test' if options.verbose: cmdline += ' --verbose' cmdline += " %s:%s" % (app.id, build.vercode) chan.exec_command('bash -c ". ~/.bsenv && ' + cmdline + '"') output = '' while not chan.exit_status_ready(): while chan.recv_ready(): output += chan.recv(1024) time.sleep(0.1) logging.info("...getting exit status") returncode = chan.recv_exit_status() while True: get = chan.recv(1024) if len(get) == 0: break output += get if returncode != 0: raise BuildException( "Build.py failed on server for {0}:{1}".format( app.id, build.version), output) # Retrieve the built files... logging.info("Retrieving build output...") if force: ftp.chdir(homedir + '/tmp') else: ftp.chdir(homedir + '/unsigned') apkfile = common.getapkname(app, build) tarball = common.getsrcname(app, build) try: ftp.get(apkfile, os.path.join(output_dir, apkfile)) if not options.notarball: ftp.get(tarball, os.path.join(output_dir, tarball)) except: raise BuildException( "Build failed for %s:%s - missing output files".format( app.id, build.version), output) ftp.close() finally: # Suspend the build server. release_vm()