def vagrant(params, cwd=None, printout=False): """Run a vagrant command. :param: list of parameters to pass to vagrant :cwd: directory to run in, or None for current directory :returns: (ret, out) where ret is the return code, and out is the stdout (and stderr) from vagrant """ p = FDroidPopen(['vagrant'] + params, cwd=cwd) return (p.returncode, p.output)
def adapt_gradle(build_dir): for root, dirs, files in os.walk(build_dir): if 'build.gradle' in files: path = os.path.join(root, 'build.gradle') logging.debug("Adapting build.gradle at %s" % path) FDroidPopen([ 'sed', '-i', r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"' + config['build_tools'] + '"@g', path ])
def genkey(keystore, repo_keyalias, password, keydname): '''generate a new keystore with a new key in it for signing repos''' print('Generating a new key in "' + keystore + '"...') p = FDroidPopen(['keytool', '-genkey', '-keystore', keystore, '-alias', repo_keyalias, '-keyalg', 'RSA', '-keysize', '4096', '-sigalg', 'SHA256withRSA', '-validity', '10000', '-storepass', password, '-keypass', password, '-dname', keydname]) if p.returncode != 0: raise BuildException("Failed to generate key", p.stdout, p.stderr) # now show the lovely key that was just generated p = subprocess.Popen(['keytool', '-list', '-v', '-keystore', keystore, '-alias', repo_keyalias], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output = p.communicate(password)[0] print(output.lstrip().strip() + '\n\n')
def devices(): p = FDroidPopen([config['adb'], "devices"]) if p.returncode != 0: raise FDroidException("An error occured when finding devices: %s" % p.output) lines = p.output.splitlines() if lines[0].startswith('* daemon not running'): lines = lines[2:] if len(lines) < 3: return [] lines = lines[1:-1] return [l.split()[0] for l in lines]
def genkey(keystore, repo_keyalias, password, keydname): '''generate a new keystore with a new key in it for signing repos''' logging.info('Generating a new key in "' + keystore + '"...') common.write_password_file("keystorepass", password) common.write_password_file("keypass", password) p = FDroidPopen(['keytool', '-genkey', '-keystore', keystore, '-alias', repo_keyalias, '-keyalg', 'RSA', '-keysize', '4096', '-sigalg', 'SHA256withRSA', '-validity', '10000', '-storepass:file', config['keystorepassfile'], '-keypass:file', config['keypassfile'], '-dname', keydname]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to generate key", p.output) # now show the lovely key that was just generated p = FDroidPopen(['keytool', '-list', '-v', '-keystore', keystore, '-alias', repo_keyalias, '-storepass:file', config['keystorepassfile']]) logging.info(p.output.strip() + '\n\n')
def extract_pubkey(): p = FDroidPopen([ 'keytool', '-exportcert', '-alias', config['repo_keyalias'], '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'] ] + config['smartcardoptions'], output=False) if p.returncode != 0: msg = "Failed to get repo pubkey!" if config['keystore'] == 'NONE': msg += ' Is your crypto smartcard plugged in?' logging.critical(msg) sys.exit(1) global repo_pubkey_fingerprint repo_pubkey_fingerprint = cert_fingerprint(p.output) return "".join("%02x" % ord(b) for b in p.output)
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) repodirs = ['repo'] if config['archive_older'] != 0: repodirs.append('archive') signed = 0 for output_dir in repodirs: if not os.path.isdir(output_dir): logging.error("Missing output directory '" + output_dir + "'") sys.exit(1) unsigned = os.path.join(output_dir, 'index_unsigned.jar') if os.path.exists(unsigned): args = ['jarsigner', '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'], '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA', unsigned, config['repo_keyalias']] if config['keystore'] == 'NONE': args += config['smartcardoptions'] else: # smardcards never use -keypass args += ['-keypass:file', config['keypassfile']] p = FDroidPopen(args) if p.returncode != 0: logging.critical("Failed to sign index") sys.exit(1) os.rename(unsigned, os.path.join(output_dir, 'index.jar')) logging.info('Signed index in ' + output_dir) signed += 1 if signed == 0: logging.info("Nothing to do")
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) repodirs = ['repo'] if config['archive_older'] != 0: repodirs.append('archive') for output_dir in repodirs: if not os.path.isdir(output_dir): logging.error("Missing output directory '" + output_dir + "'") sys.exit(1) # Process any apks that are waiting to be signed... for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))): apkfilename = os.path.basename(apkfile) sigfilename = apkfilename + ".asc" sigpath = os.path.join(output_dir, sigfilename) if not os.path.exists(sigpath): gpgargs = ['gpg', '-a', '--output', sigpath, '--detach-sig'] if 'gpghome' in config: gpgargs.extend(['--homedir', config['gpghome']]) gpgargs.append(os.path.join(output_dir, apkfilename)) p = FDroidPopen(gpgargs) if p.returncode != 0: logging.error("Signing failed.") sys.exit(1) logging.info('Signed ' + apkfilename)
def get_clean_vm(reset=False): """Get a clean VM ready to do a buildserver build. This might involve creating and starting a new virtual machine from scratch, or it might be as simple (unless overridden by the reset parameter) as re-using a snapshot created previously. A BuildException will be raised if anything goes wrong. :reset: True to force creating from scratch. :returns: A dictionary containing 'hostname', 'port', 'user' and 'idfile' """ # Reset existing builder machine to a clean state if possible. vm_ok = False if not reset: logging.info("Checking for valid existing build server") if got_valid_builder_vm(): logging.info("...VM is present") p = FDroidPopen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details' ], cwd='builder') if 'fdroidclean' in p.output: logging.info("...snapshot exists - resetting build server to " "clean state") retcode, output = vagrant(['status'], cwd='builder') if 'running' in output: logging.info("...suspending") vagrant(['suspend'], cwd='builder') logging.info("...waiting a sec...") time.sleep(10) p = FDroidPopen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean' ], cwd='builder') if p.returncode == 0: logging.info("...reset to snapshot - server is valid") retcode, output = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") logging.info("...waiting a sec...") time.sleep(10) sshinfo = get_vagrant_sshinfo() vm_ok = True else: logging.info("...failed to reset to snapshot") else: logging.info("...snapshot doesn't exist - " "VBoxManage snapshot list:\n" + p.output) # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: if os.path.exists('builder'): logging.info("Removing broken/incomplete/unwanted build server") vagrant(['destroy', '-f'], cwd='builder') shutil.rmtree('builder') os.mkdir('builder') p = subprocess.Popen('vagrant --version', shell=True, stdout=subprocess.PIPE) vver = p.communicate()[0] if vver.startswith('Vagrant version 1.2'): with open('builder/Vagrantfile', 'w') as vf: vf.write('Vagrant.configure("2") do |config|\n') vf.write('config.vm.box = "buildserver"\n') vf.write('end\n') else: with open('builder/Vagrantfile', 'w') as vf: vf.write('Vagrant::Config.run do |config|\n') vf.write('config.vm.box = "buildserver"\n') vf.write('end\n') logging.info("Starting new build server") retcode, _ = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") # Open SSH connection to make sure it's working and ready... logging.info("Connecting to virtual machine...") sshinfo = get_vagrant_sshinfo() sshs = paramiko.SSHClient() sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) sshs.close() logging.info("Saving clean state of new build server") retcode, _ = vagrant(['suspend'], cwd='builder') if retcode != 0: raise BuildException("Failed to suspend build server") logging.info("...waiting a sec...") time.sleep(10) p = FDroidPopen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean' ], cwd='builder') if p.returncode != 0: raise BuildException("Failed to take snapshot") logging.info("...waiting a sec...") time.sleep(10) logging.info("Restarting new build server") retcode, _ = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") logging.info("...waiting a sec...") time.sleep(10) # Make sure it worked... p = FDroidPopen([ 'VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details' ], cwd='builder') if 'fdroidclean' not in p.output: raise BuildException("Failed to take snapshot.") return sshinfo
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options] " "[APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) log_dir = 'logs' if not os.path.isdir(log_dir): logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) output_dir = 'repo' if not os.path.isdir(output_dir): logging.info("Creating output directory") os.makedirs(output_dir) unsigned_dir = 'unsigned' if not os.path.isdir(unsigned_dir): logging.warning("No unsigned directory - nothing to do") sys.exit(1) for f in [ config['keystorepassfile'], config['keystore'], config['keypassfile'] ]: if not os.path.exists(f): logging.error("Config error - missing '{0}'".format(f)) sys.exit(1) # It was suggested at # https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit # that a package could be crafted, such that it would use the same signing # key as an existing app. While it may be theoretically possible for such a # colliding package ID to be generated, it seems virtually impossible that # the colliding ID would be something that would be a) a valid package ID, # and b) a sane-looking ID that would make its way into the repo. # Nonetheless, to be sure, before publishing we check that there are no # collisions, and refuse to do any publishing if that's the case... allapps = metadata.read_metadata() vercodes = common.read_pkg_args(args, True) allaliases = [] for appid in allapps: m = md5.new() m.update(appid) keyalias = m.hexdigest()[:8] if keyalias in allaliases: logging.error("There is a keyalias collision - publishing halted") sys.exit(1) allaliases.append(keyalias) logging.info("{0} apps, {0} key aliases".format(len(allapps), len(allaliases))) # Process any apks that are waiting to be signed... for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): appid, vercode = common.apknameinfo(apkfile) apkfilename = os.path.basename(apkfile) if vercodes and appid not in vercodes: continue if appid in vercodes and vercodes[appid]: if vercode not in vercodes[appid]: continue logging.info("Processing " + apkfile) # There ought to be valid metadata for this app, otherwise why are we # trying to publish it? if appid not in allapps: logging.error("Unexpected {0} found in unsigned directory".format( apkfilename)) sys.exit(1) app = allapps[appid] if app.get('Binaries', None): # It's an app where we build from source, and verify the apk # contents against a developer's binary, and then publish their # version if everything checks out. # The binary should already have been retrieved during the build # process. srcapk = apkfile + ".binary" # Compare our unsigned one with the downloaded one... compare_result = common.verify_apks(srcapk, apkfile, tmp_dir) if compare_result: logging.error("...verification failed - publish skipped : " + compare_result) continue # Success! So move the downloaded file to the repo, and remove # our built version. shutil.move(srcapk, os.path.join(output_dir, apkfilename)) os.remove(apkfile) else: # It's a 'normal' app, i.e. we sign and publish it... # Figure out the key alias name we'll use. Only the first 8 # characters are significant, so we'll use the first 8 from # the MD5 of the app's ID and hope there are no collisions. # If a collision does occur later, we're going to have to # come up with a new alogrithm, AND rename all existing keys # in the keystore! if appid in config['keyaliases']: # For this particular app, the key alias is overridden... keyalias = config['keyaliases'][appid] if keyalias.startswith('@'): m = md5.new() m.update(keyalias[1:]) keyalias = m.hexdigest()[:8] else: m = md5.new() m.update(appid) keyalias = m.hexdigest()[:8] logging.info("Key alias: " + keyalias) # See if we already have a key for this application, and # if not generate one... p = FDroidPopen([ 'keytool', '-list', '-alias', keyalias, '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'] ]) if p.returncode != 0: logging.info("Key does not exist - generating...") p = FDroidPopen([ 'keytool', '-genkey', '-keystore', config['keystore'], '-alias', keyalias, '-keyalg', 'RSA', '-keysize', '2048', '-validity', '10000', '-storepass:file', config['keystorepassfile'], '-keypass:file', config['keypassfile'], '-dname', config['keydname'] ]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to generate key") # Sign the application... p = FDroidPopen([ 'jarsigner', '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'], '-keypass:file', config['keypassfile'], '-sigalg', 'MD5withRSA', '-digestalg', 'SHA1', apkfile, keyalias ]) # TODO keypass should be sent via stdin if p.returncode != 0: raise BuildException("Failed to sign application") # Zipalign it... p = SdkToolsPopen([ 'zipalign', '-v', '4', apkfile, os.path.join(output_dir, apkfilename) ]) if p.returncode != 0: raise BuildException("Failed to align application") os.remove(apkfile) # Move the source tarball into the output directory... tarfilename = apkfilename[:-4] + '_src.tar.gz' tarfile = os.path.join(unsigned_dir, tarfilename) if os.path.exists(tarfile): shutil.move(tarfile, os.path.join(output_dir, tarfilename)) logging.info('Published ' + apkfilename)
def make_index(apps, sortedids, apks, repodir, archive, categories): """Make a repo index. :param apps: fully populated apps list :param apks: full populated apks list :param repodir: the repo directory :param archive: True if this is the archive repo, False if it's the main one. :param categories: list of categories """ doc = Document() def addElement(name, value, doc, parent): el = doc.createElement(name) el.appendChild(doc.createTextNode(value)) parent.appendChild(el) def addElementCDATA(name, value, doc, parent): el = doc.createElement(name) el.appendChild(doc.createCDATASection(value)) parent.appendChild(el) root = doc.createElement("fdroid") doc.appendChild(root) repoel = doc.createElement("repo") if archive: repoel.setAttribute("name", config['archive_name']) if config['repo_maxage'] != 0: repoel.setAttribute("maxage", str(config['repo_maxage'])) repoel.setAttribute("icon", os.path.basename(config['archive_icon'])) repoel.setAttribute("url", config['archive_url']) addElement('description', config['archive_description'], doc, repoel) else: repoel.setAttribute("name", config['repo_name']) if config['repo_maxage'] != 0: repoel.setAttribute("maxage", str(config['repo_maxage'])) repoel.setAttribute("icon", os.path.basename(config['repo_icon'])) repoel.setAttribute("url", config['repo_url']) addElement('description', config['repo_description'], doc, repoel) repoel.setAttribute("version", "12") repoel.setAttribute("timestamp", str(int(time.time()))) if 'repo_keyalias' in config: # Generate a certificate fingerprint the same way keytool does it # (but with slightly different formatting) def cert_fingerprint(data): digest = hashlib.sha256(data).digest() ret = [] ret.append(' '.join("%02X" % ord(b) for b in digest)) return " ".join(ret) def extract_pubkey(): p = FDroidPopen([ 'keytool', '-exportcert', '-alias', config['repo_keyalias'], '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'] ] + config['smartcardoptions'], output=False) if p.returncode != 0: msg = "Failed to get repo pubkey!" if config['keystore'] == 'NONE': msg += ' Is your crypto smartcard plugged in?' logging.critical(msg) sys.exit(1) global repo_pubkey_fingerprint repo_pubkey_fingerprint = cert_fingerprint(p.output) return "".join("%02x" % ord(b) for b in p.output) repoel.setAttribute("pubkey", extract_pubkey()) root.appendChild(repoel) for appid in sortedids: app = apps[appid] if app['Disabled'] is not None: continue # Get a list of the apks for this app... apklist = [] for apk in apks: if apk['id'] == appid: apklist.append(apk) if len(apklist) == 0: continue apel = doc.createElement("application") apel.setAttribute("id", app['id']) root.appendChild(apel) addElement('id', app['id'], doc, apel) if 'added' in app: addElement('added', time.strftime('%Y-%m-%d', app['added']), doc, apel) if 'lastupdated' in app: addElement('lastupdated', time.strftime('%Y-%m-%d', app['lastupdated']), doc, apel) addElement('name', app['Name'], doc, apel) addElement('summary', app['Summary'], doc, apel) if app['icon']: addElement('icon', app['icon'], doc, apel) def linkres(appid): if appid in apps: return ("fdroid:app" + appid, apps[appid]['Name']) raise MetaDataException("Cannot resolve app id " + appid) addElement('desc', metadata.description_html(app['Description'], linkres), doc, apel) addElement('license', app['License'], doc, apel) if 'Categories' in app: addElement('categories', ','.join(app["Categories"]), doc, apel) # We put the first (primary) category in LAST, which will have # the desired effect of making clients that only understand one # category see that one. addElement('category', app["Categories"][0], doc, apel) addElement('web', app['Web Site'], doc, apel) addElement('source', app['Source Code'], doc, apel) addElement('tracker', app['Issue Tracker'], doc, apel) if app['Donate']: addElement('donate', app['Donate'], doc, apel) if app['Bitcoin']: addElement('bitcoin', app['Bitcoin'], doc, apel) if app['Litecoin']: addElement('litecoin', app['Litecoin'], doc, apel) if app['Dogecoin']: addElement('dogecoin', app['Dogecoin'], doc, apel) if app['FlattrID']: addElement('flattr', app['FlattrID'], doc, apel) # These elements actually refer to the current version (i.e. which # one is recommended. They are historically mis-named, and need # changing, but stay like this for now to support existing clients. addElement('marketversion', app['Current Version'], doc, apel) addElement('marketvercode', app['Current Version Code'], doc, apel) if app['AntiFeatures']: af = app['AntiFeatures'].split(',') # TODO: Temporarily not including UpstreamNonFree in the index, # because current F-Droid clients do not understand it, and also # look ugly when they encounter an unknown antifeature. This # filtering can be removed in time... if 'UpstreamNonFree' in af: af.remove('UpstreamNonFree') if af: addElement('antifeatures', ','.join(af), doc, apel) if app['Provides']: pv = app['Provides'].split(',') addElement('provides', ','.join(pv), doc, apel) if app['Requires Root']: addElement('requirements', 'root', doc, apel) # Sort the apk list into version order, just so the web site # doesn't have to do any work by default... apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True) # Check for duplicates - they will make the client unhappy... for i in range(len(apklist) - 1): if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']: logging.critical( "duplicate versions: '%s' - '%s'" % (apklist[i]['apkname'], apklist[i + 1]['apkname'])) sys.exit(1) for apk in apklist: apkel = doc.createElement("package") apel.appendChild(apkel) addElement('version', apk['version'], doc, apkel) addElement('versioncode', str(apk['versioncode']), doc, apkel) addElement('apkname', apk['apkname'], doc, apkel) if 'srcname' in apk: addElement('srcname', apk['srcname'], doc, apkel) for hash_type in ['sha256']: if hash_type not in apk: continue hashel = doc.createElement("hash") hashel.setAttribute("type", hash_type) hashel.appendChild(doc.createTextNode(apk[hash_type])) apkel.appendChild(hashel) addElement('sig', apk['sig'], doc, apkel) addElement('size', str(apk['size']), doc, apkel) addElement('sdkver', str(apk['sdkversion']), doc, apkel) if 'maxsdkversion' in apk: addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel) if 'added' in apk: addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel) if app['Requires Root']: if 'ACCESS_SUPERUSER' not in apk['permissions']: apk['permissions'].add('ACCESS_SUPERUSER') if len(apk['permissions']) > 0: addElement('permissions', ','.join(apk['permissions']), doc, apkel) if 'nativecode' in apk and len(apk['nativecode']) > 0: addElement('nativecode', ','.join(apk['nativecode']), doc, apkel) if len(apk['features']) > 0: addElement('features', ','.join(apk['features']), doc, apkel) of = open(os.path.join(repodir, 'index.xml'), 'wb') if options.pretty: output = doc.toprettyxml() else: output = doc.toxml() of.write(output) of.close() if 'repo_keyalias' in config: logging.info("Creating signed index with this key (SHA256):") logging.info("%s" % repo_pubkey_fingerprint) # Create a jar of the index... p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir) if p.returncode != 0: logging.critical("Failed to create jar file") sys.exit(1) # Sign the index... args = [ 'jarsigner', '-keystore', config['keystore'], '-storepass:file', config['keystorepassfile'], '-digestalg', 'SHA1', '-sigalg', 'MD5withRSA', os.path.join(repodir, 'index.jar'), config['repo_keyalias'] ] if config['keystore'] == 'NONE': args += config['smartcardoptions'] else: # smardcards never use -keypass args += ['-keypass:file', config['keypassfile']] p = FDroidPopen(args) # TODO keypass should be sent via stdin if p.returncode != 0: logging.critical("Failed to sign index") sys.exit(1) # Copy the repo icon into the repo directory... icon_dir = os.path.join(repodir, 'icons') iconfilename = os.path.join(icon_dir, os.path.basename(config['repo_icon'])) shutil.copyfile(config['repo_icon'], iconfilename) # Write a category list in the repo to allow quick access... catdata = '' for cat in categories: catdata += cat + '\n' f = open(os.path.join(repodir, 'categories.txt'), 'w') f.write(catdata) f.close()
def main(): global options, config # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): logging.info("Creating temporary directory") os.makedirs(tmp_dir) unsigned_dir = 'unsigned' if not os.path.isdir(unsigned_dir): logging.error("No unsigned directory - nothing to do") sys.exit(0) verified = 0 notverified = 0 vercodes = common.read_pkg_args(args, True) for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))): apkfilename = os.path.basename(apkfile) appid, vercode = common.apknameinfo(apkfile) if vercodes and appid not in vercodes: continue if vercodes[appid] and vercode not in vercodes[appid]: continue try: logging.info("Processing " + apkfilename) remoteapk = os.path.join(tmp_dir, apkfilename) if os.path.exists(remoteapk): os.remove(remoteapk) url = 'https://f-droid.org/repo/' + apkfilename logging.info("...retrieving " + url) p = FDroidPopen(['wget', url], cwd=tmp_dir) if p.returncode != 0: raise FDroidException("Failed to get " + apkfilename) thisdir = os.path.join(tmp_dir, 'this_apk') thatdir = os.path.join(tmp_dir, 'that_apk') for d in [thisdir, thatdir]: if os.path.exists(d): shutil.rmtree(d) os.mkdir(d) if subprocess.call(['jar', 'xf', os.path.join("..", "..", unsigned_dir, apkfilename)], cwd=thisdir) != 0: raise FDroidException("Failed to unpack local build of " + apkfilename) if subprocess.call(['jar', 'xf', os.path.join("..", "..", remoteapk)], cwd=thatdir) != 0: raise FDroidException("Failed to unpack remote build of " + apkfilename) p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir) lines = p.output.splitlines() if len(lines) != 1 or 'META-INF' not in lines[0]: raise FDroidException("Unexpected diff output - " + p.output) logging.info("...successfully verified") verified += 1 except FDroidException, e: logging.info("...NOT verified - {0}".format(e)) notverified += 1
def get_clean_vm(reset=False): """Get a clean VM ready to do a buildserver build. This might involve creating and starting a new virtual machine from scratch, or it might be as simple (unless overridden by the reset parameter) as re-using a snapshot created previously. A BuildException will be raised if anything goes wrong. :reset: True to force creating from scratch. :returns: A dictionary containing 'hostname', 'port', 'user' and 'idfile' """ # Reset existing builder machine to a clean state if possible. vm_ok = False if not reset: logging.info("Checking for valid existing build server") if got_valid_builder_vm(): logging.info("...VM is present") p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'], cwd='builder') if 'fdroidclean' in p.output: logging.info("...snapshot exists - resetting build server to " "clean state") retcode, output = vagrant(['status'], cwd='builder') if 'running' in output: logging.info("...suspending") vagrant(['suspend'], cwd='builder') logging.info("...waiting a sec...") time.sleep(10) p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'restore', 'fdroidclean'], cwd='builder') if p.returncode == 0: logging.info("...reset to snapshot - server is valid") retcode, output = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") logging.info("...waiting a sec...") time.sleep(10) sshinfo = get_vagrant_sshinfo() vm_ok = True else: logging.info("...failed to reset to snapshot") else: logging.info("...snapshot doesn't exist - " "VBoxManage snapshot list:\n" + p.output) # If we can't use the existing machine for any reason, make a # new one from scratch. if not vm_ok: if os.path.exists('builder'): logging.info("Removing broken/incomplete/unwanted build server") vagrant(['destroy', '-f'], cwd='builder') shutil.rmtree('builder') os.mkdir('builder') p = subprocess.Popen(['vagrant', '--version'], stdout=subprocess.PIPE) vver = p.communicate()[0].strip().split(' ')[1] if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4: raise BuildException("Unsupported vagrant version {0}".format(vver)) with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf: vf.write('Vagrant.configure("2") do |config|\n') vf.write('config.vm.box = "buildserver"\n') vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n') vf.write('end\n') logging.info("Starting new build server") retcode, _ = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") # Open SSH connection to make sure it's working and ready... logging.info("Connecting to virtual machine...") sshinfo = get_vagrant_sshinfo() sshs = paramiko.SSHClient() sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy()) sshs.connect(sshinfo['hostname'], username=sshinfo['user'], port=sshinfo['port'], timeout=300, look_for_keys=False, key_filename=sshinfo['idfile']) sshs.close() logging.info("Saving clean state of new build server") retcode, _ = vagrant(['suspend'], cwd='builder') if retcode != 0: raise BuildException("Failed to suspend build server") logging.info("...waiting a sec...") time.sleep(10) p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'take', 'fdroidclean'], cwd='builder') if p.returncode != 0: raise BuildException("Failed to take snapshot") logging.info("...waiting a sec...") time.sleep(10) logging.info("Restarting new build server") retcode, _ = vagrant(['up'], cwd='builder') if retcode != 0: raise BuildException("Failed to start build server") logging.info("...waiting a sec...") time.sleep(10) # Make sure it worked... p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(), 'list', '--details'], cwd='builder') if 'fdroidclean' not in p.output: raise BuildException("Failed to take snapshot.") return sshinfo
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver): """Do a build locally.""" # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones p = None if thisbuild.get('maven', 'no') != 'no': print "Cleaning Maven project..." cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) maven_dir = os.path.normpath(maven_dir) else: maven_dir = root_dir p = FDroidPopen(cmd, cwd=maven_dir) elif thisbuild.get('gradle', 'no') != 'no': print "Cleaning Gradle project..." cmd = [config['gradle'], 'clean'] if '@' in thisbuild['gradle']: gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@',1)[1]) gradle_dir = os.path.normpath(gradle_dir) else: gradle_dir = root_dir p = FDroidPopen(cmd, cwd=gradle_dir) elif thisbuild.get('update', '.') != 'no' and thisbuild.get('kivy', 'no') == 'no': print "Cleaning Ant project..." cmd = ['ant', 'clean'] p = FDroidPopen(cmd, cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) # Also clean jni print "Cleaning jni dirs..." for baddir in [ 'libs/armeabi-v7a', 'libs/armeabi', 'libs/mips', 'libs/x86', 'obj']: badpath = os.path.join(build_dir, baddir) if os.path.exists(badpath): print "Removing '%s'" % badpath shutil.rmtree(badpath) # Scan before building... print "Scanning source for common problems..." buildprobs = common.scan_source(build_dir, root_dir, thisbuild) if len(buildprobs) > 0: print 'Scanner found ' + str(len(buildprobs)) + ' problems:' for problem in buildprobs: print '...' + problem if not force: raise BuildException("Can't build due to " + str(len(buildprobs)) + " scanned problems") # Build the source tarball right before we build the release... print "Creating source tarball..." tarname = common.getsrcname(app,thisbuild) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(f): for vcs_dir in ['.svn', '.git', '.hg', '.bzr']: if f.endswith(vcs_dir): return True return False tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Run a build command if one is required... if 'build' in thisbuild: cmd = common.replace_config_vars(thisbuild['build']) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) if options.verbose: print "Running 'build' commands in %s" % root_dir p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, 'no'): print "Building native libraries..." jni_components = thisbuild.get('buildjni') if jni_components == 'yes': jni_components = [''] else: jni_components = [c.strip() for c in jni_components.split(';')] ndkbuild = os.path.join(config['ndk_path'], "ndk-build") for d in jni_components: if options.verbose: print "Running ndk-build in " + root_dir + '/' + d manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d)) if p.returncode != 0: raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) p = None # Build the release... if thisbuild.get('maven', 'no') != 'no': print "Building Maven project..." if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) else: maven_dir = root_dir mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']] if install: mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy'] else: mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'] if 'target' in thisbuild: target = thisbuild["target"].split('-')[1] subprocess.call(['sed', '-i', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 'pom.xml'], cwd=root_dir) if '@' in thisbuild['maven']: subprocess.call(['sed', '-i', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 'pom.xml'], cwd=maven_dir) if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] p = FDroidPopen(mvncmd, cwd=maven_dir) bindir = os.path.join(root_dir, 'target') elif thisbuild.get('kivy', 'no') != 'no': print "Building Kivy project..." spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException("Expected to find buildozer-compatible spec at {0}" .format(spec)) defaults = {'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18"} bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) distdir = 'python-for-android/dist/fdroid' if os.path.exists(distdir): shutil.rmtree(distdir) modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] cmd += ' ANDROIDNDK=' + config['ndk_path'] cmd += ' ANDROIDNDKVER=r9' cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -d fdroid' if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0: raise BuildException("Distribute build failed") cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name') if cid != app['id']: raise BuildException("Package ID mismatch between metadata and spec") orientation = bconfig.get('app', 'orientation', 'landscape') if orientation == 'all': orientation = 'sensor' cmd = ['./build.py' '--dir', root_dir, '--name', bconfig.get('app', 'title'), '--package', app['id'], '--version', bconfig.get('app', 'version'), '--orientation', orientation, ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): cmd.extend(['--permission', perm]) if config.get('app', 'fullscreen') == 0: cmd.append('--window') icon = bconfig.get('app', 'icon.filename') if icon: cmd.extend(['--icon', os.path.join(root_dir, icon)]) cmd.append('release') p = FDroidPopen(cmd, cwd=distdir) elif thisbuild.get('gradle', 'no') != 'no': print "Building Gradle project..." if '@' in thisbuild['gradle']: flavour = thisbuild['gradle'].split('@')[0] gradle_dir = thisbuild['gradle'].split('@')[1] gradle_dir = os.path.join(root_dir, gradle_dir) else: flavour = thisbuild['gradle'] gradle_dir = root_dir if 'compilesdk' in thisbuild: level = thisbuild["compilesdk"].split('-')[1] subprocess.call(['sed', '-i', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g', 'build.gradle'], cwd=root_dir) if '@' in thisbuild['gradle']: subprocess.call(['sed', '-i', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g', 'build.gradle'], cwd=gradle_dir) for root, dirs, files in os.walk(build_dir): for f in files: if f == 'build.gradle': adapt_gradle(os.path.join(root, f)) break if flavour in ['main', 'yes', '']: flavour = '' commands = [config['gradle']] if 'preassemble' in thisbuild: for task in thisbuild['preassemble'].split(): commands.append(task) if install: commands += ['assemble'+flavour+'Debug', 'install'+flavour+'Debug'] else: commands += ['assemble'+flavour+'Release'] p = FDroidPopen(commands, cwd=gradle_dir) else: print "Building Ant project..." cmd = ['ant'] if install: cmd += ['debug','install'] elif 'antcommand' in thisbuild: cmd += [thisbuild['antcommand']] else: cmd += ['release'] p = FDroidPopen(cmd, cwd=root_dir) bindir = os.path.join(root_dir, 'bin') if p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] if install: return # Find the apk name in the output... if 'bindir' in thisbuild: bindir = os.path.join(build_dir, thisbuild['bindir']) if thisbuild.get('maven', 'no') != 'no': stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if any(a in line for a in ('.apk','.ap_'))]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S|re.M) if not m: m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", stdout_apk, re.S|re.M) if not m: m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S|re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif thisbuild.get('kivy', 'no') != 'no': src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format( bconfig.get('app', 'title'), bconfig.get('app', 'version')) elif thisbuild.get('gradle', 'no') != 'no': dd = build_dir if 'subdir' in thisbuild: dd = os.path.join(dd, thisbuild['subdir']) if flavour in ['main', 'yes', '']: name = '-'.join([os.path.basename(dd), 'release', 'unsigned']) else: name = '-'.join([os.path.basename(dd), flavour, 'release', 'unsigned']) src = os.path.join(dd, 'build', 'apk', name+'.apk') else: stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S|re.M).group(1) src = os.path.join(bindir, src) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... print "Checking " + src if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'), 'dump', 'badging', src], stdout=subprocess.PIPE) output = p.communicate()[0] vercode = None version = None foundid = None for line in output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") foundid = re.match(pat, line).group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") vercode = re.match(pat, line).group(1) pat = re.compile(".*versionName='([^']*)'.*") version = re.match(pat, line).group(1) if thisbuild['novcheck']: vercode = thisbuild['vercode'] version = thisbuild['version'] if not version or not vercode: raise BuildException("Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) ) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, common.getapkname(app,thisbuild)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver): """Do a build locally.""" if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: if not config['ndk_path']: logging.critical("$ANDROID_NDK is not set!") sys.exit(3) elif not os.path.isdir(config['sdk_path']): logging.critical( "$ANDROID_NDK points to a non-existing directory!") sys.exit(3) # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones p = None if thisbuild['type'] == 'maven': logging.info("Cleaning Maven project...") cmd = [ config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path'] ] if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1]) maven_dir = os.path.normpath(maven_dir) else: maven_dir = root_dir p = FDroidPopen(cmd, cwd=maven_dir) elif thisbuild['type'] == 'gradle': logging.info("Cleaning Gradle project...") cmd = [config['gradle'], 'clean'] adapt_gradle(build_dir) for name, number, libpath in srclibpaths: adapt_gradle(libpath) p = FDroidPopen(cmd, cwd=root_dir) elif thisbuild['type'] == 'kivy': pass elif thisbuild['type'] == 'ant': logging.info("Cleaning Ant project...") p = FDroidPopen(['ant', 'clean'], cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException( "Error cleaning %s:%s" % (app['id'], thisbuild['version']), p.output) for root, dirs, files in os.walk(build_dir): # Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there if 'gradlew' in files: logging.debug("Getting rid of Gradle wrapper stuff in %s" % root) os.remove(os.path.join(root, 'gradlew')) if 'gradlew.bat' in files: os.remove(os.path.join(root, 'gradlew.bat')) if 'gradle' in dirs: shutil.rmtree(os.path.join(root, 'gradle')) if not options.skipscan: # Scan before building... logging.info("Scanning source for common problems...") count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: if force: logging.warn('Scanner found %d problems:' % count) else: raise BuildException( "Can't build due to %d errors while scanning" % count) if not options.notarball: # Build the source tarball right before we build the release... logging.info("Creating source tarball...") tarname = common.getsrcname(app, thisbuild) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(f): return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() if onserver: manifest = os.path.join(root_dir, 'AndroidManifest.xml') if os.path.exists(manifest): homedir = os.path.expanduser('~') with open(os.path.join(homedir, 'buildserverid'), 'r') as f: buildserverid = f.read() with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f: fdroidserverid = f.read() with open(manifest, 'r') as f: manifestcontent = f.read() manifestcontent = manifestcontent.replace( '</manifest>', '<fdroid buildserverid="' + buildserverid + '"' + ' fdroidserverid="' + fdroidserverid + '"' + '/></manifest>') with open(manifest, 'w') as f: f.write(manifestcontent) # Run a build command if one is required... if thisbuild['build']: logging.info("Running 'build' commands in %s" % root_dir) cmd = common.replace_config_vars(thisbuild['build']) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException( "Error running build command for %s:%s" % (app['id'], thisbuild['version']), p.output) # Build native stuff if required... if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: logging.info("Building the native code") jni_components = thisbuild['buildjni'] if jni_components == ['yes']: jni_components = [''] cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"] for d in jni_components: if d: logging.info("Building native code in '%s'" % d) else: logging.info("Building native code in the main project") manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) if p.returncode != 0: raise BuildException( "NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.output) p = None # Build the release... if thisbuild['type'] == 'maven': logging.info("Building Maven project...") if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1]) else: maven_dir = root_dir mvncmd = [ config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'], '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', '-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package' ] if thisbuild['target']: target = thisbuild["target"].split('-')[1] FDroidPopen([ 'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' + target + '</platform>@g', 'pom.xml' ], cwd=root_dir) if '@' in thisbuild['maven']: FDroidPopen([ 'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' + target + '</platform>@g', 'pom.xml' ], cwd=maven_dir) p = FDroidPopen(mvncmd, cwd=maven_dir) bindir = os.path.join(root_dir, 'target') elif thisbuild['type'] == 'kivy': logging.info("Building Kivy project...") spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException( "Expected to find buildozer-compatible spec at {0}".format( spec)) defaults = { 'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18" } bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) distdir = 'python-for-android/dist/fdroid' if os.path.exists(distdir): shutil.rmtree(distdir) modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] cmd += ' ANDROIDNDK=' + config['ndk_path'] cmd += ' ANDROIDNDKVER=r9' cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -d fdroid' p = FDroidPopen(cmd, cwd='python-for-android', shell=True) if p.returncode != 0: raise BuildException("Distribute build failed") cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get( 'app', 'package.name') if cid != app['id']: raise BuildException( "Package ID mismatch between metadata and spec") orientation = bconfig.get('app', 'orientation', 'landscape') if orientation == 'all': orientation = 'sensor' cmd = [ './build.py' '--dir', root_dir, '--name', bconfig.get('app', 'title'), '--package', app['id'], '--version', bconfig.get('app', 'version'), '--orientation', orientation ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): cmd.extend(['--permission', perm]) if config.get('app', 'fullscreen') == 0: cmd.append('--window') icon = bconfig.get('app', 'icon.filename') if icon: cmd.extend(['--icon', os.path.join(root_dir, icon)]) cmd.append('release') p = FDroidPopen(cmd, cwd=distdir) elif thisbuild['type'] == 'gradle': logging.info("Building Gradle project...") flavours = thisbuild['gradle'].split(',') if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']: flavours[0] = '' commands = [config['gradle']] if thisbuild['preassemble']: commands += thisbuild['preassemble'].split() flavours_cmd = ''.join(flavours) if flavours_cmd: flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:] commands += ['assemble' + flavours_cmd + 'Release'] # Avoid having to use lintOptions.abortOnError false if thisbuild['gradlepluginver'] >= LooseVersion('0.7'): with open(os.path.join(root_dir, 'build.gradle'), "a") as f: f.write( "\nandroid { lintOptions { checkReleaseBuilds false } }\n") p = FDroidPopen(commands, cwd=root_dir) elif thisbuild['type'] == 'ant': logging.info("Building Ant project...") cmd = ['ant'] if thisbuild['antcommand']: cmd += [thisbuild['antcommand']] else: cmd += ['release'] p = FDroidPopen(cmd, cwd=root_dir) bindir = os.path.join(root_dir, 'bin') if p is not None and p.returncode != 0: raise BuildException( "Build failed for %s:%s" % (app['id'], thisbuild['version']), p.output) logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id']) if thisbuild['type'] == 'maven': stdout_apk = '\n'.join([ line for line in p.output.splitlines() if any(a in line for a in ('.apk', '.ap_', '.jar')) ]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M) if not m: m = re.match( r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", stdout_apk, re.S | re.M) if not m: m = re.match( r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M) if not m: m = re.match( r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", stdout_apk, re.S | re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif thisbuild['type'] == 'kivy': src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format( bconfig.get('app', 'title'), bconfig.get('app', 'version')) elif thisbuild['type'] == 'gradle': if thisbuild['gradlepluginver'] >= LooseVersion('0.11'): apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk') else: apks_dir = os.path.join(root_dir, 'build', 'apk') apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk')) if len(apks) > 1: raise BuildException( 'More than one resulting apks found in %s' % apks_dir, '\n'.join(apks)) if len(apks) < 1: raise BuildException('Failed to find gradle output in %s' % apks_dir) src = apks[0] elif thisbuild['type'] == 'ant': stdout_apk = '\n'.join( [line for line in p.output.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M).group(1) src = os.path.join(bindir, src) elif thisbuild['type'] == 'raw': src = os.path.join(root_dir, thisbuild['output']) src = os.path.normpath(src) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = SilentPopen([config['aapt'], 'dump', 'badging', src]) vercode = None version = None foundid = None nativecode = None for line in p.output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") m = pat.match(line) if m: foundid = m.group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") m = pat.match(line) if m: vercode = m.group(1) pat = re.compile(".*versionName='([^']*)'.*") m = pat.match(line) if m: version = m.group(1) elif line.startswith("native-code:"): nativecode = line[12:] # Ignore empty strings or any kind of space/newline chars that we don't # care about if nativecode is not None: nativecode = nativecode.strip() nativecode = None if not nativecode else nativecode if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: if nativecode is None: raise BuildException( "Native code should have been built but none was packaged") if thisbuild['novcheck']: vercode = thisbuild['vercode'] version = thisbuild['version'] if not version or not vercode: raise BuildException( "Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, common.getapkname(app, thisbuild)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir and not options.notarball: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
def main(): global options, config # Parse command line... parser = OptionParser( usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") parser.add_option("-a", "--all", action="store_true", default=False, help="Install all signed applications available") (options, args) = parser.parse_args() if not args and not options.all: raise OptionError( "If you really want to install all the signed apps, use --all", "all") config = common.read_config(options) output_dir = 'repo' if not os.path.isdir(output_dir): logging.info("No signed output directory - nothing to do") sys.exit(0) if args: vercodes = common.read_pkg_args(args, True) apks = {appid: None for appid in vercodes} # Get the signed apk with the highest vercode for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))): try: appid, vercode = common.apknameinfo(apkfile) except FDroidException: continue if appid not in apks: continue if vercodes[appid] and vercode not in vercodes[appid]: continue apks[appid] = apkfile for appid, apk in apks.iteritems(): if not apk: raise FDroidException("No signed apk available for %s" % appid) else: apks = { common.apknameinfo(apkfile)[0]: apkfile for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))) } for appid, apk in apks.iteritems(): # Get device list each time to avoid device not found errors devs = devices() if not devs: raise FDroidException("No attached devices found") logging.info("Installing %s..." % apk) for dev in devs: logging.info("Installing %s on %s..." % (apk, dev)) p = FDroidPopen([config['adb'], "-s", dev, "install", apk]) fail = "" for line in p.output.splitlines(): if line.startswith("Failure"): fail = line[9:-1] if not fail: continue if fail == "INSTALL_FAILED_ALREADY_EXISTS": logging.warn("%s is already installed on %s." % (apk, dev)) else: raise FDroidException("Failed to install %s on %s: %s" % (apk, dev, fail)) logging.info("\nFinished")
t = f.read(1024) if len(t) == 0: break sha.update(t) thisinfo['sha256'] = sha.hexdigest() # Get the signature (or md5 of, to be precise)... getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig') if not os.path.exists(getsig_dir + "/getsig.class"): logging.critical( "getsig.class not found. To fix: cd '%s' && ./make.sh" % getsig_dir) sys.exit(1) p = FDroidPopen([ 'java', '-cp', os.path.join(os.path.dirname(__file__), 'getsig'), 'getsig', os.path.join(os.getcwd(), apkfile) ]) thisinfo['sig'] = None for line in p.output.splitlines(): if line.startswith('Result:'): thisinfo['sig'] = line[7:].strip() break if p.returncode != 0 or not thisinfo['sig']: logging.critical("Failed to get apk signature") sys.exit(1) apk = zipfile.ZipFile(apkfile, 'r') iconfilename = "%s.%s.png" % (thisinfo['id'], thisinfo['versioncode'])