def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver): """Do a build locally.""" if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: if not config['ndk_path']: logging.critical("$ANDROID_NDK is not set!") sys.exit(3) elif not os.path.isdir(config['sdk_path']): logging.critical( "$ANDROID_NDK points to a non-existing directory!") sys.exit(3) # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones p = None if thisbuild['type'] == 'maven': logging.info("Cleaning Maven project...") cmd = [ config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path'] ] if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1]) maven_dir = os.path.normpath(maven_dir) else: maven_dir = root_dir p = FDroidPopen(cmd, cwd=maven_dir) elif thisbuild['type'] == 'gradle': logging.info("Cleaning Gradle project...") cmd = [config['gradle'], 'clean'] adapt_gradle(build_dir) for name, number, libpath in srclibpaths: adapt_gradle(libpath) p = FDroidPopen(cmd, cwd=root_dir) elif thisbuild['type'] == 'kivy': pass elif thisbuild['type'] == 'ant': logging.info("Cleaning Ant project...") p = FDroidPopen(['ant', 'clean'], cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException( "Error cleaning %s:%s" % (app['id'], thisbuild['version']), p.output) for root, dirs, files in os.walk(build_dir): # Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there if 'gradlew' in files: logging.debug("Getting rid of Gradle wrapper stuff in %s" % root) os.remove(os.path.join(root, 'gradlew')) if 'gradlew.bat' in files: os.remove(os.path.join(root, 'gradlew.bat')) if 'gradle' in dirs: shutil.rmtree(os.path.join(root, 'gradle')) if not options.skipscan: # Scan before building... logging.info("Scanning source for common problems...") count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: if force: logging.warn('Scanner found %d problems:' % count) else: raise BuildException( "Can't build due to %d errors while scanning" % count) if not options.notarball: # Build the source tarball right before we build the release... logging.info("Creating source tarball...") tarname = common.getsrcname(app, thisbuild) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(f): return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() if onserver: manifest = os.path.join(root_dir, 'AndroidManifest.xml') if os.path.exists(manifest): homedir = os.path.expanduser('~') with open(os.path.join(homedir, 'buildserverid'), 'r') as f: buildserverid = f.read() with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f: fdroidserverid = f.read() with open(manifest, 'r') as f: manifestcontent = f.read() manifestcontent = manifestcontent.replace( '</manifest>', '<fdroid buildserverid="' + buildserverid + '"' + ' fdroidserverid="' + fdroidserverid + '"' + '/></manifest>') with open(manifest, 'w') as f: f.write(manifestcontent) # Run a build command if one is required... if thisbuild['build']: logging.info("Running 'build' commands in %s" % root_dir) cmd = common.replace_config_vars(thisbuild['build']) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException( "Error running build command for %s:%s" % (app['id'], thisbuild['version']), p.output) # Build native stuff if required... if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: logging.info("Building the native code") jni_components = thisbuild['buildjni'] if jni_components == ['yes']: jni_components = [''] cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"] for d in jni_components: if d: logging.info("Building native code in '%s'" % d) else: logging.info("Building native code in the main project") manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) if p.returncode != 0: raise BuildException( "NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.output) p = None # Build the release... if thisbuild['type'] == 'maven': logging.info("Building Maven project...") if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@', 1)[1]) else: maven_dir = root_dir mvncmd = [ config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'], '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', '-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package' ] if thisbuild['target']: target = thisbuild["target"].split('-')[1] FDroidPopen([ 'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' + target + '</platform>@g', 'pom.xml' ], cwd=root_dir) if '@' in thisbuild['maven']: FDroidPopen([ 'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' + target + '</platform>@g', 'pom.xml' ], cwd=maven_dir) p = FDroidPopen(mvncmd, cwd=maven_dir) bindir = os.path.join(root_dir, 'target') elif thisbuild['type'] == 'kivy': logging.info("Building Kivy project...") spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException( "Expected to find buildozer-compatible spec at {0}".format( spec)) defaults = { 'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18" } bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) distdir = 'python-for-android/dist/fdroid' if os.path.exists(distdir): shutil.rmtree(distdir) modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] cmd += ' ANDROIDNDK=' + config['ndk_path'] cmd += ' ANDROIDNDKVER=r9' cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -d fdroid' p = FDroidPopen(cmd, cwd='python-for-android', shell=True) if p.returncode != 0: raise BuildException("Distribute build failed") cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get( 'app', 'package.name') if cid != app['id']: raise BuildException( "Package ID mismatch between metadata and spec") orientation = bconfig.get('app', 'orientation', 'landscape') if orientation == 'all': orientation = 'sensor' cmd = [ './build.py' '--dir', root_dir, '--name', bconfig.get('app', 'title'), '--package', app['id'], '--version', bconfig.get('app', 'version'), '--orientation', orientation ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): cmd.extend(['--permission', perm]) if config.get('app', 'fullscreen') == 0: cmd.append('--window') icon = bconfig.get('app', 'icon.filename') if icon: cmd.extend(['--icon', os.path.join(root_dir, icon)]) cmd.append('release') p = FDroidPopen(cmd, cwd=distdir) elif thisbuild['type'] == 'gradle': logging.info("Building Gradle project...") flavours = thisbuild['gradle'].split(',') if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']: flavours[0] = '' commands = [config['gradle']] if thisbuild['preassemble']: commands += thisbuild['preassemble'].split() flavours_cmd = ''.join(flavours) if flavours_cmd: flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:] commands += ['assemble' + flavours_cmd + 'Release'] # Avoid having to use lintOptions.abortOnError false if thisbuild['gradlepluginver'] >= LooseVersion('0.7'): with open(os.path.join(root_dir, 'build.gradle'), "a") as f: f.write( "\nandroid { lintOptions { checkReleaseBuilds false } }\n") p = FDroidPopen(commands, cwd=root_dir) elif thisbuild['type'] == 'ant': logging.info("Building Ant project...") cmd = ['ant'] if thisbuild['antcommand']: cmd += [thisbuild['antcommand']] else: cmd += ['release'] p = FDroidPopen(cmd, cwd=root_dir) bindir = os.path.join(root_dir, 'bin') if p is not None and p.returncode != 0: raise BuildException( "Build failed for %s:%s" % (app['id'], thisbuild['version']), p.output) logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id']) if thisbuild['type'] == 'maven': stdout_apk = '\n'.join([ line for line in p.output.splitlines() if any(a in line for a in ('.apk', '.ap_', '.jar')) ]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M) if not m: m = re.match( r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", stdout_apk, re.S | re.M) if not m: m = re.match( r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M) if not m: m = re.match( r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", stdout_apk, re.S | re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif thisbuild['type'] == 'kivy': src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format( bconfig.get('app', 'title'), bconfig.get('app', 'version')) elif thisbuild['type'] == 'gradle': if thisbuild['gradlepluginver'] >= LooseVersion('0.11'): apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk') else: apks_dir = os.path.join(root_dir, 'build', 'apk') apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk')) if len(apks) > 1: raise BuildException( 'More than one resulting apks found in %s' % apks_dir, '\n'.join(apks)) if len(apks) < 1: raise BuildException('Failed to find gradle output in %s' % apks_dir) src = apks[0] elif thisbuild['type'] == 'ant': stdout_apk = '\n'.join( [line for line in p.output.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M).group(1) src = os.path.join(bindir, src) elif thisbuild['type'] == 'raw': src = os.path.join(root_dir, thisbuild['output']) src = os.path.normpath(src) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = SilentPopen([config['aapt'], 'dump', 'badging', src]) vercode = None version = None foundid = None nativecode = None for line in p.output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") m = pat.match(line) if m: foundid = m.group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") m = pat.match(line) if m: vercode = m.group(1) pat = re.compile(".*versionName='([^']*)'.*") m = pat.match(line) if m: version = m.group(1) elif line.startswith("native-code:"): nativecode = line[12:] # Ignore empty strings or any kind of space/newline chars that we don't # care about if nativecode is not None: nativecode = nativecode.strip() nativecode = None if not nativecode else nativecode if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: if nativecode is None: raise BuildException( "Native code should have been built but none was packaged") if thisbuild['novcheck']: vercode = thisbuild['vercode'] version = thisbuild['version'] if not version or not vercode: raise BuildException( "Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, common.getapkname(app, thisbuild)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir and not options.notarball: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver): """Do a build locally.""" # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose, onserver) # Scan before building... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) if len(buildprobs) > 0: print 'Scanner found ' + str(len(buildprobs)) + ' problems:' for problem in buildprobs: print '...' + problem if not force: raise BuildException("Can't build due to " + str(len(buildprobs)) + " scanned problems") # Build the source tarball right before we build the release... tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' tarball = tarfile.open(os.path.join(tmp_dir, tarname + '.tar.gz'), "w:gz") def tarexc(f): for vcs_dir in ['.svn', '.git', '.hg', '.bzr']: if f.endswith(vcs_dir): return True return False tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Run a build command if one is required... if 'build' in thisbuild: prebuild = thisbuild['build'] # Substitute source library paths into prebuild commands... for name, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) prebuild = prebuild.replace('$$' + name + '$$', libpath) prebuild = prebuild.replace('$$SDK$$', sdk_path) prebuild = prebuild.replace('$$NDK$$', ndk_path) prebuild = prebuild.replace('$$MVN3$$', mvn3) p = subprocess.Popen(prebuild, cwd=root_dir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode != 0: raise BuildException("Error running build command", out, err) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, 'no'): jni_components = thisbuild.get('buildjni') if jni_components == 'yes': jni_components = [''] else: jni_components = jni_components.split(';') ndkbuild = os.path.join(ndk_path, "ndk-build") for d in jni_components: if options.verbose: print "Running ndk-build in " + root_dir + '/' + d manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = subprocess.Popen([ndkbuild], cwd=root_dir + '/' + d, stdout=subprocess.PIPE) output = p.communicate()[0] if p.returncode != 0: print output raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) # Build the release... if 'maven' in thisbuild: mvncmd = [mvn3, 'clean', 'package', '-Dandroid.sdk.path=' + sdk_path] if install: mvncmd += ['-Dandroid.sign.debug=true'] else: mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true'] if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] p = subprocess.Popen(mvncmd, cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: if install: antcommands = ['debug', 'install'] elif 'antcommand' in thisbuild: antcommands = [thisbuild['antcommand']] else: antcommands = ['release'] p = subprocess.Popen(['ant'] + antcommands, cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: raise BuildException( "Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) if verbose: print output if install: if 'maven' in thisbuild: p = subprocess.Popen( [mvn3, 'android:deploy', '-Dandroid.sdk.path=' + sdk_path], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: raise BuildException( "Warning: Could not deploy %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) return print "Build successful" # Find the apk name in the output... if 'bindir' in thisbuild: bindir = os.path.join(build_dir, thisbuild['bindir']) else: bindir = os.path.join(root_dir, 'bin') if thisbuild.get('initfun', 'no') == "yes": # Special case (again!) for funambol... src = ("funambol-android-sync-client-" + thisbuild['version'] + "-unsigned.apk") src = os.path.join(bindir, src) elif 'maven' in thisbuild: m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", output, re.S | re.M) if not m: m = re.match( r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk", output, re.S | re.M) if not m: # This format is found in com.github.mobile, com.yubico.yubitotp and com.botbrew.basil for example... m = re.match( r".*^\[INFO\] [^$]*aapt \[package,[^$]*" + app['id'] + "/" + thisbuild['bindir'] + "/([^/]+)\.ap[_k][,\]]", output, re.S | re.M) if not m: print output raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' else: src = re.match(r".*^.*Creating (.+) for release.*$.*", output, re.S | re.M).group(1) src = os.path.join(bindir, src) # Make sure it's not debuggable... if not install and common.isApkDebuggable(src, sdk_path): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... print "Checking " + src if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = subprocess.Popen([ os.path.join(sdk_path, 'platform-tools', 'aapt'), 'dump', 'badging', src ], stdout=subprocess.PIPE) output = p.communicate()[0] if thisbuild.get('novcheck', 'no') == "yes": vercode = thisbuild['vercode'] version = thisbuild['version'] else: vercode = None version = None foundid = None for line in output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") foundid = re.match(pat, line).group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") vercode = re.match(pat, line).group(1) pat = re.compile(".*versionName='([^']*)'.*") version = re.match(pat, line).group(1) if not version or not vercode: raise BuildException( "Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, app['id'] + '_' + thisbuild['vercode'] + '.apk') shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir: tarfilename = tarname + '.tar.gz' shutil.move(os.path.join(tmp_dir, tarfilename), os.path.join(output_dir, tarfilename))
def main(): global config, options # Parse command line... parser = OptionParser( usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(args, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app['Disabled']: logging.info("Skipping %s: disabled" % appid) continue if not app['builds']: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: build_dir = 'build/' + appid # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) for thisbuild in app['builds']: if thisbuild['disable']: logging.info( "...skipping version %s - %s" % (thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:]))) else: logging.info("...scanning version " + thisbuild['version']) # Prepare the source code... root_dir, _ = common.prepare_source( vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % (count, appid, thisbuild['vercode'])) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % (appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % (appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print "%d app(s) with problems" % probcount
def build_local(app, build, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): """Do a build locally.""" ndk_path = build.ndk_path() if build.buildjni and build.buildjni != ['no']: if not ndk_path: logging.critical("Android NDK version '%s' could not be found!" % build.ndk or 'r10e') logging.critical("Configured versions:") for k, v in config['ndk_paths'].iteritems(): if k.endswith("_orig"): continue logging.critical(" %s: %s" % (k, v)) sys.exit(3) elif not os.path.isdir(ndk_path): logging.critical("Android NDK '%s' is not a directory!" % ndk_path) sys.exit(3) # Set up environment vars that depend on each build for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']: common.env[n] = ndk_path common.reset_env_path() # Set up the current NDK to the PATH common.add_to_env_path(ndk_path) # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver, refresh) # We need to clean via the build tool in case the binary dirs are # different from the default ones p = None gradletasks = [] bmethod = build.build_method() if bmethod == 'maven': logging.info("Cleaning Maven project...") cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] if '@' in build.maven: maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) maven_dir = os.path.normpath(maven_dir) else: maven_dir = root_dir p = FDroidPopen(cmd, cwd=maven_dir) elif bmethod == 'gradle': logging.info("Cleaning Gradle project...") if build.preassemble: gradletasks += build.preassemble flavours = build.gradle if flavours == ['yes']: flavours = [] flavours_cmd = ''.join([capitalize_intact(f) for f in flavours]) gradletasks += ['assemble' + flavours_cmd + 'Release'] adapt_gradle(build_dir) for name, number, libpath in srclibpaths: adapt_gradle(libpath) cmd = [config['gradle']] if build.gradleprops: cmd += ['-P' + kv for kv in build.gradleprops] cmd += ['clean'] p = FDroidPopen(cmd, cwd=root_dir) elif bmethod == 'kivy': pass elif bmethod == 'ant': logging.info("Cleaning Ant project...") p = FDroidPopen(['ant', 'clean'], cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % (app.id, build.version), p.output) for root, dirs, files in os.walk(build_dir): def del_dirs(dl): for d in dl: if d in dirs: shutil.rmtree(os.path.join(root, d)) def del_files(fl): for f in fl: if f in files: os.remove(os.path.join(root, f)) if 'build.gradle' in files: # Even when running clean, gradle stores task/artifact caches in # .gradle/ as binary files. To avoid overcomplicating the scanner, # manually delete them, just like `gradle clean` should have removed # the build/ dirs. del_dirs(['build', '.gradle']) del_files(['gradlew', 'gradlew.bat']) if 'pom.xml' in files: del_dirs(['target']) if any(f in files for f in ['ant.properties', 'project.properties', 'build.xml']): del_dirs(['bin', 'gen']) if 'jni' in dirs: del_dirs(['obj']) if options.skipscan: if build.scandelete: raise BuildException("Refusing to skip source scan since scandelete is present") else: # Scan before building... logging.info("Scanning source for common problems...") count = scanner.scan_source(build_dir, root_dir, build) if count > 0: if force: logging.warn('Scanner found %d problems' % count) else: raise BuildException("Can't build due to %d errors while scanning" % count) if not options.notarball: # Build the source tarball right before we build the release... logging.info("Creating source tarball...") tarname = common.getsrcname(app, build) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(f): return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr']) tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Run a build command if one is required... if build.build: logging.info("Running 'build' commands in %s" % root_dir) cmd = common.replace_config_vars(build.build, build) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % (app.id, build.version), p.output) # Build native stuff if required... if build.buildjni and build.buildjni != ['no']: logging.info("Building the native code") jni_components = build.buildjni if jni_components == ['yes']: jni_components = [''] cmd = [os.path.join(ndk_path, "ndk-build"), "-j1"] for d in jni_components: if d: logging.info("Building native code in '%s'" % d) else: logging.info("Building native code in the main project") manifest = os.path.join(root_dir, d, 'AndroidManifest.xml') if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d)) if p.returncode != 0: raise BuildException("NDK build failed for %s:%s" % (app.id, build.version), p.output) p = None # Build the release... if bmethod == 'maven': logging.info("Building Maven project...") if '@' in build.maven: maven_dir = os.path.join(root_dir, build.maven.split('@', 1)[1]) else: maven_dir = root_dir mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'], '-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true', '-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'] if build.target: target = build.target.split('-')[1] common.regsub_file(r'<platform>[0-9]*</platform>', r'<platform>%s</platform>' % target, os.path.join(root_dir, 'pom.xml')) if '@' in build.maven: common.regsub_file(r'<platform>[0-9]*</platform>', r'<platform>%s</platform>' % target, os.path.join(maven_dir, 'pom.xml')) p = FDroidPopen(mvncmd, cwd=maven_dir) bindir = os.path.join(root_dir, 'target') elif bmethod == 'kivy': logging.info("Building Kivy project...") spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException("Expected to find buildozer-compatible spec at {0}" .format(spec)) defaults = {'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18"} bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) distdir = os.path.join('python-for-android', 'dist', 'fdroid') if os.path.exists(distdir): shutil.rmtree(distdir) modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] cmd += ' ANDROIDNDK=' + ndk_path cmd += ' ANDROIDNDKVER=' + build.ndk cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -d fdroid' p = subprocess.Popen(cmd, cwd='python-for-android', shell=True) if p.returncode != 0: raise BuildException("Distribute build failed") cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name') if cid != app.id: raise BuildException("Package ID mismatch between metadata and spec") orientation = bconfig.get('app', 'orientation', 'landscape') if orientation == 'all': orientation = 'sensor' cmd = ['./build.py' '--dir', root_dir, '--name', bconfig.get('app', 'title'), '--package', app.id, '--version', bconfig.get('app', 'version'), '--orientation', orientation ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): cmd.extend(['--permission', perm]) if config.get('app', 'fullscreen') == 0: cmd.append('--window') icon = bconfig.get('app', 'icon.filename') if icon: cmd.extend(['--icon', os.path.join(root_dir, icon)]) cmd.append('release') p = FDroidPopen(cmd, cwd=distdir) elif bmethod == 'gradle': logging.info("Building Gradle project...") cmd = [config['gradle']] if build.gradleprops: cmd += ['-P' + kv for kv in build.gradleprops] cmd += gradletasks p = FDroidPopen(cmd, cwd=root_dir) elif bmethod == 'ant': logging.info("Building Ant project...") cmd = ['ant'] if build.antcommands: cmd += build.antcommands else: cmd += ['release'] p = FDroidPopen(cmd, cwd=root_dir) bindir = os.path.join(root_dir, 'bin') if p is not None and p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app.id, build.version), p.output) logging.info("Successfully built version " + build.version + ' of ' + app.id) omethod = build.output_method() if omethod == 'maven': stdout_apk = '\n'.join([ line for line in p.output.splitlines() if any( a in line for a in ('.apk', '.ap_', '.jar'))]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S | re.M) if not m: m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", stdout_apk, re.S | re.M) if not m: m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M) if not m: m = re.match(r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar", stdout_apk, re.S | re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif omethod == 'kivy': src = os.path.join('python-for-android', 'dist', 'default', 'bin', '{0}-{1}-release.apk'.format( bconfig.get('app', 'title'), bconfig.get('app', 'version'))) elif omethod == 'gradle': src = None for apks_dir in [ os.path.join(root_dir, 'build', 'outputs', 'apk'), os.path.join(root_dir, 'build', 'apk'), ]: for apkglob in ['*-release-unsigned.apk', '*-unsigned.apk', '*.apk']: apks = glob.glob(os.path.join(apks_dir, apkglob)) if len(apks) > 1: raise BuildException('More than one resulting apks found in %s' % apks_dir, '\n'.join(apks)) if len(apks) == 1: src = apks[0] break if src is not None: break if src is None: raise BuildException('Failed to find any output apks') elif omethod == 'ant': stdout_apk = '\n'.join([ line for line in p.output.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S | re.M).group(1) src = os.path.join(bindir, src) elif omethod == 'raw': globpath = os.path.join(root_dir, build.output) apks = glob.glob(globpath) if len(apks) > 1: raise BuildException('Multiple apks match %s' % globpath, '\n'.join(apks)) if len(apks) < 1: raise BuildException('No apks match %s' % globpath) src = os.path.normpath(apks[0]) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... logging.debug("Checking " + src) if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = SdkToolsPopen(['aapt', 'dump', 'badging', src], output=False) vercode = None version = None foundid = None nativecode = None for line in p.output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") m = pat.match(line) if m: foundid = m.group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") m = pat.match(line) if m: vercode = m.group(1) pat = re.compile(".*versionName='([^']*)'.*") m = pat.match(line) if m: version = m.group(1) elif line.startswith("native-code:"): nativecode = line[12:] # Ignore empty strings or any kind of space/newline chars that we don't # care about if nativecode is not None: nativecode = nativecode.strip() nativecode = None if not nativecode else nativecode if build.buildjni and build.buildjni != ['no']: if nativecode is None: raise BuildException("Native code should have been built but none was packaged") if build.novcheck: vercode = build.vercode version = build.version if not version or not vercode: raise BuildException("Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app.id: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app.id) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != build.version or vercode != build.vercode): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), build.version, str(build.vercode)) ) # Add information for 'fdroid verify' to be able to reproduce the build # environment. if onserver: metadir = os.path.join(tmp_dir, 'META-INF') if not os.path.exists(metadir): os.mkdir(metadir) homedir = os.path.expanduser('~') for fn in ['buildserverid', 'fdroidserverid']: shutil.copyfile(os.path.join(homedir, fn), os.path.join(metadir, fn)) subprocess.call(['jar', 'uf', os.path.abspath(src), 'META-INF/' + fn], cwd=tmp_dir) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, common.getapkname(app, build)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir and not options.notarball: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, verbose, onserver): """Do a build locally.""" # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, verbose, onserver) # Scan before building... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) if len(buildprobs) > 0: print 'Scanner found ' + str(len(buildprobs)) + ' problems:' for problem in buildprobs: print '...' + problem if not force: raise BuildException("Can't build due to " + str(len(buildprobs)) + " scanned problems") # Build the source tarball right before we build the release... tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' tarball = tarfile.open(os.path.join(tmp_dir, tarname + '.tar.gz'), "w:gz") def tarexc(f): for vcs_dir in ['.svn', '.git', '.hg', '.bzr']: if f.endswith(vcs_dir): return True return False tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Run a build command if one is required... if 'build' in thisbuild: prebuild = thisbuild['build'] # Substitute source library paths into prebuild commands... for name, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) prebuild = prebuild.replace('$$' + name + '$$', libpath) prebuild = prebuild.replace('$$SDK$$', sdk_path) prebuild = prebuild.replace('$$NDK$$', ndk_path) prebuild = prebuild.replace('$$MVN3$$', mvn3) p = subprocess.Popen(prebuild, cwd=root_dir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() if p.returncode != 0: raise BuildException("Error running build command", out, err) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, 'no'): jni_components = thisbuild.get('buildjni') if jni_components == 'yes': jni_components = [''] else: jni_components = jni_components.split(';') ndkbuild = os.path.join(ndk_path, "ndk-build") for d in jni_components: if options.verbose: print "Running ndk-build in " + root_dir + '/' + d manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = subprocess.Popen([ndkbuild], cwd=root_dir + '/' + d, stdout=subprocess.PIPE) output = p.communicate()[0] if p.returncode != 0: print output raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version'])) # Build the release... if 'maven' in thisbuild: mvncmd = [mvn3, 'clean', 'package', '-Dandroid.sdk.path=' + sdk_path] if install: mvncmd += ['-Dandroid.sign.debug=true'] else: mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true'] if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] p = subprocess.Popen(mvncmd, cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: if install: antcommands = ['debug','install'] elif 'antcommand' in thisbuild: antcommands = [thisbuild['antcommand']] else: antcommands = ['release'] p = subprocess.Popen(['ant'] + antcommands, cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) if verbose: print output if install: if 'maven' in thisbuild: p = subprocess.Popen([mvn3, 'android:deploy', '-Dandroid.sdk.path=' + sdk_path], cwd=root_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = p.communicate() if p.returncode != 0: raise BuildException("Warning: Could not deploy %s:%s" % (app['id'], thisbuild['version']), output.strip(), error.strip()) return print "Build successful" # Find the apk name in the output... if 'bindir' in thisbuild: bindir = os.path.join(build_dir, thisbuild['bindir']) else: bindir = os.path.join(root_dir, 'bin') if thisbuild.get('initfun', 'no') == "yes": # Special case (again!) for funambol... src = ("funambol-android-sync-client-" + thisbuild['version'] + "-unsigned.apk") src = os.path.join(bindir, src) elif 'maven' in thisbuild: m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", output, re.S|re.M) if not m: m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk", output, re.S|re.M) if not m: # This format is found in com.github.mobile, com.yubico.yubitotp and com.botbrew.basil for example... m = re.match(r".*^\[INFO\] [^$]*aapt \[package,[^$]*" + app['id'] + "/" + thisbuild['bindir'] + "/([^/]+)\.ap[_k][,\]]", output, re.S|re.M) if not m: print output raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' else: src = re.match(r".*^.*Creating (.+) for release.*$.*", output, re.S|re.M).group(1) src = os.path.join(bindir, src) # Make sure it's not debuggable... if not install and common.isApkDebuggable(src, sdk_path): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... print "Checking " + src if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'), 'dump', 'badging', src], stdout=subprocess.PIPE) output = p.communicate()[0] if thisbuild.get('novcheck', 'no') == "yes": vercode = thisbuild['vercode'] version = thisbuild['version'] else: vercode = None version = None foundid = None for line in output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") foundid = re.match(pat, line).group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") vercode = re.match(pat, line).group(1) pat = re.compile(".*versionName='([^']*)'.*") version = re.match(pat, line).group(1) if not version or not vercode: raise BuildException("Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) ) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, app['id'] + '_' + thisbuild['vercode'] + '.apk') shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir: tarfilename = tarname + '.tar.gz' shutil.move(os.path.join(tmp_dir, tarfilename), os.path.join(output_dir, tarfilename))
def main(): global config, options # Parse command line... parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") common.setup_global_opts(parser) parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]") options = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(options.appid, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app.Disabled: logging.info("Skipping %s: disabled" % appid) continue if not app.builds: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: if app.RepoType == 'srclib': build_dir = os.path.join('build', 'srclib', app.Repo) else: build_dir = os.path.join('build', appid) # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app.RepoType, app.Repo, build_dir) for build in app.builds: if build.disable: logging.info("...skipping version %s - %s" % ( build.version, build.get('disable', build.commit[1:]))) else: logging.info("...scanning version " + build.version) # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = scan_source(build_dir, root_dir, build) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % ( count, appid, build.vercode)) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % ( appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % ( appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print("%d problems found" % probcount)
def main(): global config, options # Parse command line... parser = OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-p", "--package", default=None, help="Scan only the specified package") parser.add_option("--nosvn", action="store_true", default=False, help="Skip svn repositories - for test purposes, because they are too slow.") (options, args) = parser.parse_args() config = common.read_config(options) # Get all apps... apps = metadata.read_metadata() # Filter apps according to command-line options if options.package: apps = [app for app in apps if app['id'] == options.package] if len(apps) == 0: print "No such package" sys.exit(1) problems = [] build_dir = 'build' if not os.path.isdir(build_dir): print "Creating build directory" os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for app in apps: skip = False if app['Disabled']: print "Skipping %s: disabled" % app['id'] skip = True elif not app['builds']: print "Skipping %s: no builds specified" % app['id'] skip = True elif options.nosvn and app['Repo Type'] == 'svn': skip = True if not skip: print "Processing " + app['id'] try: build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) for thisbuild in app['builds']: if 'disable' in thisbuild: print ("..skipping version " + thisbuild['version'] + " - " + thisbuild.get('disable', thisbuild['commit'][1:])) else: print "..scanning version " + thisbuild['version'] # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, False) # Do the scan... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) for problem in buildprobs: problems.append(problem + ' in ' + app['id'] + ' ' + thisbuild['version']) except BuildException as be: msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) problems.append(msg) except VCSException as vcse: msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) problems.append(msg) except Exception: msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) problems.append(msg) print "Finished:" for problem in problems: print problem print str(len(problems)) + ' problems.'
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver): """Do a build locally.""" # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, onserver) # We need to clean via the build tool in case the binary dirs are # different from the default ones p = None if thisbuild.get('maven', 'no') != 'no': print "Cleaning Maven project..." cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']] if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) maven_dir = os.path.normpath(maven_dir) else: maven_dir = root_dir p = FDroidPopen(cmd, cwd=maven_dir) elif thisbuild.get('gradle', 'no') != 'no': print "Cleaning Gradle project..." cmd = [config['gradle'], 'clean'] if '@' in thisbuild['gradle']: gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@',1)[1]) gradle_dir = os.path.normpath(gradle_dir) else: gradle_dir = root_dir p = FDroidPopen(cmd, cwd=gradle_dir) elif thisbuild.get('update', '.') != 'no' and thisbuild.get('kivy', 'no') == 'no': print "Cleaning Ant project..." cmd = ['ant', 'clean'] p = FDroidPopen(cmd, cwd=root_dir) if p is not None and p.returncode != 0: raise BuildException("Error cleaning %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) # Also clean jni print "Cleaning jni dirs..." for baddir in [ 'libs/armeabi-v7a', 'libs/armeabi', 'libs/mips', 'libs/x86', 'obj']: badpath = os.path.join(build_dir, baddir) if os.path.exists(badpath): print "Removing '%s'" % badpath shutil.rmtree(badpath) # Scan before building... print "Scanning source for common problems..." buildprobs = common.scan_source(build_dir, root_dir, thisbuild) if len(buildprobs) > 0: print 'Scanner found ' + str(len(buildprobs)) + ' problems:' for problem in buildprobs: print '...' + problem if not force: raise BuildException("Can't build due to " + str(len(buildprobs)) + " scanned problems") # Build the source tarball right before we build the release... print "Creating source tarball..." tarname = common.getsrcname(app,thisbuild) tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz") def tarexc(f): for vcs_dir in ['.svn', '.git', '.hg', '.bzr']: if f.endswith(vcs_dir): return True return False tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Run a build command if one is required... if 'build' in thisbuild: cmd = common.replace_config_vars(thisbuild['build']) # Substitute source library paths into commands... for name, number, libpath in srclibpaths: libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) if options.verbose: print "Running 'build' commands in %s" % root_dir p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running build command for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) # Build native stuff if required... if thisbuild.get('buildjni') not in (None, 'no'): print "Building native libraries..." jni_components = thisbuild.get('buildjni') if jni_components == 'yes': jni_components = [''] else: jni_components = [c.strip() for c in jni_components.split(';')] ndkbuild = os.path.join(config['ndk_path'], "ndk-build") for d in jni_components: if options.verbose: print "Running ndk-build in " + root_dir + '/' + d manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid # the ndk r8c or later 'wordlist' errors. The outcome of this # under gnu/linux is the same as when using tools like # dos2unix, but the native python way is faster and will # work in non-unix systems. manifest_text = open(manifest, 'U').read() open(manifest, 'w').write(manifest_text) # In case the AM.xml read was big, free the memory del manifest_text p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d)) if p.returncode != 0: raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) p = None # Build the release... if thisbuild.get('maven', 'no') != 'no': print "Building Maven project..." if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) else: maven_dir = root_dir mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']] if install: mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy'] else: mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'] if 'target' in thisbuild: target = thisbuild["target"].split('-')[1] subprocess.call(['sed', '-i', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 'pom.xml'], cwd=root_dir) if '@' in thisbuild['maven']: subprocess.call(['sed', '-i', 's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g', 'pom.xml'], cwd=maven_dir) if 'mvnflags' in thisbuild: mvncmd += thisbuild['mvnflags'] p = FDroidPopen(mvncmd, cwd=maven_dir) bindir = os.path.join(root_dir, 'target') elif thisbuild.get('kivy', 'no') != 'no': print "Building Kivy project..." spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): raise BuildException("Expected to find buildozer-compatible spec at {0}" .format(spec)) defaults = {'orientation': 'landscape', 'icon': '', 'permissions': '', 'android.api': "18"} bconfig = ConfigParser(defaults, allow_no_value=True) bconfig.read(spec) distdir = 'python-for-android/dist/fdroid' if os.path.exists(distdir): shutil.rmtree(distdir) modules = bconfig.get('app', 'requirements').split(',') cmd = 'ANDROIDSDK=' + config['sdk_path'] cmd += ' ANDROIDNDK=' + config['ndk_path'] cmd += ' ANDROIDNDKVER=r9' cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api')) cmd += ' VIRTUALENV=virtualenv' cmd += ' ./distribute.sh' cmd += ' -m ' + "'" + ' '.join(modules) + "'" cmd += ' -d fdroid' if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0: raise BuildException("Distribute build failed") cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name') if cid != app['id']: raise BuildException("Package ID mismatch between metadata and spec") orientation = bconfig.get('app', 'orientation', 'landscape') if orientation == 'all': orientation = 'sensor' cmd = ['./build.py' '--dir', root_dir, '--name', bconfig.get('app', 'title'), '--package', app['id'], '--version', bconfig.get('app', 'version'), '--orientation', orientation, ] perms = bconfig.get('app', 'permissions') for perm in perms.split(','): cmd.extend(['--permission', perm]) if config.get('app', 'fullscreen') == 0: cmd.append('--window') icon = bconfig.get('app', 'icon.filename') if icon: cmd.extend(['--icon', os.path.join(root_dir, icon)]) cmd.append('release') p = FDroidPopen(cmd, cwd=distdir) elif thisbuild.get('gradle', 'no') != 'no': print "Building Gradle project..." if '@' in thisbuild['gradle']: flavour = thisbuild['gradle'].split('@')[0] gradle_dir = thisbuild['gradle'].split('@')[1] gradle_dir = os.path.join(root_dir, gradle_dir) else: flavour = thisbuild['gradle'] gradle_dir = root_dir if 'compilesdk' in thisbuild: level = thisbuild["compilesdk"].split('-')[1] subprocess.call(['sed', '-i', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g', 'build.gradle'], cwd=root_dir) if '@' in thisbuild['gradle']: subprocess.call(['sed', '-i', 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g', 'build.gradle'], cwd=gradle_dir) for root, dirs, files in os.walk(build_dir): for f in files: if f == 'build.gradle': adapt_gradle(os.path.join(root, f)) break if flavour in ['main', 'yes', '']: flavour = '' commands = [config['gradle']] if 'preassemble' in thisbuild: for task in thisbuild['preassemble'].split(): commands.append(task) if install: commands += ['assemble'+flavour+'Debug', 'install'+flavour+'Debug'] else: commands += ['assemble'+flavour+'Release'] p = FDroidPopen(commands, cwd=gradle_dir) else: print "Building Ant project..." cmd = ['ant'] if install: cmd += ['debug','install'] elif 'antcommand' in thisbuild: cmd += [thisbuild['antcommand']] else: cmd += ['release'] p = FDroidPopen(cmd, cwd=root_dir) bindir = os.path.join(root_dir, 'bin') if p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr) print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] if install: return # Find the apk name in the output... if 'bindir' in thisbuild: bindir = os.path.join(build_dir, thisbuild['bindir']) if thisbuild.get('maven', 'no') != 'no': stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if any(a in line for a in ('.apk','.ap_'))]) m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk, re.S|re.M) if not m: m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]", stdout_apk, re.S|re.M) if not m: m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S|re.M) if not m: raise BuildException('Failed to find output') src = m.group(1) src = os.path.join(bindir, src) + '.apk' elif thisbuild.get('kivy', 'no') != 'no': src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format( bconfig.get('app', 'title'), bconfig.get('app', 'version')) elif thisbuild.get('gradle', 'no') != 'no': dd = build_dir if 'subdir' in thisbuild: dd = os.path.join(dd, thisbuild['subdir']) if flavour in ['main', 'yes', '']: name = '-'.join([os.path.basename(dd), 'release', 'unsigned']) else: name = '-'.join([os.path.basename(dd), flavour, 'release', 'unsigned']) src = os.path.join(dd, 'build', 'apk', name+'.apk') else: stdout_apk = '\n'.join([ line for line in p.stdout.splitlines() if '.apk' in line]) src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk, re.S|re.M).group(1) src = os.path.join(bindir, src) # Make sure it's not debuggable... if common.isApkDebuggable(src, config): raise BuildException("APK is debuggable") # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... print "Checking " + src if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) p = subprocess.Popen([os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'), 'dump', 'badging', src], stdout=subprocess.PIPE) output = p.communicate()[0] vercode = None version = None foundid = None for line in output.splitlines(): if line.startswith("package:"): pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*") foundid = re.match(pat, line).group(1) pat = re.compile(".*versionCode='([0-9]*)'.*") vercode = re.match(pat, line).group(1) pat = re.compile(".*versionName='([^']*)'.*") version = re.match(pat, line).group(1) if thisbuild['novcheck']: vercode = thisbuild['vercode'] version = thisbuild['version'] if not version or not vercode: raise BuildException("Could not find version information in build in output") if not foundid: raise BuildException("Could not find package ID in output") if foundid != app['id']: raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id']) # Some apps (e.g. Timeriffic) have had the bonkers idea of # including the entire changelog in the version number. Remove # it so we can compare. (TODO: might be better to remove it # before we compile, in fact) index = version.find(" //") if index != -1: version = version[:index] if (version != thisbuild['version'] or vercode != thisbuild['vercode']): raise BuildException(("Unexpected version/version code in output;" " APK: '%s' / '%s', " " Expected: '%s' / '%s'") % (version, str(vercode), thisbuild['version'], str(thisbuild['vercode'])) ) # Copy the unsigned apk to our destination directory for further # processing (by publish.py)... dest = os.path.join(output_dir, common.getapkname(app,thisbuild)) shutil.copyfile(src, dest) # Move the source tarball into the output directory... if output_dir != tmp_dir: shutil.move(os.path.join(tmp_dir, tarname), os.path.join(output_dir, tarname))
def main(): global config, options # Parse command line... parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]") parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors") (options, args) = parser.parse_args() config = common.read_config(options) # Read all app and srclib metadata allapps = metadata.read_metadata() apps = common.read_app_args(args, allapps, True) probcount = 0 build_dir = 'build' if not os.path.isdir(build_dir): logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') for appid, app in apps.iteritems(): if app['Disabled']: logging.info("Skipping %s: disabled" % appid) continue if not app['builds']: logging.info("Skipping %s: no builds specified" % appid) continue logging.info("Processing " + appid) try: build_dir = 'build/' + appid # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) for thisbuild in app['builds']: if thisbuild['disable']: logging.info("...skipping version %s - %s" % ( thisbuild['version'], thisbuild.get('disable', thisbuild['commit'][1:]))) else: logging.info("...scanning version " + thisbuild['version']) # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir, False) # Do the scan... count = common.scan_source(build_dir, root_dir, thisbuild) if count > 0: logging.warn('Scanner found %d problems in %s (%s)' % ( count, appid, thisbuild['vercode'])) probcount += count except BuildException as be: logging.warn("Could not scan app %s due to BuildException: %s" % ( appid, be)) probcount += 1 except VCSException as vcse: logging.warn("VCS error while scanning app %s: %s" % (appid, vcse)) probcount += 1 except Exception: logging.warn("Could not scan app %s due to unknown error: %s" % ( appid, traceback.format_exc())) probcount += 1 logging.info("Finished:") print "%d app(s) with problems" % probcount
def main(): # Read configuration... global build_server_always, mvn3 globals()['build_server_always'] = False globals()['mvn3'] = "mvn3" execfile('config.py', globals()) # Parse command line... parser = OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") parser.add_option("-p", "--package", default=None, help="Scan only the specified package") parser.add_option("--nosvn", action="store_true", default=False, help="Skip svn repositories - for test purposes, because they are too slow.") (options, args) = parser.parse_args() # Get all apps... apps = common.read_metadata(options.verbose) # Filter apps according to command-line options if options.package: apps = [app for app in apps if app['id'] == options.package] if len(apps) == 0: print "No such package" sys.exit(1) html_parser = HTMLParser.HTMLParser() problems = [] extlib_dir = os.path.join('build', 'extlib') for app in apps: skip = False if app['Disabled']: print "Skipping %s: disabled" % app['id'] skip = True elif not app['builds']: print "Skipping %s: no builds specified" % app['id'] skip = True elif options.nosvn and app['Repo Type'] == 'svn': skip = True if not skip: print "Processing " + app['id'] try: build_dir = 'build/' + app['id'] # Set up vcs interface and make sure we have the latest code... vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path) for thisbuild in app['builds']: if thisbuild['commit'].startswith('!'): print ("..skipping version " + thisbuild['version'] + " - " + thisbuild['commit'][1:]) else: print "..scanning version " + thisbuild['version'] # Prepare the source code... root_dir, _ = common.prepare_source(vcs, app, thisbuild, build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, options.verbose) # Do the scan... buildprobs = common.scan_source(build_dir, root_dir, thisbuild) for problem in buildprobs: problems.append(problem + ' in ' + app['id'] + ' ' + thisbuild['version']) except BuildException as be: msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be) problems.append(msg) except VCSException as vcse: msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) problems.append(msg) except Exception: msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) problems.append(msg) print "Finished:" for problem in problems: print problem print str(len(problems)) + ' problems.'