def vagrant(params, cwd=None, printout=False):
"""Run a vagrant command.
:param: list of parameters to pass to vagrant
:cwd: directory to run in, or None for current directory
:returns: (ret, out) where ret is the return code, and out
is the stdout (and stderr) from vagrant
"""
p = FDroidPopen(['vagrant'] + params, cwd=cwd)
return (p.returncode, p.output)
def adapt_gradle(build_dir):
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
logging.debug("Adapting build.gradle at %s" % path)
FDroidPopen([
'sed', '-i',
r's@buildToolsVersion\([ =]*\)["\'][0-9\.]*["\']@buildToolsVersion\1"'
+ config['build_tools'] + '"@g', path
])
def genkey(keystore, repo_keyalias, password, keydname):
'''generate a new keystore with a new key in it for signing repos'''
print('Generating a new key in "' + keystore + '"...')
p = FDroidPopen(['keytool', '-genkey',
'-keystore', keystore, '-alias', repo_keyalias,
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
'-storepass', password, '-keypass', password,
'-dname', keydname])
if p.returncode != 0:
raise BuildException("Failed to generate key", p.stdout, p.stderr)
# now show the lovely key that was just generated
p = subprocess.Popen(['keytool', '-list', '-v',
'-keystore', keystore, '-alias', repo_keyalias],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = p.communicate(password)[0]
print(output.lstrip().strip() + '\n\n')
def devices():
p = FDroidPopen([config['adb'], "devices"])
if p.returncode != 0:
raise FDroidException("An error occured when finding devices: %s" %
p.output)
lines = p.output.splitlines()
if lines[0].startswith('* daemon not running'):
lines = lines[2:]
if len(lines) < 3:
return []
lines = lines[1:-1]
return [l.split()[0] for l in lines]
def genkey(keystore, repo_keyalias, password, keydname):
'''generate a new keystore with a new key in it for signing repos'''
logging.info('Generating a new key in "' + keystore + '"...')
common.write_password_file("keystorepass", password)
common.write_password_file("keypass", password)
p = FDroidPopen(['keytool', '-genkey',
'-keystore', keystore, '-alias', repo_keyalias,
'-keyalg', 'RSA', '-keysize', '4096',
'-sigalg', 'SHA256withRSA',
'-validity', '10000',
'-storepass:file', config['keystorepassfile'],
'-keypass:file', config['keypassfile'],
'-dname', keydname])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
# now show the lovely key that was just generated
p = FDroidPopen(['keytool', '-list', '-v',
'-keystore', keystore, '-alias', repo_keyalias,
'-storepass:file', config['keystorepassfile']])
logging.info(p.output.strip() + '\n\n')
def extract_pubkey():
p = FDroidPopen([
'keytool', '-exportcert', '-alias', config['repo_keyalias'],
'-keystore', config['keystore'], '-storepass:file',
config['keystorepassfile']
] + config['smartcardoptions'],
output=False)
if p.returncode != 0:
msg = "Failed to get repo pubkey!"
if config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
logging.critical(msg)
sys.exit(1)
global repo_pubkey_fingerprint
repo_pubkey_fingerprint = cert_fingerprint(p.output)
return "".join("%02x" % ord(b) for b in p.output)
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
signed = 0
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
unsigned = os.path.join(output_dir, 'index_unsigned.jar')
if os.path.exists(unsigned):
args = ['jarsigner', '-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile'],
'-digestalg', 'SHA1', '-sigalg', 'MD5withRSA',
unsigned, config['repo_keyalias']]
if config['keystore'] == 'NONE':
args += config['smartcardoptions']
else: # smardcards never use -keypass
args += ['-keypass:file', config['keypassfile']]
p = FDroidPopen(args)
if p.returncode != 0:
logging.critical("Failed to sign index")
sys.exit(1)
os.rename(unsigned, os.path.join(output_dir, 'index.jar'))
logging.info('Signed index in ' + output_dir)
signed += 1
if signed == 0:
logging.info("Nothing to do")
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
sigfilename = apkfilename + ".asc"
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
gpgargs = ['gpg', '-a',
'--output', sigpath,
'--detach-sig']
if 'gpghome' in config:
gpgargs.extend(['--homedir', config['gpghome']])
gpgargs.append(os.path.join(output_dir, apkfilename))
p = FDroidPopen(gpgargs)
if p.returncode != 0:
logging.error("Signing failed.")
sys.exit(1)
logging.info('Signed ' + apkfilename)
def get_clean_vm(reset=False):
"""Get a clean VM ready to do a buildserver build.
This might involve creating and starting a new virtual machine from
scratch, or it might be as simple (unless overridden by the reset
parameter) as re-using a snapshot created previously.
A BuildException will be raised if anything goes wrong.
:reset: True to force creating from scratch.
:returns: A dictionary containing 'hostname', 'port', 'user'
and 'idfile'
"""
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not reset:
logging.info("Checking for valid existing build server")
if got_valid_builder_vm():
logging.info("...VM is present")
p = FDroidPopen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder')
if 'fdroidclean' in p.output:
logging.info("...snapshot exists - resetting build server to "
"clean state")
retcode, output = vagrant(['status'], cwd='builder')
if 'running' in output:
logging.info("...suspending")
vagrant(['suspend'], cwd='builder')
logging.info("...waiting a sec...")
time.sleep(10)
p = FDroidPopen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'restore', 'fdroidclean'
],
cwd='builder')
if p.returncode == 0:
logging.info("...reset to snapshot - server is valid")
retcode, output = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
logging.info("...waiting a sec...")
time.sleep(10)
sshinfo = get_vagrant_sshinfo()
vm_ok = True
else:
logging.info("...failed to reset to snapshot")
else:
logging.info("...snapshot doesn't exist - "
"VBoxManage snapshot list:\n" + p.output)
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
logging.info("Removing broken/incomplete/unwanted build server")
vagrant(['destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
p = subprocess.Popen('vagrant --version',
shell=True,
stdout=subprocess.PIPE)
vver = p.communicate()[0]
if vver.startswith('Vagrant version 1.2'):
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant.configure("2") do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('end\n')
else:
with open('builder/Vagrantfile', 'w') as vf:
vf.write('Vagrant::Config.run do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('end\n')
logging.info("Starting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
logging.info("Connecting to virtual machine...")
sshinfo = get_vagrant_sshinfo()
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'],
username=sshinfo['user'],
port=sshinfo['port'],
timeout=300,
look_for_keys=False,
key_filename=sshinfo['idfile'])
sshs.close()
logging.info("Saving clean state of new build server")
retcode, _ = vagrant(['suspend'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to suspend build server")
logging.info("...waiting a sec...")
time.sleep(10)
p = FDroidPopen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'take', 'fdroidclean'
],
cwd='builder')
if p.returncode != 0:
raise BuildException("Failed to take snapshot")
logging.info("...waiting a sec...")
time.sleep(10)
logging.info("Restarting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
logging.info("...waiting a sec...")
time.sleep(10)
# Make sure it worked...
p = FDroidPopen([
'VBoxManage', 'snapshot',
get_builder_vm_id(), 'list', '--details'
],
cwd='builder')
if 'fdroidclean' not in p.output:
raise BuildException("Failed to take snapshot.")
return sshinfo
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] "
"[APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.warning("No unsigned directory - nothing to do")
sys.exit(1)
for f in [
config['keystorepassfile'], config['keystore'],
config['keypassfile']
]:
if not os.path.exists(f):
logging.error("Config error - missing '{0}'".format(f))
sys.exit(1)
# It was suggested at
# https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit
# that a package could be crafted, such that it would use the same signing
# key as an existing app. While it may be theoretically possible for such a
# colliding package ID to be generated, it seems virtually impossible that
# the colliding ID would be something that would be a) a valid package ID,
# and b) a sane-looking ID that would make its way into the repo.
# Nonetheless, to be sure, before publishing we check that there are no
# collisions, and refuse to do any publishing if that's the case...
allapps = metadata.read_metadata()
vercodes = common.read_pkg_args(args, True)
allaliases = []
for appid in allapps:
m = md5.new()
m.update(appid)
keyalias = m.hexdigest()[:8]
if keyalias in allaliases:
logging.error("There is a keyalias collision - publishing halted")
sys.exit(1)
allaliases.append(keyalias)
logging.info("{0} apps, {0} key aliases".format(len(allapps),
len(allaliases)))
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
appid, vercode = common.apknameinfo(apkfile)
apkfilename = os.path.basename(apkfile)
if vercodes and appid not in vercodes:
continue
if appid in vercodes and vercodes[appid]:
if vercode not in vercodes[appid]:
continue
logging.info("Processing " + apkfile)
# There ought to be valid metadata for this app, otherwise why are we
# trying to publish it?
if appid not in allapps:
logging.error("Unexpected {0} found in unsigned directory".format(
apkfilename))
sys.exit(1)
app = allapps[appid]
if app.get('Binaries', None):
# It's an app where we build from source, and verify the apk
# contents against a developer's binary, and then publish their
# version if everything checks out.
# The binary should already have been retrieved during the build
# process.
srcapk = apkfile + ".binary"
# Compare our unsigned one with the downloaded one...
compare_result = common.verify_apks(srcapk, apkfile, tmp_dir)
if compare_result:
logging.error("...verification failed - publish skipped : " +
compare_result)
continue
# Success! So move the downloaded file to the repo, and remove
# our built version.
shutil.move(srcapk, os.path.join(output_dir, apkfilename))
os.remove(apkfile)
else:
# It's a 'normal' app, i.e. we sign and publish it...
# Figure out the key alias name we'll use. Only the first 8
# characters are significant, so we'll use the first 8 from
# the MD5 of the app's ID and hope there are no collisions.
# If a collision does occur later, we're going to have to
# come up with a new alogrithm, AND rename all existing keys
# in the keystore!
if appid in config['keyaliases']:
# For this particular app, the key alias is overridden...
keyalias = config['keyaliases'][appid]
if keyalias.startswith('@'):
m = md5.new()
m.update(keyalias[1:])
keyalias = m.hexdigest()[:8]
else:
m = md5.new()
m.update(appid)
keyalias = m.hexdigest()[:8]
logging.info("Key alias: " + keyalias)
# See if we already have a key for this application, and
# if not generate one...
p = FDroidPopen([
'keytool', '-list', '-alias', keyalias, '-keystore',
config['keystore'], '-storepass:file',
config['keystorepassfile']
])
if p.returncode != 0:
logging.info("Key does not exist - generating...")
p = FDroidPopen([
'keytool', '-genkey', '-keystore', config['keystore'],
'-alias', keyalias, '-keyalg', 'RSA', '-keysize', '2048',
'-validity', '10000', '-storepass:file',
config['keystorepassfile'], '-keypass:file',
config['keypassfile'], '-dname', config['keydname']
])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to generate key")
# Sign the application...
p = FDroidPopen([
'jarsigner', '-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile'], '-keypass:file',
config['keypassfile'], '-sigalg', 'MD5withRSA', '-digestalg',
'SHA1', apkfile, keyalias
])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to sign application")
# Zipalign it...
p = SdkToolsPopen([
'zipalign', '-v', '4', apkfile,
os.path.join(output_dir, apkfilename)
])
if p.returncode != 0:
raise BuildException("Failed to align application")
os.remove(apkfile)
# Move the source tarball into the output directory...
tarfilename = apkfilename[:-4] + '_src.tar.gz'
tarfile = os.path.join(unsigned_dir, tarfilename)
if os.path.exists(tarfile):
shutil.move(tarfile, os.path.join(output_dir, tarfilename))
logging.info('Published ' + apkfilename)
def make_index(apps, sortedids, apks, repodir, archive, categories):
"""Make a repo index.
:param apps: fully populated apps list
:param apks: full populated apks list
:param repodir: the repo directory
:param archive: True if this is the archive repo, False if it's the
main one.
:param categories: list of categories
"""
doc = Document()
def addElement(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createTextNode(value))
parent.appendChild(el)
def addElementCDATA(name, value, doc, parent):
el = doc.createElement(name)
el.appendChild(doc.createCDATASection(value))
parent.appendChild(el)
root = doc.createElement("fdroid")
doc.appendChild(root)
repoel = doc.createElement("repo")
if archive:
repoel.setAttribute("name", config['archive_name'])
if config['repo_maxage'] != 0:
repoel.setAttribute("maxage", str(config['repo_maxage']))
repoel.setAttribute("icon", os.path.basename(config['archive_icon']))
repoel.setAttribute("url", config['archive_url'])
addElement('description', config['archive_description'], doc, repoel)
else:
repoel.setAttribute("name", config['repo_name'])
if config['repo_maxage'] != 0:
repoel.setAttribute("maxage", str(config['repo_maxage']))
repoel.setAttribute("icon", os.path.basename(config['repo_icon']))
repoel.setAttribute("url", config['repo_url'])
addElement('description', config['repo_description'], doc, repoel)
repoel.setAttribute("version", "12")
repoel.setAttribute("timestamp", str(int(time.time())))
if 'repo_keyalias' in config:
# Generate a certificate fingerprint the same way keytool does it
# (but with slightly different formatting)
def cert_fingerprint(data):
digest = hashlib.sha256(data).digest()
ret = []
ret.append(' '.join("%02X" % ord(b) for b in digest))
return " ".join(ret)
def extract_pubkey():
p = FDroidPopen([
'keytool', '-exportcert', '-alias', config['repo_keyalias'],
'-keystore', config['keystore'], '-storepass:file',
config['keystorepassfile']
] + config['smartcardoptions'],
output=False)
if p.returncode != 0:
msg = "Failed to get repo pubkey!"
if config['keystore'] == 'NONE':
msg += ' Is your crypto smartcard plugged in?'
logging.critical(msg)
sys.exit(1)
global repo_pubkey_fingerprint
repo_pubkey_fingerprint = cert_fingerprint(p.output)
return "".join("%02x" % ord(b) for b in p.output)
repoel.setAttribute("pubkey", extract_pubkey())
root.appendChild(repoel)
for appid in sortedids:
app = apps[appid]
if app['Disabled'] is not None:
continue
# Get a list of the apks for this app...
apklist = []
for apk in apks:
if apk['id'] == appid:
apklist.append(apk)
if len(apklist) == 0:
continue
apel = doc.createElement("application")
apel.setAttribute("id", app['id'])
root.appendChild(apel)
addElement('id', app['id'], doc, apel)
if 'added' in app:
addElement('added', time.strftime('%Y-%m-%d', app['added']), doc,
apel)
if 'lastupdated' in app:
addElement('lastupdated',
time.strftime('%Y-%m-%d', app['lastupdated']), doc,
apel)
addElement('name', app['Name'], doc, apel)
addElement('summary', app['Summary'], doc, apel)
if app['icon']:
addElement('icon', app['icon'], doc, apel)
def linkres(appid):
if appid in apps:
return ("fdroid:app" + appid, apps[appid]['Name'])
raise MetaDataException("Cannot resolve app id " + appid)
addElement('desc',
metadata.description_html(app['Description'], linkres), doc,
apel)
addElement('license', app['License'], doc, apel)
if 'Categories' in app:
addElement('categories', ','.join(app["Categories"]), doc, apel)
# We put the first (primary) category in LAST, which will have
# the desired effect of making clients that only understand one
# category see that one.
addElement('category', app["Categories"][0], doc, apel)
addElement('web', app['Web Site'], doc, apel)
addElement('source', app['Source Code'], doc, apel)
addElement('tracker', app['Issue Tracker'], doc, apel)
if app['Donate']:
addElement('donate', app['Donate'], doc, apel)
if app['Bitcoin']:
addElement('bitcoin', app['Bitcoin'], doc, apel)
if app['Litecoin']:
addElement('litecoin', app['Litecoin'], doc, apel)
if app['Dogecoin']:
addElement('dogecoin', app['Dogecoin'], doc, apel)
if app['FlattrID']:
addElement('flattr', app['FlattrID'], doc, apel)
# These elements actually refer to the current version (i.e. which
# one is recommended. They are historically mis-named, and need
# changing, but stay like this for now to support existing clients.
addElement('marketversion', app['Current Version'], doc, apel)
addElement('marketvercode', app['Current Version Code'], doc, apel)
if app['AntiFeatures']:
af = app['AntiFeatures'].split(',')
# TODO: Temporarily not including UpstreamNonFree in the index,
# because current F-Droid clients do not understand it, and also
# look ugly when they encounter an unknown antifeature. This
# filtering can be removed in time...
if 'UpstreamNonFree' in af:
af.remove('UpstreamNonFree')
if af:
addElement('antifeatures', ','.join(af), doc, apel)
if app['Provides']:
pv = app['Provides'].split(',')
addElement('provides', ','.join(pv), doc, apel)
if app['Requires Root']:
addElement('requirements', 'root', doc, apel)
# Sort the apk list into version order, just so the web site
# doesn't have to do any work by default...
apklist = sorted(apklist,
key=lambda apk: apk['versioncode'],
reverse=True)
# Check for duplicates - they will make the client unhappy...
for i in range(len(apklist) - 1):
if apklist[i]['versioncode'] == apklist[i + 1]['versioncode']:
logging.critical(
"duplicate versions: '%s' - '%s'" %
(apklist[i]['apkname'], apklist[i + 1]['apkname']))
sys.exit(1)
for apk in apklist:
apkel = doc.createElement("package")
apel.appendChild(apkel)
addElement('version', apk['version'], doc, apkel)
addElement('versioncode', str(apk['versioncode']), doc, apkel)
addElement('apkname', apk['apkname'], doc, apkel)
if 'srcname' in apk:
addElement('srcname', apk['srcname'], doc, apkel)
for hash_type in ['sha256']:
if hash_type not in apk:
continue
hashel = doc.createElement("hash")
hashel.setAttribute("type", hash_type)
hashel.appendChild(doc.createTextNode(apk[hash_type]))
apkel.appendChild(hashel)
addElement('sig', apk['sig'], doc, apkel)
addElement('size', str(apk['size']), doc, apkel)
addElement('sdkver', str(apk['sdkversion']), doc, apkel)
if 'maxsdkversion' in apk:
addElement('maxsdkver', str(apk['maxsdkversion']), doc, apkel)
if 'added' in apk:
addElement('added', time.strftime('%Y-%m-%d', apk['added']),
doc, apkel)
if app['Requires Root']:
if 'ACCESS_SUPERUSER' not in apk['permissions']:
apk['permissions'].add('ACCESS_SUPERUSER')
if len(apk['permissions']) > 0:
addElement('permissions', ','.join(apk['permissions']), doc,
apkel)
if 'nativecode' in apk and len(apk['nativecode']) > 0:
addElement('nativecode', ','.join(apk['nativecode']), doc,
apkel)
if len(apk['features']) > 0:
addElement('features', ','.join(apk['features']), doc, apkel)
of = open(os.path.join(repodir, 'index.xml'), 'wb')
if options.pretty:
output = doc.toprettyxml()
else:
output = doc.toxml()
of.write(output)
of.close()
if 'repo_keyalias' in config:
logging.info("Creating signed index with this key (SHA256):")
logging.info("%s" % repo_pubkey_fingerprint)
# Create a jar of the index...
p = FDroidPopen(['jar', 'cf', 'index.jar', 'index.xml'], cwd=repodir)
if p.returncode != 0:
logging.critical("Failed to create jar file")
sys.exit(1)
# Sign the index...
args = [
'jarsigner', '-keystore', config['keystore'], '-storepass:file',
config['keystorepassfile'], '-digestalg', 'SHA1', '-sigalg',
'MD5withRSA',
os.path.join(repodir, 'index.jar'), config['repo_keyalias']
]
if config['keystore'] == 'NONE':
args += config['smartcardoptions']
else: # smardcards never use -keypass
args += ['-keypass:file', config['keypassfile']]
p = FDroidPopen(args)
# TODO keypass should be sent via stdin
if p.returncode != 0:
logging.critical("Failed to sign index")
sys.exit(1)
# Copy the repo icon into the repo directory...
icon_dir = os.path.join(repodir, 'icons')
iconfilename = os.path.join(icon_dir,
os.path.basename(config['repo_icon']))
shutil.copyfile(config['repo_icon'], iconfilename)
# Write a category list in the repo to allow quick access...
catdata = ''
for cat in categories:
catdata += cat + '\n'
f = open(os.path.join(repodir, 'categories.txt'), 'w')
f.write(catdata)
f.close()
def main():
global options, config
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.error("No unsigned directory - nothing to do")
sys.exit(0)
verified = 0
notverified = 0
vercodes = common.read_pkg_args(args, True)
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
appid, vercode = common.apknameinfo(apkfile)
if vercodes and appid not in vercodes:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
try:
logging.info("Processing " + apkfilename)
remoteapk = os.path.join(tmp_dir, apkfilename)
if os.path.exists(remoteapk):
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
logging.info("...retrieving " + url)
p = FDroidPopen(['wget', url], cwd=tmp_dir)
if p.returncode != 0:
raise FDroidException("Failed to get " + apkfilename)
thisdir = os.path.join(tmp_dir, 'this_apk')
thatdir = os.path.join(tmp_dir, 'that_apk')
for d in [thisdir, thatdir]:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d)
if subprocess.call(['jar', 'xf',
os.path.join("..", "..", unsigned_dir, apkfilename)],
cwd=thisdir) != 0:
raise FDroidException("Failed to unpack local build of " + apkfilename)
if subprocess.call(['jar', 'xf',
os.path.join("..", "..", remoteapk)],
cwd=thatdir) != 0:
raise FDroidException("Failed to unpack remote build of " + apkfilename)
p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir)
lines = p.output.splitlines()
if len(lines) != 1 or 'META-INF' not in lines[0]:
raise FDroidException("Unexpected diff output - " + p.output)
logging.info("...successfully verified")
verified += 1
except FDroidException, e:
logging.info("...NOT verified - {0}".format(e))
notverified += 1
def get_clean_vm(reset=False):
"""Get a clean VM ready to do a buildserver build.
This might involve creating and starting a new virtual machine from
scratch, or it might be as simple (unless overridden by the reset
parameter) as re-using a snapshot created previously.
A BuildException will be raised if anything goes wrong.
:reset: True to force creating from scratch.
:returns: A dictionary containing 'hostname', 'port', 'user'
and 'idfile'
"""
# Reset existing builder machine to a clean state if possible.
vm_ok = False
if not reset:
logging.info("Checking for valid existing build server")
if got_valid_builder_vm():
logging.info("...VM is present")
p = FDroidPopen(['VBoxManage', 'snapshot',
get_builder_vm_id(), 'list',
'--details'], cwd='builder')
if 'fdroidclean' in p.output:
logging.info("...snapshot exists - resetting build server to "
"clean state")
retcode, output = vagrant(['status'], cwd='builder')
if 'running' in output:
logging.info("...suspending")
vagrant(['suspend'], cwd='builder')
logging.info("...waiting a sec...")
time.sleep(10)
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
'restore', 'fdroidclean'],
cwd='builder')
if p.returncode == 0:
logging.info("...reset to snapshot - server is valid")
retcode, output = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
logging.info("...waiting a sec...")
time.sleep(10)
sshinfo = get_vagrant_sshinfo()
vm_ok = True
else:
logging.info("...failed to reset to snapshot")
else:
logging.info("...snapshot doesn't exist - "
"VBoxManage snapshot list:\n" + p.output)
# If we can't use the existing machine for any reason, make a
# new one from scratch.
if not vm_ok:
if os.path.exists('builder'):
logging.info("Removing broken/incomplete/unwanted build server")
vagrant(['destroy', '-f'], cwd='builder')
shutil.rmtree('builder')
os.mkdir('builder')
p = subprocess.Popen(['vagrant', '--version'],
stdout=subprocess.PIPE)
vver = p.communicate()[0].strip().split(' ')[1]
if vver.split('.')[0] != '1' or int(vver.split('.')[1]) < 4:
raise BuildException("Unsupported vagrant version {0}".format(vver))
with open(os.path.join('builder', 'Vagrantfile'), 'w') as vf:
vf.write('Vagrant.configure("2") do |config|\n')
vf.write('config.vm.box = "buildserver"\n')
vf.write('config.vm.synced_folder ".", "/vagrant", disabled: true\n')
vf.write('end\n')
logging.info("Starting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
# Open SSH connection to make sure it's working and ready...
logging.info("Connecting to virtual machine...")
sshinfo = get_vagrant_sshinfo()
sshs = paramiko.SSHClient()
sshs.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshs.connect(sshinfo['hostname'], username=sshinfo['user'],
port=sshinfo['port'], timeout=300,
look_for_keys=False,
key_filename=sshinfo['idfile'])
sshs.close()
logging.info("Saving clean state of new build server")
retcode, _ = vagrant(['suspend'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to suspend build server")
logging.info("...waiting a sec...")
time.sleep(10)
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
'take', 'fdroidclean'],
cwd='builder')
if p.returncode != 0:
raise BuildException("Failed to take snapshot")
logging.info("...waiting a sec...")
time.sleep(10)
logging.info("Restarting new build server")
retcode, _ = vagrant(['up'], cwd='builder')
if retcode != 0:
raise BuildException("Failed to start build server")
logging.info("...waiting a sec...")
time.sleep(10)
# Make sure it worked...
p = FDroidPopen(['VBoxManage', 'snapshot', get_builder_vm_id(),
'list', '--details'],
cwd='builder')
if 'fdroidclean' not in p.output:
raise BuildException("Failed to take snapshot.")
return sshinfo
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, install, force, onserver):
"""Do a build locally."""
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir, extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild.get('maven', 'no') != 'no':
print "Cleaning Maven project..."
cmd = [config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Cleaning Gradle project..."
cmd = [config['gradle'], 'clean']
if '@' in thisbuild['gradle']:
gradle_dir = os.path.join(root_dir, thisbuild['gradle'].split('@',1)[1])
gradle_dir = os.path.normpath(gradle_dir)
else:
gradle_dir = root_dir
p = FDroidPopen(cmd, cwd=gradle_dir)
elif thisbuild.get('update', '.') != 'no' and thisbuild.get('kivy', 'no') == 'no':
print "Cleaning Ant project..."
cmd = ['ant', 'clean']
p = FDroidPopen(cmd, cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException("Error cleaning %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Also clean jni
print "Cleaning jni dirs..."
for baddir in [
'libs/armeabi-v7a', 'libs/armeabi',
'libs/mips', 'libs/x86', 'obj']:
badpath = os.path.join(build_dir, baddir)
if os.path.exists(badpath):
print "Removing '%s'" % badpath
shutil.rmtree(badpath)
# Scan before building...
print "Scanning source for common problems..."
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
if len(buildprobs) > 0:
print 'Scanner found ' + str(len(buildprobs)) + ' problems:'
for problem in buildprobs:
print '...' + problem
if not force:
raise BuildException("Can't build due to " +
str(len(buildprobs)) + " scanned problems")
# Build the source tarball right before we build the release...
print "Creating source tarball..."
tarname = common.getsrcname(app,thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
for vcs_dir in ['.svn', '.git', '.hg', '.bzr']:
if f.endswith(vcs_dir):
return True
return False
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
# Run a build command if one is required...
if 'build' in thisbuild:
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
if options.verbose:
print "Running 'build' commands in %s" % root_dir
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.stdout, p.stderr)
# Build native stuff if required...
if thisbuild.get('buildjni') not in (None, 'no'):
print "Building native libraries..."
jni_components = thisbuild.get('buildjni')
if jni_components == 'yes':
jni_components = ['']
else:
jni_components = [c.strip() for c in jni_components.split(';')]
ndkbuild = os.path.join(config['ndk_path'], "ndk-build")
for d in jni_components:
if options.verbose:
print "Running ndk-build in " + root_dir + '/' + d
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen([ndkbuild], cwd=os.path.join(root_dir,d))
if p.returncode != 0:
raise BuildException("NDK build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
p = None
# Build the release...
if thisbuild.get('maven', 'no') != 'no':
print "Building Maven project..."
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1])
else:
maven_dir = root_dir
mvncmd = [config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path']]
if install:
mvncmd += ['-Dandroid.sign.debug=true', 'package', 'android:deploy']
else:
mvncmd += ['-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package']
if 'target' in thisbuild:
target = thisbuild["target"].split('-')[1]
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=root_dir)
if '@' in thisbuild['maven']:
subprocess.call(['sed', '-i',
's@<platform>[0-9]*</platform>@<platform>'+target+'</platform>@g',
'pom.xml'], cwd=maven_dir)
if 'mvnflags' in thisbuild:
mvncmd += thisbuild['mvnflags']
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild.get('kivy', 'no') != 'no':
print "Building Kivy project..."
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException("Expected to find buildozer-compatible spec at {0}"
.format(spec))
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
if subprocess.call(cmd, cwd='python-for-android', shell=True) != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
if cid != app['id']:
raise BuildException("Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = ['./build.py'
'--dir', root_dir,
'--name', bconfig.get('app', 'title'),
'--package', app['id'],
'--version', bconfig.get('app', 'version'),
'--orientation', orientation,
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild.get('gradle', 'no') != 'no':
print "Building Gradle project..."
if '@' in thisbuild['gradle']:
flavour = thisbuild['gradle'].split('@')[0]
gradle_dir = thisbuild['gradle'].split('@')[1]
gradle_dir = os.path.join(root_dir, gradle_dir)
else:
flavour = thisbuild['gradle']
gradle_dir = root_dir
if 'compilesdk' in thisbuild:
level = thisbuild["compilesdk"].split('-')[1]
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=root_dir)
if '@' in thisbuild['gradle']:
subprocess.call(['sed', '-i',
's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+level+'@g',
'build.gradle'], cwd=gradle_dir)
for root, dirs, files in os.walk(build_dir):
for f in files:
if f == 'build.gradle':
adapt_gradle(os.path.join(root, f))
break
if flavour in ['main', 'yes', '']:
flavour = ''
commands = [config['gradle']]
if 'preassemble' in thisbuild:
for task in thisbuild['preassemble'].split():
commands.append(task)
if install:
commands += ['assemble'+flavour+'Debug', 'install'+flavour+'Debug']
else:
commands += ['assemble'+flavour+'Release']
p = FDroidPopen(commands, cwd=gradle_dir)
else:
print "Building Ant project..."
cmd = ['ant']
if install:
cmd += ['debug','install']
elif 'antcommand' in thisbuild:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p.returncode != 0:
raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout, p.stderr)
print "Successfully built version " + thisbuild['version'] + ' of ' + app['id']
if install:
return
# Find the apk name in the output...
if 'bindir' in thisbuild:
bindir = os.path.join(build_dir, thisbuild['bindir'])
if thisbuild.get('maven', 'no') != 'no':
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if any(a in line for a in ('.apk','.ap_'))])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S|re.M)
if not m:
m = re.match(r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir + r'/([^/]+)\.ap[_k][,\]]',
stdout_apk, re.S|re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild.get('kivy', 'no') != 'no':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild.get('gradle', 'no') != 'no':
dd = build_dir
if 'subdir' in thisbuild:
dd = os.path.join(dd, thisbuild['subdir'])
if flavour in ['main', 'yes', '']:
name = '-'.join([os.path.basename(dd), 'release', 'unsigned'])
else:
name = '-'.join([os.path.basename(dd), flavour, 'release', 'unsigned'])
src = os.path.join(dd, 'build', 'apk', name+'.apk')
else:
stdout_apk = '\n'.join([
line for line in p.stdout.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S|re.M).group(1)
src = os.path.join(bindir, src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
print "Checking " + src
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " + src)
p = subprocess.Popen([os.path.join(config['sdk_path'],
'build-tools', config['build_tools'], 'aapt'),
'dump', 'badging', src],
stdout=subprocess.PIPE)
output = p.communicate()[0]
vercode = None
version = None
foundid = None
for line in output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
foundid = re.match(pat, line).group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
vercode = re.match(pat, line).group(1)
pat = re.compile(".*versionName='([^']*)'.*")
version = re.match(pat, line).group(1)
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException("Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid + " but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or
vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'")
% (version, str(vercode), thisbuild['version'], str(thisbuild['vercode']))
)
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app,thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir,
extlib_dir, tmp_dir, force, onserver):
"""Do a build locally."""
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if not config['ndk_path']:
logging.critical("$ANDROID_NDK is not set!")
sys.exit(3)
elif not os.path.isdir(config['sdk_path']):
logging.critical(
"$ANDROID_NDK points to a non-existing directory!")
sys.exit(3)
# Prepare the source code...
root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
build_dir, srclib_dir,
extlib_dir, onserver)
# We need to clean via the build tool in case the binary dirs are
# different from the default ones
p = None
if thisbuild['type'] == 'maven':
logging.info("Cleaning Maven project...")
cmd = [
config['mvn3'], 'clean', '-Dandroid.sdk.path=' + config['sdk_path']
]
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
maven_dir = os.path.normpath(maven_dir)
else:
maven_dir = root_dir
p = FDroidPopen(cmd, cwd=maven_dir)
elif thisbuild['type'] == 'gradle':
logging.info("Cleaning Gradle project...")
cmd = [config['gradle'], 'clean']
adapt_gradle(build_dir)
for name, number, libpath in srclibpaths:
adapt_gradle(libpath)
p = FDroidPopen(cmd, cwd=root_dir)
elif thisbuild['type'] == 'kivy':
pass
elif thisbuild['type'] == 'ant':
logging.info("Cleaning Ant project...")
p = FDroidPopen(['ant', 'clean'], cwd=root_dir)
if p is not None and p.returncode != 0:
raise BuildException(
"Error cleaning %s:%s" % (app['id'], thisbuild['version']),
p.output)
for root, dirs, files in os.walk(build_dir):
# Don't remove possibly necessary 'gradle' dirs if 'gradlew' is not there
if 'gradlew' in files:
logging.debug("Getting rid of Gradle wrapper stuff in %s" % root)
os.remove(os.path.join(root, 'gradlew'))
if 'gradlew.bat' in files:
os.remove(os.path.join(root, 'gradlew.bat'))
if 'gradle' in dirs:
shutil.rmtree(os.path.join(root, 'gradle'))
if not options.skipscan:
# Scan before building...
logging.info("Scanning source for common problems...")
count = common.scan_source(build_dir, root_dir, thisbuild)
if count > 0:
if force:
logging.warn('Scanner found %d problems:' % count)
else:
raise BuildException(
"Can't build due to %d errors while scanning" % count)
if not options.notarball:
# Build the source tarball right before we build the release...
logging.info("Creating source tarball...")
tarname = common.getsrcname(app, thisbuild)
tarball = tarfile.open(os.path.join(tmp_dir, tarname), "w:gz")
def tarexc(f):
return any(f.endswith(s) for s in ['.svn', '.git', '.hg', '.bzr'])
tarball.add(build_dir, tarname, exclude=tarexc)
tarball.close()
if onserver:
manifest = os.path.join(root_dir, 'AndroidManifest.xml')
if os.path.exists(manifest):
homedir = os.path.expanduser('~')
with open(os.path.join(homedir, 'buildserverid'), 'r') as f:
buildserverid = f.read()
with open(os.path.join(homedir, 'fdroidserverid'), 'r') as f:
fdroidserverid = f.read()
with open(manifest, 'r') as f:
manifestcontent = f.read()
manifestcontent = manifestcontent.replace(
'</manifest>',
'<fdroid buildserverid="' + buildserverid + '"' +
' fdroidserverid="' + fdroidserverid + '"' + '/></manifest>')
with open(manifest, 'w') as f:
f.write(manifestcontent)
# Run a build command if one is required...
if thisbuild['build']:
logging.info("Running 'build' commands in %s" % root_dir)
cmd = common.replace_config_vars(thisbuild['build'])
# Substitute source library paths into commands...
for name, number, libpath in srclibpaths:
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException(
"Error running build command for %s:%s" %
(app['id'], thisbuild['version']), p.output)
# Build native stuff if required...
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
logging.info("Building the native code")
jni_components = thisbuild['buildjni']
if jni_components == ['yes']:
jni_components = ['']
cmd = [os.path.join(config['ndk_path'], "ndk-build"), "-j1"]
for d in jni_components:
if d:
logging.info("Building native code in '%s'" % d)
else:
logging.info("Building native code in the main project")
manifest = root_dir + '/' + d + '/AndroidManifest.xml'
if os.path.exists(manifest):
# Read and write the whole AM.xml to fix newlines and avoid
# the ndk r8c or later 'wordlist' errors. The outcome of this
# under gnu/linux is the same as when using tools like
# dos2unix, but the native python way is faster and will
# work in non-unix systems.
manifest_text = open(manifest, 'U').read()
open(manifest, 'w').write(manifest_text)
# In case the AM.xml read was big, free the memory
del manifest_text
p = FDroidPopen(cmd, cwd=os.path.join(root_dir, d))
if p.returncode != 0:
raise BuildException(
"NDK build failed for %s:%s" %
(app['id'], thisbuild['version']), p.output)
p = None
# Build the release...
if thisbuild['type'] == 'maven':
logging.info("Building Maven project...")
if '@' in thisbuild['maven']:
maven_dir = os.path.join(root_dir,
thisbuild['maven'].split('@', 1)[1])
else:
maven_dir = root_dir
mvncmd = [
config['mvn3'], '-Dandroid.sdk.path=' + config['sdk_path'],
'-Dmaven.jar.sign.skip=true', '-Dmaven.test.skip=true',
'-Dandroid.sign.debug=false', '-Dandroid.release=true', 'package'
]
if thisbuild['target']:
target = thisbuild["target"].split('-')[1]
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=root_dir)
if '@' in thisbuild['maven']:
FDroidPopen([
'sed', '-i', 's@<platform>[0-9]*</platform>@<platform>' +
target + '</platform>@g', 'pom.xml'
],
cwd=maven_dir)
p = FDroidPopen(mvncmd, cwd=maven_dir)
bindir = os.path.join(root_dir, 'target')
elif thisbuild['type'] == 'kivy':
logging.info("Building Kivy project...")
spec = os.path.join(root_dir, 'buildozer.spec')
if not os.path.exists(spec):
raise BuildException(
"Expected to find buildozer-compatible spec at {0}".format(
spec))
defaults = {
'orientation': 'landscape',
'icon': '',
'permissions': '',
'android.api': "18"
}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
distdir = 'python-for-android/dist/fdroid'
if os.path.exists(distdir):
shutil.rmtree(distdir)
modules = bconfig.get('app', 'requirements').split(',')
cmd = 'ANDROIDSDK=' + config['sdk_path']
cmd += ' ANDROIDNDK=' + config['ndk_path']
cmd += ' ANDROIDNDKVER=r9'
cmd += ' ANDROIDAPI=' + str(bconfig.get('app', 'android.api'))
cmd += ' VIRTUALENV=virtualenv'
cmd += ' ./distribute.sh'
cmd += ' -m ' + "'" + ' '.join(modules) + "'"
cmd += ' -d fdroid'
p = FDroidPopen(cmd, cwd='python-for-android', shell=True)
if p.returncode != 0:
raise BuildException("Distribute build failed")
cid = bconfig.get('app', 'package.domain') + '.' + bconfig.get(
'app', 'package.name')
if cid != app['id']:
raise BuildException(
"Package ID mismatch between metadata and spec")
orientation = bconfig.get('app', 'orientation', 'landscape')
if orientation == 'all':
orientation = 'sensor'
cmd = [
'./build.py'
'--dir', root_dir, '--name',
bconfig.get('app', 'title'), '--package', app['id'], '--version',
bconfig.get('app', 'version'), '--orientation', orientation
]
perms = bconfig.get('app', 'permissions')
for perm in perms.split(','):
cmd.extend(['--permission', perm])
if config.get('app', 'fullscreen') == 0:
cmd.append('--window')
icon = bconfig.get('app', 'icon.filename')
if icon:
cmd.extend(['--icon', os.path.join(root_dir, icon)])
cmd.append('release')
p = FDroidPopen(cmd, cwd=distdir)
elif thisbuild['type'] == 'gradle':
logging.info("Building Gradle project...")
flavours = thisbuild['gradle'].split(',')
if len(flavours) == 1 and flavours[0] in ['main', 'yes', '']:
flavours[0] = ''
commands = [config['gradle']]
if thisbuild['preassemble']:
commands += thisbuild['preassemble'].split()
flavours_cmd = ''.join(flavours)
if flavours_cmd:
flavours_cmd = flavours_cmd[0].upper() + flavours_cmd[1:]
commands += ['assemble' + flavours_cmd + 'Release']
# Avoid having to use lintOptions.abortOnError false
if thisbuild['gradlepluginver'] >= LooseVersion('0.7'):
with open(os.path.join(root_dir, 'build.gradle'), "a") as f:
f.write(
"\nandroid { lintOptions { checkReleaseBuilds false } }\n")
p = FDroidPopen(commands, cwd=root_dir)
elif thisbuild['type'] == 'ant':
logging.info("Building Ant project...")
cmd = ['ant']
if thisbuild['antcommand']:
cmd += [thisbuild['antcommand']]
else:
cmd += ['release']
p = FDroidPopen(cmd, cwd=root_dir)
bindir = os.path.join(root_dir, 'bin')
if p is not None and p.returncode != 0:
raise BuildException(
"Build failed for %s:%s" % (app['id'], thisbuild['version']),
p.output)
logging.info("Successfully built version " + thisbuild['version'] +
' of ' + app['id'])
if thisbuild['type'] == 'maven':
stdout_apk = '\n'.join([
line for line in p.output.splitlines()
if any(a in line for a in ('.apk', '.ap_', '.jar'))
])
m = re.match(r".*^\[INFO\] .*apkbuilder.*/([^/]*)\.apk", stdout_apk,
re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Creating additional unsigned apk file .*/([^/]+)\.apk[^l]",
stdout_apk, re.S | re.M)
if not m:
m = re.match(
r'.*^\[INFO\] [^$]*aapt \[package,[^$]*' + bindir +
r'/([^/]+)\.ap[_k][,\]]', stdout_apk, re.S | re.M)
if not m:
m = re.match(
r".*^\[INFO\] Building jar: .*/" + bindir + r"/(.+)\.jar",
stdout_apk, re.S | re.M)
if not m:
raise BuildException('Failed to find output')
src = m.group(1)
src = os.path.join(bindir, src) + '.apk'
elif thisbuild['type'] == 'kivy':
src = 'python-for-android/dist/default/bin/{0}-{1}-release.apk'.format(
bconfig.get('app', 'title'), bconfig.get('app', 'version'))
elif thisbuild['type'] == 'gradle':
if thisbuild['gradlepluginver'] >= LooseVersion('0.11'):
apks_dir = os.path.join(root_dir, 'build', 'outputs', 'apk')
else:
apks_dir = os.path.join(root_dir, 'build', 'apk')
apks = glob.glob(os.path.join(apks_dir, '*-release-unsigned.apk'))
if len(apks) > 1:
raise BuildException(
'More than one resulting apks found in %s' % apks_dir,
'\n'.join(apks))
if len(apks) < 1:
raise BuildException('Failed to find gradle output in %s' %
apks_dir)
src = apks[0]
elif thisbuild['type'] == 'ant':
stdout_apk = '\n'.join(
[line for line in p.output.splitlines() if '.apk' in line])
src = re.match(r".*^.*Creating (.+) for release.*$.*", stdout_apk,
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif thisbuild['type'] == 'raw':
src = os.path.join(root_dir, thisbuild['output'])
src = os.path.normpath(src)
# Make sure it's not debuggable...
if common.isApkDebuggable(src, config):
raise BuildException("APK is debuggable")
# By way of a sanity check, make sure the version and version
# code in our new apk match what we expect...
logging.debug("Checking " + src)
if not os.path.exists(src):
raise BuildException("Unsigned apk is not at expected location of " +
src)
p = SilentPopen([config['aapt'], 'dump', 'badging', src])
vercode = None
version = None
foundid = None
nativecode = None
for line in p.output.splitlines():
if line.startswith("package:"):
pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
m = pat.match(line)
if m:
foundid = m.group(1)
pat = re.compile(".*versionCode='([0-9]*)'.*")
m = pat.match(line)
if m:
vercode = m.group(1)
pat = re.compile(".*versionName='([^']*)'.*")
m = pat.match(line)
if m:
version = m.group(1)
elif line.startswith("native-code:"):
nativecode = line[12:]
# Ignore empty strings or any kind of space/newline chars that we don't
# care about
if nativecode is not None:
nativecode = nativecode.strip()
nativecode = None if not nativecode else nativecode
if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
if nativecode is None:
raise BuildException(
"Native code should have been built but none was packaged")
if thisbuild['novcheck']:
vercode = thisbuild['vercode']
version = thisbuild['version']
if not version or not vercode:
raise BuildException(
"Could not find version information in build in output")
if not foundid:
raise BuildException("Could not find package ID in output")
if foundid != app['id']:
raise BuildException("Wrong package ID - build " + foundid +
" but expected " + app['id'])
# Some apps (e.g. Timeriffic) have had the bonkers idea of
# including the entire changelog in the version number. Remove
# it so we can compare. (TODO: might be better to remove it
# before we compile, in fact)
index = version.find(" //")
if index != -1:
version = version[:index]
if (version != thisbuild['version'] or vercode != thisbuild['vercode']):
raise BuildException(("Unexpected version/version code in output;"
" APK: '%s' / '%s', "
" Expected: '%s' / '%s'") %
(version, str(vercode), thisbuild['version'],
str(thisbuild['vercode'])))
# Copy the unsigned apk to our destination directory for further
# processing (by publish.py)...
dest = os.path.join(output_dir, common.getapkname(app, thisbuild))
shutil.copyfile(src, dest)
# Move the source tarball into the output directory...
if output_dir != tmp_dir and not options.notarball:
shutil.move(os.path.join(tmp_dir, tarname),
os.path.join(output_dir, tarname))
def main():
global options, config
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-a",
"--all",
action="store_true",
default=False,
help="Install all signed applications available")
(options, args) = parser.parse_args()
if not args and not options.all:
raise OptionError(
"If you really want to install all the signed apps, use --all",
"all")
config = common.read_config(options)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("No signed output directory - nothing to do")
sys.exit(0)
if args:
vercodes = common.read_pkg_args(args, True)
apks = {appid: None for appid in vercodes}
# Get the signed apk with the highest vercode
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
try:
appid, vercode = common.apknameinfo(apkfile)
except FDroidException:
continue
if appid not in apks:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
apks[appid] = apkfile
for appid, apk in apks.iteritems():
if not apk:
raise FDroidException("No signed apk available for %s" % appid)
else:
apks = {
common.apknameinfo(apkfile)[0]: apkfile
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk')))
}
for appid, apk in apks.iteritems():
# Get device list each time to avoid device not found errors
devs = devices()
if not devs:
raise FDroidException("No attached devices found")
logging.info("Installing %s..." % apk)
for dev in devs:
logging.info("Installing %s on %s..." % (apk, dev))
p = FDroidPopen([config['adb'], "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
fail = line[9:-1]
if not fail:
continue
if fail == "INSTALL_FAILED_ALREADY_EXISTS":
logging.warn("%s is already installed on %s." % (apk, dev))
else:
raise FDroidException("Failed to install %s on %s: %s" %
(apk, dev, fail))
logging.info("\nFinished")
t = f.read(1024)
if len(t) == 0:
break
sha.update(t)
thisinfo['sha256'] = sha.hexdigest()
# Get the signature (or md5 of, to be precise)...
getsig_dir = os.path.join(os.path.dirname(__file__), 'getsig')
if not os.path.exists(getsig_dir + "/getsig.class"):
logging.critical(
"getsig.class not found. To fix: cd '%s' && ./make.sh" %
getsig_dir)
sys.exit(1)
p = FDroidPopen([
'java', '-cp',
os.path.join(os.path.dirname(__file__), 'getsig'), 'getsig',
os.path.join(os.getcwd(), apkfile)
])
thisinfo['sig'] = None
for line in p.output.splitlines():
if line.startswith('Result:'):
thisinfo['sig'] = line[7:].strip()
break
if p.returncode != 0 or not thisinfo['sig']:
logging.critical("Failed to get apk signature")
sys.exit(1)
apk = zipfile.ZipFile(apkfile, 'r')
iconfilename = "%s.%s.png" % (thisinfo['id'],
thisinfo['versioncode'])
