def main():
parser = OptionParser(usage="Usage: %prog")
parser.parse_args()
common.read_config(None)
metadata.read_metadata(xref=True)
def main():
parser = ArgumentParser(usage="%(prog)s")
common.setup_global_opts(parser)
parser.parse_args()
common.read_config(None)
metadata.read_metadata(xref=True)
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Process only the specified package")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
apps = metadata.read_metadata(package=options.package, xref=False)
if len(apps) == 0 and options.package:
print "No such package"
sys.exit(1)
for app in apps:
print "Writing " + app['id']
metadata.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
print "Finished."
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
(options, args) = parser.parse_args()
config = common.read_config(options)
if len(args) != 1:
print "Specify a single command"
sys.exit(1)
if args[0] != 'init' and args[0] != 'update':
print "The only commands currently supported are 'init' and 'update'"
sys.exit(1)
serverwebroot = config['serverwebroot'].rstrip('/').replace('//', '/')
host, fdroiddir = serverwebroot.split(':')
serverrepobase = os.path.basename(fdroiddir)
if 'nonstandardwebroot' in config and config['nonstandardwebroot'] == True:
standardwebroot = False
else:
standardwebroot = True
if serverrepobase != 'fdroid' and standardwebroot:
print('ERROR: serverwebroot does not end with "fdroid", '
+ 'perhaps you meant one of these:\n\t'
+ serverwebroot.rstrip('/') + '/fdroid\n\t'
+ serverwebroot.rstrip('/').rstrip(serverrepobase) + 'fdroid')
sys.exit(1)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
for repodir in repodirs:
if args[0] == 'init':
if subprocess.call(['ssh', '-v', host,
'mkdir -p', fdroiddir + '/' + repodir]) != 0:
sys.exit(1)
elif args[0] == 'update':
index = os.path.join(repodir, 'index.xml')
indexjar = os.path.join(repodir, 'index.jar')
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
'--exclude', index, '--exclude', indexjar,
repodir, config['serverwebroot']]) != 0:
sys.exit(1)
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
index,
config['serverwebroot'] + '/' + repodir]) != 0:
sys.exit(1)
if subprocess.call(['rsync', '-u', '-v', '-r', '--delete',
indexjar,
config['serverwebroot'] + '/' + repodir]) != 0:
sys.exit(1)
sys.exit(0)
def is_admin(nick, log=logger.log):
try:
adminlist = common.read_config('admins.json')
except Exception as e:
log('error', e)
return False
if not isinstance(adminlist, list):
log('error', 'admins.json should contain a list')
return False
return nick in adminlist
def main():
settings = common.read_config('config.json')
if not valid_settings(settings):
exit()
state = statekeeper.StateKeeper()
while True:
if irc.run(settings, state) == 'quit':
break
sleep(settings['irc']['reconnect_delay'])
def main():
global config, options
anywarns = False
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
common.setup_global_opts(parser)
parser.add_argument("-f", "--format", action="store_true", default=False,
help="Also warn about formatting issues, like rewritemeta -l")
parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
options = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=True)
apps = common.read_app_args(options.appid, allapps, False)
for appid, app in apps.iteritems():
if app.Disabled:
continue
warns = []
for check_func in [
check_regexes,
check_ucm_tags,
check_char_limits,
check_old_links,
check_checkupdates_ran,
check_useless_fields,
check_empty_fields,
check_categories,
check_duplicates,
check_mediawiki_links,
check_bulleted_lists,
check_builds,
]:
warns += check_func(app)
if options.format:
if not rewritemeta.proper_format(app):
warns.append("Run rewritemeta to fix formatting")
if warns:
anywarns = True
for warn in warns:
print("%s: %s" % (appid, warn))
if anywarns:
sys.exit(1)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
common.setup_global_opts(parser)
parser.add_argument("-l", "--list", action="store_true", default=False,
help="List files that would be reformatted")
parser.add_argument("-t", "--to", default=None,
help="Rewrite to a specific format")
parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
options = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=True)
apps = common.read_app_args(options.appid, allapps, False)
if options.list and options.to is not None:
parser.error("Cannot use --list and --to at the same time")
supported = ['txt', 'yaml']
if options.to is not None and options.to not in supported:
parser.error("Must give a valid format to --to")
for appid, app in apps.iteritems():
base, ext = common.get_extension(app.metadatapath)
if not options.to and ext not in supported:
logging.info("Ignoring %s file at '%s'" % (ext, app.metadatapath))
continue
to_ext = ext
if options.to is not None:
to_ext = options.to
if options.list:
if not proper_format(app):
print app.metadatapath
continue
with open(base + '.' + to_ext, 'w') as f:
metadata.write_metadata(to_ext, f, app)
if ext != to_ext:
os.remove(app.metadatapath)
logging.debug("Finished.")
def check_domains(domain_base): import os.path app_config = common.read_config() extensions = read_extensions(common.get_full_path(app_config['ExtensionsFile'])) domains_to_check = set() for e in extensions: domains_to_check.add(domain_base+'.'+e) domains_info = api_check_domains(domains_to_check, app_config['RrpproxyLogin'], app_config['RrpproxyPw']) write_csv(domains_info, domain_base, os.path.join(app_config['OutputPath'], domain_base+'.csv'))
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options]")
common.setup_global_opts(parser)
options = parser.parse_args()
config = common.read_config(options)
if 'jarsigner' not in config:
logging.critical('Java jarsigner not found! Install in standard location or set java_paths!')
sys.exit(1)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
signed = 0
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
unsigned = os.path.join(output_dir, 'index_unsigned.jar')
if os.path.exists(unsigned):
args = [config['jarsigner'], '-keystore', config['keystore'],
'-storepass:file', config['keystorepassfile'],
'-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
unsigned, config['repo_keyalias']]
if config['keystore'] == 'NONE':
args += config['smartcardoptions']
else: # smardcards never use -keypass
args += ['-keypass:file', config['keypassfile']]
p = FDroidPopen(args)
if p.returncode != 0:
logging.critical("Failed to sign index")
sys.exit(1)
os.rename(unsigned, os.path.join(output_dir, 'index.jar'))
logging.info('Signed index in ' + output_dir)
signed += 1
if signed == 0:
logging.info("Nothing to do")
def browser(self):
import sys, subprocess, os.path, os
if sys.platform == 'darwin':
def openFolder(path):
subprocess.call(['open', path])
elif sys.platform.startswith('linux'):
def openFolder(path):
subprocess.call(['gnome-open', path])
elif sys.platform == 'win32':
def openFolder(path):
subprocess.call(['explorer', path])
app_config = common.read_config()
output_path = common.get_full_path(app_config['OutputPath'])
os.makedirs(output_path, 0o755, True)
openFolder(output_path)
def main():
startuptime = time()
# Crash like hell if the config is crap!
settings = common.read_config('config')
# Kind of like state
message = ''
while True:
result = irc.run(message, settings, startuptime)
message = ''
if result == 'reconnect':
continue
elif result == 'restart':
try:
reload(irc)
except Exception as e:
message = str(e)
continue
else:
break
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options]")
parser.add_option(
"-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal"
)
parser.add_option(
"-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors"
)
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ["repo"]
if config["archive_older"] != 0:
repodirs.append("archive")
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(output_dir, "*.apk"))):
apkfilename = os.path.basename(apkfile)
sigfilename = apkfilename + ".asc"
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
gpgargs = ["gpg", "-a", "--output", sigpath, "--detach-sig"]
if "gpghome" in config:
gpgargs.extend(["--homedir", config["gpghome"]])
gpgargs.append(os.path.join(output_dir, apkfilename))
p = FDroidPopen(gpgargs)
if p.returncode != 0:
logging.error("Signing failed.")
sys.exit(1)
logging.info("Signed " + apkfilename)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options]")
common.setup_global_opts(parser)
options = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
sigfilename = apkfilename + ".asc"
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
gpgargs = ['gpg', '-a',
'--output', sigpath,
'--detach-sig']
if 'gpghome' in config:
gpgargs.extend(['--homedir', config['gpghome']])
if 'gpgkey' in config:
gpgargs.extend(['--local-user', config['gpgkey']])
gpgargs.append(os.path.join(output_dir, apkfilename))
p = FDroidPopen(gpgargs)
if p.returncode != 0:
logging.error("Signing failed.")
sys.exit(1)
logging.info('Signed ' + apkfilename)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options]")
common.setup_global_opts(parser)
options = parser.parse_args()
config = common.read_config(options)
repodirs = ["repo"]
if config["archive_older"] != 0:
repodirs.append("archive")
for output_dir in repodirs:
if not os.path.isdir(output_dir):
logging.error("Missing output directory '" + output_dir + "'")
sys.exit(1)
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(output_dir, "*.apk"))):
apkfilename = os.path.basename(apkfile)
sigfilename = apkfilename + ".asc"
sigpath = os.path.join(output_dir, sigfilename)
if not os.path.exists(sigpath):
gpgargs = ["gpg", "-a", "--output", sigpath, "--detach-sig"]
if "gpghome" in config:
gpgargs.extend(["--homedir", config["gpghome"]])
gpgargs.append(os.path.join(output_dir, apkfilename))
p = FDroidPopen(gpgargs)
if p.returncode != 0:
logging.error("Signing failed.")
sys.exit(1)
logging.info("Signed " + apkfilename)
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=True)
apps = common.read_app_args(args, allapps, False)
for appid, app in apps.iteritems():
logging.info("Writing " + appid)
metadata.write_metadata(os.path.join('metadata', appid) + '.txt', app)
logging.info("Finished.")
def main():
# parse options
(options, args) = parse_options()
if os.isatty(sys.stdin.fileno()):
raise RuntimeError('Need configuration in stdin.')
config = common.read_config(sys.stdin)
conn = common.connect(config.s3)
bucket = None
try:
# setup
real_stdout = sys.stdout
sys.stdout = sys.stderr
# verify all required config items are present
if 'roundtrip' not in config:
raise RuntimeError('roundtrip section not found in config')
for item in ['readers', 'writers', 'duration', 'files', 'bucket']:
if item not in config.roundtrip:
raise RuntimeError(
"Missing roundtrip config item: {item}".format(item=item))
for item in ['num', 'size', 'stddev']:
if item not in config.roundtrip.files:
raise RuntimeError(
"Missing roundtrip config item: files.{item}".format(
item=item))
seeds = dict(config.roundtrip.get('random_seed', {}))
seeds.setdefault('main', random.randrange(2**32))
rand = random.Random(seeds['main'])
for name in ['names', 'contents', 'writer', 'reader']:
seeds.setdefault(name, rand.randrange(2**32))
print 'Using random seeds: {seeds}'.format(seeds=seeds)
# setup bucket and other objects
bucket_name = common.choose_bucket_prefix(config.roundtrip.bucket,
max_len=30)
bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name)
objnames = realistic.names(
mean=15,
stddev=4,
seed=seeds['names'],
)
objnames = itertools.islice(objnames, config.roundtrip.files.num)
objnames = list(objnames)
files = realistic.files(
mean=1024 * config.roundtrip.files.size,
stddev=1024 * config.roundtrip.files.stddev,
seed=seeds['contents'],
)
q = gevent.queue.Queue()
logger_g = gevent.spawn_link_exception(yaml.safe_dump_all,
q,
stream=real_stdout)
print "Writing {num} objects with {w} workers...".format(
num=config.roundtrip.files.num,
w=config.roundtrip.writers,
)
pool = gevent.pool.Pool(size=config.roundtrip.writers)
start = time.time()
for objname in objnames:
fp = next(files)
pool.spawn_link_exception(
writer,
bucket=bucket,
objname=objname,
fp=fp,
queue=q,
)
pool.join()
stop = time.time()
elapsed = stop - start
q.put(
dict(
type='write_done',
duration=int(round(elapsed * NANOSECOND)),
))
print "Reading {num} objects with {w} workers...".format(
num=config.roundtrip.files.num,
w=config.roundtrip.readers,
)
# avoid accessing them in the same order as the writing
rand.shuffle(objnames)
pool = gevent.pool.Pool(size=config.roundtrip.readers)
start = time.time()
for objname in objnames:
pool.spawn_link_exception(
reader,
bucket=bucket,
objname=objname,
queue=q,
)
pool.join()
stop = time.time()
elapsed = stop - start
q.put(
dict(
type='read_done',
duration=int(round(elapsed * NANOSECOND)),
))
q.put(StopIteration)
logger_g.get()
finally:
# cleanup
if options.cleanup:
if bucket is not None:
common.nuke_bucket(bucket)
def main():
# parse options
(options, args) = parse_options()
if os.isatty(sys.stdin.fileno()):
raise RuntimeError('Need configuration in stdin.')
config = common.read_config(sys.stdin)
conn = common.connect(config.s3)
bucket = None
try:
# setup
real_stdout = sys.stdout
sys.stdout = sys.stderr
# verify all required config items are present
if 'readwrite' not in config:
raise RuntimeError('readwrite section not found in config')
for item in ['readers', 'writers', 'duration', 'files', 'bucket']:
if item not in config.readwrite:
raise RuntimeError("Missing readwrite config item: {item}".format(item=item))
for item in ['num', 'size', 'stddev']:
if item not in config.readwrite.files:
raise RuntimeError("Missing readwrite config item: files.{item}".format(item=item))
seeds = dict(config.readwrite.get('random_seed', {}))
seeds.setdefault('main', random.randrange(2**32))
rand = random.Random(seeds['main'])
for name in ['names', 'contents', 'writer', 'reader']:
seeds.setdefault(name, rand.randrange(2**32))
print 'Using random seeds: {seeds}'.format(seeds=seeds)
# setup bucket and other objects
bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30)
bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name)
file_names = realistic.names(
mean=15,
stddev=4,
seed=seeds['names'],
)
file_names = itertools.islice(file_names, config.readwrite.files.num)
file_names = list(file_names)
files = realistic.files2(
mean=1024 * config.readwrite.files.size,
stddev=1024 * config.readwrite.files.stddev,
seed=seeds['contents'],
)
q = gevent.queue.Queue()
# warmup - get initial set of files uploaded
print "Uploading initial set of {num} files".format(num=config.readwrite.files.num)
warmup_pool = gevent.pool.Pool(size=100)
for file_name in file_names:
fp = next(files)
warmup_pool.spawn_link_exception(
write_file,
bucket=bucket,
file_name=file_name,
fp=fp,
)
warmup_pool.join()
# main work
print "Starting main worker loop."
print "Using file size: {size} +- {stddev}".format(size=config.readwrite.files.size, stddev=config.readwrite.files.stddev)
print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers)
group = gevent.pool.Group()
rand_writer = random.Random(seeds['writer'])
for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32))
group.spawn_link_exception(
writer,
bucket=bucket,
worker_id=x,
file_names=file_names,
files=files,
queue=q,
rand=this_rand,
)
rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32))
group.spawn_link_exception(
reader,
bucket=bucket,
worker_id=x,
file_names=file_names,
queue=q,
rand=this_rand,
)
def stop():
group.kill(block=True)
q.put(StopIteration)
gevent.spawn_later(config.readwrite.duration, stop)
yaml.safe_dump_all(q, stream=real_stdout)
finally:
# cleanup
if options.cleanup:
if bucket is not None:
common.nuke_bucket(bucket)
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] "
"[APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
log_dir = 'logs'
if not os.path.isdir(log_dir):
logging.info("Creating log directory")
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("Creating output directory")
os.makedirs(output_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.warning("No unsigned directory - nothing to do")
sys.exit(1)
for f in [
config['keystorepassfile'], config['keystore'],
config['keypassfile']
]:
if not os.path.exists(f):
logging.error("Config error - missing '{0}'".format(f))
sys.exit(1)
# It was suggested at
# https://dev.guardianproject.info/projects/bazaar/wiki/FDroid_Audit
# that a package could be crafted, such that it would use the same signing
# key as an existing app. While it may be theoretically possible for such a
# colliding package ID to be generated, it seems virtually impossible that
# the colliding ID would be something that would be a) a valid package ID,
# and b) a sane-looking ID that would make its way into the repo.
# Nonetheless, to be sure, before publishing we check that there are no
# collisions, and refuse to do any publishing if that's the case...
allapps = metadata.read_metadata()
vercodes = common.read_pkg_args(args, True)
allaliases = []
for appid in allapps:
m = md5.new()
m.update(appid)
keyalias = m.hexdigest()[:8]
if keyalias in allaliases:
logging.error("There is a keyalias collision - publishing halted")
sys.exit(1)
allaliases.append(keyalias)
logging.info("{0} apps, {0} key aliases".format(len(allapps),
len(allaliases)))
# Process any apks that are waiting to be signed...
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
appid, vercode = common.apknameinfo(apkfile)
apkfilename = os.path.basename(apkfile)
if vercodes and appid not in vercodes:
continue
if appid in vercodes and vercodes[appid]:
if vercode not in vercodes[appid]:
continue
logging.info("Processing " + apkfile)
# Figure out the key alias name we'll use. Only the first 8
# characters are significant, so we'll use the first 8 from
# the MD5 of the app's ID and hope there are no collisions.
# If a collision does occur later, we're going to have to
# come up with a new alogrithm, AND rename all existing keys
# in the keystore!
if appid in config['keyaliases']:
# For this particular app, the key alias is overridden...
keyalias = config['keyaliases'][appid]
if keyalias.startswith('@'):
m = md5.new()
m.update(keyalias[1:])
keyalias = m.hexdigest()[:8]
else:
m = md5.new()
m.update(appid)
keyalias = m.hexdigest()[:8]
logging.info("Key alias: " + keyalias)
# See if we already have a key for this application, and
# if not generate one...
p = FDroidPopen([
'keytool', '-list', '-alias', keyalias, '-keystore',
config['keystore'], '-storepass:file', config['keystorepassfile']
])
if p.returncode != 0:
logging.info("Key does not exist - generating...")
p = FDroidPopen([
'keytool', '-genkey', '-keystore', config['keystore'],
'-alias', keyalias, '-keyalg', 'RSA', '-keysize', '2048',
'-validity', '10000', '-storepass:file',
config['keystorepassfile'], '-keypass:file',
config['keypassfile'], '-dname', config['keydname']
])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to generate key")
# Sign the application...
p = FDroidPopen([
'jarsigner', '-keystore', config['keystore'], '-storepass:file',
config['keystorepassfile'], '-keypass:file', config['keypassfile'],
'-sigalg', 'MD5withRSA', '-digestalg', 'SHA1', apkfile, keyalias
])
# TODO keypass should be sent via stdin
if p.returncode != 0:
raise BuildException("Failed to sign application")
# Zipalign it...
p = FDroidPopen([
config['zipalign'], '-v', '4', apkfile,
os.path.join(output_dir, apkfilename)
])
if p.returncode != 0:
raise BuildException("Failed to align application")
os.remove(apkfile)
# Move the source tarball into the output directory...
tarfilename = apkfilename[:-4] + '_src.tar.gz'
tarfile = os.path.join(unsigned_dir, tarfilename)
if os.path.exists(tarfile):
shutil.move(tarfile, os.path.join(output_dir, tarfilename))
logging.info('Published ' + apkfilename)
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'channel_id' not in config:
write_status(output_dir, False, 'No channel token given')
return 1
channel = config['channel_id']
success, msg, client = new_client(config)
if not success:
write_status(output_dir, False, msg)
return 1
if 'deadlines' not in config:
write_status(output_dir, True, 'No deadlines to report')
return 1
deadlines = config['deadlines']
if not isinstance(deadlines, dict):
write_status(output_dir, False, 'Invalid deadlines structure')
return 0
attachments = []
present = datetime.datetime.now()
for (name, info) in deadlines.items():
if 'date' not in info:
write_status(
output_dir, False,
'Date missing in entry {} under {}'.format(info, name))
return 1
date = None
try:
date = datetime.datetime.strptime(info['date'],
'%Y-%m-%d %H:%M:%S')
except Exception as e:
write_status(output_dir, False,
'Could not parse date {}'.format(info['date']))
return 1
diff = date - present
days_left = diff.days
if days_left < 0:
# elapsed, so forget it
continue
alert = days_left <= 7
time_left_msg = '{} days, {:.2f} hours left'.format(
diff.days, diff.seconds / 3600)
fields = [build_field(value=time_left_msg)]
if alert and 'ping' in info:
pings = generate_ping_list(info['ping'])
fields.append(build_field(value='Beware {}!'.format(pings)))
attachments.append(
build_attachment(title=name,
text='Deadline: {}'.format(info['date']),
fields=fields,
color='#fa0000' if alert else '#0fbf24'))
if not attachments:
write_status(output_dir, True, 'All deadlines elapsed')
return 0
success, _, report = post_message(
client, channel,
build_message(text='*Upcoming Deadlines*', attachments=attachments))
write_status(output_dir, success, report)
def main():
# parse options
(options, args) = parse_options()
if os.isatty(sys.stdin.fileno()):
raise RuntimeError('Need configuration in stdin.')
config = common.read_config(sys.stdin)
conn = common.connect(config.s3)
bucket = None
try:
# setup
real_stdout = sys.stdout
sys.stdout = sys.stderr
# verify all required config items are present
if 'readwrite' not in config:
raise RuntimeError('readwrite section not found in config')
for item in ['readers', 'writers', 'duration', 'files', 'bucket']:
if item not in config.readwrite:
raise RuntimeError("Missing readwrite config item: {item}".format(item=item))
for item in ['num', 'size', 'stddev']:
if item not in config.readwrite.files:
raise RuntimeError("Missing readwrite config item: files.{item}".format(item=item))
seeds = dict(config.readwrite.get('random_seed', {}))
seeds.setdefault('main', random.randrange(2**32))
rand = random.Random(seeds['main'])
for name in ['names', 'contents', 'writer', 'reader']:
seeds.setdefault(name, rand.randrange(2**32))
print 'Using random seeds: {seeds}'.format(seeds=seeds)
# setup bucket and other objects
bucket_name = common.choose_bucket_prefix(config.readwrite.bucket, max_len=30)
bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name)
# check flag for deterministic file name creation
if not config.readwrite.get('deterministic_file_names'):
print 'Creating random file names'
file_names = realistic.names(
mean=15,
stddev=4,
seed=seeds['names'],
)
file_names = itertools.islice(file_names, config.readwrite.files.num)
file_names = list(file_names)
else:
print 'Creating file names that are deterministic'
file_names = []
for x in xrange(config.readwrite.files.num):
file_names.append('test_file_{num}'.format(num=x))
files = realistic.files2(
mean=1024 * config.readwrite.files.size,
stddev=1024 * config.readwrite.files.stddev,
seed=seeds['contents'],
)
q = gevent.queue.Queue()
# warmup - get initial set of files uploaded if there are any writers specified
if config.readwrite.writers > 0:
print "Uploading initial set of {num} files".format(num=config.readwrite.files.num)
warmup_pool = gevent.pool.Pool(size=100)
for file_name in file_names:
fp = next(files)
warmup_pool.spawn_link_exception(
write_file,
bucket=bucket,
file_name=file_name,
fp=fp,
)
warmup_pool.join()
# main work
print "Starting main worker loop."
print "Using file size: {size} +- {stddev}".format(size=config.readwrite.files.size, stddev=config.readwrite.files.stddev)
print "Spawning {w} writers and {r} readers...".format(w=config.readwrite.writers, r=config.readwrite.readers)
group = gevent.pool.Group()
rand_writer = random.Random(seeds['writer'])
# Don't create random files if deterministic_files_names is set and true
if not config.readwrite.get('deterministic_file_names'):
for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32))
group.spawn_link_exception(
writer,
bucket=bucket,
worker_id=x,
file_names=file_names,
files=files,
queue=q,
rand=this_rand,
)
# Since the loop generating readers already uses config.readwrite.readers
# and the file names are already generated (randomly or deterministically),
# this loop needs no additional qualifiers. If zero readers are specified,
# it will behave as expected (no data is read)
rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32))
group.spawn_link_exception(
reader,
bucket=bucket,
worker_id=x,
file_names=file_names,
queue=q,
rand=this_rand,
)
def stop():
group.kill(block=True)
q.put(StopIteration)
gevent.spawn_later(config.readwrite.duration, stop)
# wait for all the tests to finish
group.join()
print 'post-join, queue size {size}'.format(size=q.qsize())
if q.qsize() > 0:
for temp_dict in q:
if 'error' in temp_dict:
raise Exception('exception:\n\t{msg}\n\t{trace}'.format(
msg=temp_dict['error']['msg'],
trace=temp_dict['error']['traceback'])
)
else:
yaml.safe_dump(temp_dict, stream=real_stdout)
finally:
# cleanup
if options.cleanup:
if bucket is not None:
common.nuke_bucket(bucket)
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-d", "--distinguished-name", default=None,
help="X.509 'Distiguished Name' used when generating keys")
parser.add_option("--keystore", default=None,
help="Path to the keystore for the repo signing key")
parser.add_option("--repo-keyalias", default=None,
help="Alias of the repo signing key in the keystore")
(options, args) = parser.parse_args()
# find root install prefix
tmp = os.path.dirname(sys.argv[0])
if os.path.basename(tmp) == 'bin':
prefix = os.path.dirname(tmp)
examplesdir = prefix + '/share/doc/fdroidserver/examples'
else:
# we're running straight out of the git repo
prefix = tmp
examplesdir = prefix
fdroiddir = os.getcwd()
if not os.path.exists('config.py') and not os.path.exists('repo'):
# 'metadata' and 'tmp' are created in fdroid
os.mkdir('repo')
shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
shutil.copyfile(os.path.join(examplesdir, 'config.sample.py'), 'config.py')
os.chmod('config.py', 0o0600)
else:
print('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
sys.exit()
# now that we have a local config.py, read configuration...
config = common.read_config(options)
# track down where the Android SDK is
if os.path.isdir(config['sdk_path']):
print('Using "' + config['sdk_path'] + '" for the Android SDK')
sdk_path = config['sdk_path']
elif 'ANDROID_HOME' in os.environ.keys():
sdk_path = os.environ['ANDROID_HOME']
else:
default_sdk_path = '/opt/android-sdk'
while True:
s = raw_input('Enter the path to the Android SDK (' + default_sdk_path + '): ')
if re.match('^\s*$', s) != None:
sdk_path = default_sdk_path
else:
sdk_path = s
if os.path.isdir(os.path.join(sdk_path, 'build-tools')):
break
else:
print('"' + s + '" does not contain the Android SDK! Try again...')
if os.path.isdir(sdk_path):
write_to_config('sdk_path', sdk_path)
# try to find a working aapt, in all the recent possible paths
build_tools = os.path.join(sdk_path, 'build-tools')
aaptdirs = []
aaptdirs.append(os.path.join(build_tools, config['build_tools']))
aaptdirs.append(build_tools)
for f in sorted(os.listdir(build_tools), reverse=True):
if os.path.isdir(os.path.join(build_tools, f)):
aaptdirs.append(os.path.join(build_tools, f))
for d in aaptdirs:
if os.path.isfile(os.path.join(d, 'aapt')):
aapt = os.path.join(d, 'aapt')
break
if os.path.isfile(aapt):
dirname = os.path.basename(os.path.dirname(aapt))
if dirname == 'build-tools':
# this is the old layout, before versioned build-tools
write_to_config('build_tools', '')
else:
write_to_config('build_tools', dirname)
# track down where the Android NDK is
ndk_path = '/opt/android-ndk'
if os.path.isdir(config['ndk_path']):
ndk_path = config['ndk_path']
elif 'ANDROID_NDK' in os.environ.keys():
print('using ANDROID_NDK')
ndk_path = os.environ['ANDROID_NDK']
if os.path.isdir(ndk_path):
write_to_config('ndk_path', ndk_path)
# the NDK is optional so we don't prompt the user for it if its not found
# find or generate the keystore for the repo signing key. First try the
# path written in the default config.py. Then check if the user has
# specified a path from the command line, which will trump all others.
# Otherwise, create ~/.local/share/fdroidserver and stick it in there.
keystore = config['keystore']
if options.keystore:
if os.path.isfile(options.keystore):
keystore = options.keystore
write_to_config('keystore', keystore)
else:
print('"' + options.keystore + '" does not exist or is not a file!')
sys.exit(1)
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
write_to_config('repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
write_to_config('keydname', keydname)
if not os.path.isfile(keystore):
# no existing or specified keystore, generate the whole thing
keystoredir = os.path.join(os.getenv('HOME'),
'.local', 'share', 'fdroidserver')
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
keystore = os.path.join(keystoredir, 'keystore.jks')
write_to_config('keystore', keystore)
password = genpassword()
write_to_config('keystorepass', password)
write_to_config('keypass', password)
if not options.repo_keyalias:
repo_keyalias = socket.getfqdn()
write_to_config('repo_keyalias', repo_keyalias)
if not options.distinguished_name:
keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
write_to_config('keydname', keydname)
genkey(keystore, repo_keyalias, password, keydname)
print('Built repo based in "' + fdroiddir + '"')
print('with this config:')
print(' Android SDK:\t\t\t' + sdk_path)
print(' Android SDK Build Tools:\t' + os.path.dirname(aapt))
print(' Android NDK (optional):\t' + ndk_path)
print(' Keystore for signing key:\t' + keystore)
print('\nTo complete the setup, add your APKs to "' +
os.path.join(fdroiddir, 'repo') + '"' +
'''
then run "fdroid update -c; fdroid update". You might also want to edit
"config.py" to set the URL, repo name, and more. You should also set up
a signing key.
For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository
and https://f-droid.org/manual/fdroid.html#Signing
''')
def main():
global options, config
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.error("No unsigned directory - nothing to do")
sys.exit(0)
verified = 0
notverified = 0
vercodes = common.read_pkg_args(args, True)
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
appid, vercode = common.apknameinfo(apkfile)
if vercodes and appid not in vercodes:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
try:
logging.info("Processing " + apkfilename)
remoteapk = os.path.join(tmp_dir, apkfilename)
if os.path.exists(remoteapk):
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
logging.info("...retrieving " + url)
common.download_file(url, dldir=tmp_dir)
compare_result = common.compare_apks(
os.path.join(unsigned_dir, apkfilename), remoteapk, tmp_dir)
if compare_result:
raise FDroidException(compare_result)
logging.info("...successfully verified")
verified += 1
except FDroidException, e:
logging.info("...NOT verified - {0}".format(e))
notverified += 1
def main():
global config, options, curid, count
curid = None
count = Counter()
def warn(message):
global curid, count
if curid:
print "%s:" % curid
curid = None
count['app'] += 1
print ' %s' % message
count['warn'] += 1
def pwarn(message):
if options.pedantic:
warn(message)
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option(
"-p",
"--pedantic",
action="store_true",
default=False,
help="Show pedantic warnings that might give false positives")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=False)
apps = common.read_app_args(args, allapps, False)
for appid, app in apps.iteritems():
curid = appid
lastcommit = ''
if app['Disabled']:
continue
for build in app['builds']:
if build['commit'] and not build['disable']:
lastcommit = build['commit']
# Potentially incorrect UCM
if (app['Update Check Mode'] == 'RepoManifest'
and any(s in lastcommit for s in '.,_-/')):
pwarn(
"Last used commit '%s' looks like a tag, but Update Check Mode is '%s'"
% (lastcommit, app['Update Check Mode']))
# Summary size limit
summ_chars = len(app['Summary'])
if summ_chars > config['char_limits']['Summary']:
warn("Summary of length %s is over the %i char limit" %
(summ_chars, config['char_limits']['Summary']))
# Redundant info
if app['Web Site'] and app['Source Code']:
if app['Web Site'].lower() == app['Source Code'].lower():
warn("Website '%s' is just the app's source code link" %
app['Web Site'])
app['Web Site'] = ''
name = app['Name'] or app['Auto Name']
if app['Summary'] and name:
if app['Summary'].lower() == name.lower():
warn("Summary '%s' is just the app's name" % app['Summary'])
if app['Summary'] and app['Description']:
if app['Summary'].lower() == app['Description'][0].lower():
warn("Description '%s' is just the app's summary" %
app['Summary'])
# Description size limit
desc_chars = sum(len(l) for l in app['Description'])
if desc_chars > config['char_limits']['Description']:
warn("Description of length %s is over the %i char limit" %
(desc_chars, config['char_limits']['Description']))
# Regex checks in all kinds of fields
for f in regex_warnings:
for m, r in regex_warnings[f]:
t = metadata.metafieldtype(f)
if t == 'string':
if m.match(app[f]):
warn("%s '%s': %s" % (f, app[f], r))
elif t == 'multiline':
for l in app[f]:
if m.match(l):
warn("%s at line '%s': %s" % (f, l, r))
# Regex pedantic checks in all kinds of fields
if options.pedantic:
for f in regex_pedantic:
for m, r in regex_pedantic[f]:
if m.match(app[f]):
warn("%s '%s': %s" % (f, app[f], r))
# Build warnings
for build in app['builds']:
if build['disable']:
continue
for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
if build['commit'] and build['commit'].startswith(s):
warn("Branch '%s' used as commit in build '%s'" %
(s, build['version']))
for srclib in build['srclibs']:
ref = srclib.split('@')[1].split('/')[0]
if ref.startswith(s):
warn("Branch '%s' used as commit in srclib '%s'" %
(s, srclib))
for s in [
'git clone', 'git svn clone', 'svn checkout', 'svn co',
'hg clone'
]:
for flag in ['init', 'prebuild', 'build']:
if not build[flag]:
continue
if s in build[flag]:
# TODO: This should not be pedantic!
pwarn("'%s' used in %s '%s'" % (s, flag, build[flag]))
if not curid:
print
logging.info("Found a total of %i warnings in %i apps." %
(count['warn'], count['app']))
def main():
# parse options
(options, args) = parse_options()
if os.isatty(sys.stdin.fileno()):
raise RuntimeError('Need configuration in stdin.')
config = common.read_config(sys.stdin)
conn = common.connect(config.s3)
bucket = None
try:
# setup
real_stdout = sys.stdout
sys.stdout = sys.stderr
# verify all required config items are present
if 'readwrite' not in config:
raise RuntimeError('readwrite section not found in config')
for item in ['readers', 'writers', 'duration', 'files', 'bucket']:
if item not in config.readwrite:
raise RuntimeError(
"Missing readwrite config item: {item}".format(item=item))
for item in ['num', 'size', 'stddev']:
if item not in config.readwrite.files:
raise RuntimeError(
"Missing readwrite config item: files.{item}".format(
item=item))
seeds = dict(config.readwrite.get('random_seed', {}))
seeds.setdefault('main', random.randrange(2**32))
rand = random.Random(seeds['main'])
for name in ['names', 'contents', 'writer', 'reader']:
seeds.setdefault(name, rand.randrange(2**32))
print 'Using random seeds: {seeds}'.format(seeds=seeds)
# setup bucket and other objects
bucket_name = common.choose_bucket_prefix(config.readwrite.bucket,
max_len=30)
bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name)
# check flag for deterministic file name creation
if not config.readwrite.get('deterministic_file_names'):
print 'Creating random file names'
file_names = realistic.names(
mean=15,
stddev=4,
seed=seeds['names'],
)
file_names = itertools.islice(file_names,
config.readwrite.files.num)
file_names = list(file_names)
else:
print 'Creating file names that are deterministic'
file_names = []
for x in xrange(config.readwrite.files.num):
file_names.append('test_file_{num}'.format(num=x))
files = realistic.files2(
mean=1024 * config.readwrite.files.size,
stddev=1024 * config.readwrite.files.stddev,
seed=seeds['contents'],
)
q = gevent.queue.Queue()
# warmup - get initial set of files uploaded if there are any writers specified
if config.readwrite.writers > 0:
print "Uploading initial set of {num} files".format(
num=config.readwrite.files.num)
warmup_pool = gevent.pool.Pool(size=100)
for file_name in file_names:
fp = next(files)
warmup_pool.spawn_link_exception(
write_file,
bucket=bucket,
file_name=file_name,
fp=fp,
)
warmup_pool.join()
# main work
print "Starting main worker loop."
print "Using file size: {size} +- {stddev}".format(
size=config.readwrite.files.size,
stddev=config.readwrite.files.stddev)
print "Spawning {w} writers and {r} readers...".format(
w=config.readwrite.writers, r=config.readwrite.readers)
group = gevent.pool.Group()
rand_writer = random.Random(seeds['writer'])
# Don't create random files if deterministic_files_names is set and true
if not config.readwrite.get('deterministic_file_names'):
for x in xrange(config.readwrite.writers):
this_rand = random.Random(rand_writer.randrange(2**32))
group.spawn_link_exception(
writer,
bucket=bucket,
worker_id=x,
file_names=file_names,
files=files,
queue=q,
rand=this_rand,
)
# Since the loop generating readers already uses config.readwrite.readers
# and the file names are already generated (randomly or deterministically),
# this loop needs no additional qualifiers. If zero readers are specified,
# it will behave as expected (no data is read)
rand_reader = random.Random(seeds['reader'])
for x in xrange(config.readwrite.readers):
this_rand = random.Random(rand_reader.randrange(2**32))
group.spawn_link_exception(
reader,
bucket=bucket,
worker_id=x,
file_names=file_names,
queue=q,
rand=this_rand,
)
def stop():
group.kill(block=True)
q.put(StopIteration)
gevent.spawn_later(config.readwrite.duration, stop)
# wait for all the tests to finish
group.join()
print 'post-join, queue size {size}'.format(size=q.qsize())
if q.qsize() > 0:
for temp_dict in q:
if 'error' in temp_dict:
raise Exception('exception:\n\t{msg}\n\t{trace}'.format(
msg=temp_dict['error']['msg'],
trace=temp_dict['error']['traceback']))
else:
yaml.safe_dump(temp_dict, stream=real_stdout)
finally:
# cleanup
if options.cleanup:
if bucket is not None:
common.nuke_bucket(bucket)
def main(config_dir, home_dir, output_dir):
info = DashboardInfo(home_dir)
conf = read_config(config_dir)
# delete old report so it doesn't hang around if we exit
# without a new one
if check_file_exists(output_dir, 'report.json'):
subprocess.call(['rm', '-f', os.path.join(output_dir, 'report.json')])
time_window = -1
if 'time_window' in conf:
time_window = int(conf['time_window'])
pings = conf['notify'] if 'notify' in conf else []
# map: exp -> [(fields w/ high SD, historic mean, SD, current)]
exp_alerts = {}
for exp in info.all_present_experiments():
if not info.exp_active(exp):
continue
# not this subsystem's job to report on failures
stage_statuses = info.exp_stage_statuses(exp)
if 'run' not in stage_statuses or 'analysis' not in stage_statuses:
continue
if not stage_statuses['analysis']['success']:
continue
all_data = sort_data(info.exp_data_dir(exp))
if len(all_data) <= 1:
continue
exp_alerts[exp] = []
most_recent = all_data[-1]
past_data = all_data[:-1]
if time_window >= 1:
past_data = [
entry for entry in past_data
if time_difference(most_recent, entry).days <= time_window
]
field_values = traverse_fields(most_recent)
for fields in itertools.product(*field_values):
current_stat, _ = gather_stats([most_recent], fields)
current = current_stat[0]
past_stats, _ = gather_stats(past_data, fields)
past_sd = np.std(past_stats)
past_mean = np.mean(past_stats)
if abs(current - past_mean) > past_sd:
exp_alerts[exp].append((fields, past_mean, past_sd, current))
if not exp_alerts[exp]:
del exp_alerts[exp]
if exp_alerts:
report = {
'title': 'High SD Alerts',
'value': format_report(info, exp_alerts, pings)
}
write_json(output_dir, 'report.json', report)
write_status(output_dir, True, 'success')
def main():
global config, options, curid, count
curid = None
count = Counter()
def warn(message):
global curid, count
if curid:
print "%s:" % curid
curid = None
count['app'] += 1
print ' %s' % message
count['warn'] += 1
def pwarn(message):
if options.pedantic:
warn(message)
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-p", "--pedantic", action="store_true", default=False,
help="Show pedantic warnings that might give false positives")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=False)
apps = common.read_app_args(args, allapps, False)
for appid, app in apps.iteritems():
if app['Disabled']:
continue
curid = appid
count['app_total'] += 1
curbuild = None
for build in app['builds']:
if not curbuild or int(build['vercode']) > int(curbuild['vercode']):
curbuild = build
# Potentially incorrect UCM
if (curbuild and curbuild['commit']
and app['Update Check Mode'] == 'RepoManifest' and
any(s in curbuild['commit'] for s in '.,_-/')):
pwarn("Last used commit '%s' looks like a tag, but Update Check Mode is '%s'" % (
curbuild['commit'], app['Update Check Mode']))
# Dangerous auto updates
if curbuild and app['Auto Update Mode'] != 'None':
for flag in ['target', 'srclibs', 'scanignore']:
if curbuild[flag]:
pwarn("Auto Update Mode is enabled but '%s' is manually set at '%s'" % (flag, curbuild[flag]))
# Summary size limit
summ_chars = len(app['Summary'])
if summ_chars > config['char_limits']['Summary']:
warn("Summary of length %s is over the %i char limit" % (
summ_chars, config['char_limits']['Summary']))
# Redundant info
if app['Web Site'] and app['Source Code']:
if app['Web Site'].lower() == app['Source Code'].lower():
warn("Website '%s' is just the app's source code link" % app['Web Site'])
app['Web Site'] = ''
name = app['Name'] or app['Auto Name']
if app['Summary'] and name:
if app['Summary'].lower() == name.lower():
warn("Summary '%s' is just the app's name" % app['Summary'])
if app['Summary'] and app['Description'] and len(app['Description']) == 1:
if app['Summary'].lower() == app['Description'][0].lower():
warn("Description '%s' is just the app's summary" % app['Summary'])
# Description size limit
desc_chars = sum(len(l) for l in app['Description'])
if desc_chars > config['char_limits']['Description']:
warn("Description of length %s is over the %i char limit" % (
desc_chars, config['char_limits']['Description']))
# Regex checks in all kinds of fields
for f in regex_warnings:
for m, r in regex_warnings[f]:
t = metadata.metafieldtype(f)
if t == 'string':
if m.match(app[f]):
warn("%s '%s': %s" % (f, app[f], r))
elif t == 'multiline':
for l in app[f]:
if m.match(l):
warn("%s at line '%s': %s" % (f, l, r))
# Regex pedantic checks in all kinds of fields
if options.pedantic:
for f in regex_pedantic:
for m, r in regex_pedantic[f]:
if m.match(app[f]):
warn("%s '%s': %s" % (f, app[f], r))
# Build warnings
for build in app['builds']:
if build['disable']:
continue
for s in ['master', 'origin', 'HEAD', 'default', 'trunk']:
if build['commit'] and build['commit'].startswith(s):
warn("Branch '%s' used as commit in build '%s'" % (
s, build['version']))
for srclib in build['srclibs']:
ref = srclib.split('@')[1].split('/')[0]
if ref.startswith(s):
warn("Branch '%s' used as commit in srclib '%s'" % (
s, srclib))
for s in ['git clone', 'git svn clone', 'svn checkout', 'svn co', 'hg clone']:
for flag in ['init', 'prebuild', 'build']:
if not build[flag]:
continue
if s in build[flag]:
# TODO: This should not be pedantic!
pwarn("'%s' used in %s '%s'" % (s, flag, build[flag]))
if not curid:
print
logging.info("Found a total of %i warnings in %i apps out of %i total." % (
count['warn'], count['app'], count['app_total']))
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("-r", "--repo", default=None,
help="Allows a different repo to be specified for a multi-repo google code project")
parser.add_option("--rev", default=None,
help="Allows a different revision (or git branch) to be specified for the initial import")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/';
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, None)
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "WARNING: Couldn't find latest version name"
if not vercode:
print "WARNING: Couldn't find latest version code"
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {'orientation': 'landscape', 'icon': '',
'permissions': '', 'android.api': "18"}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get('app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
print "No android or kivy project could be found. Specify --subdir?"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version if version else '?'
build['vercode'] = vercode if vercode else '?'
build['commit'] = '?'
build['disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
print "Wrote " + metafile
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'webhook_url' not in config:
write_status(output_dir, False, 'No webhook URL given')
return 1
webhook = config['webhook_url']
info = DashboardInfo(home_dir)
failed_subsys = []
reports = []
failed_reports = []
for subsys in info.all_present_subsystems():
# ignore self
if subsys == 'subsys_reporter':
continue
if not info.subsys_active(subsys):
continue
status = info.subsys_stage_status(subsys, 'run')
if not status['success']:
failed_subsys.append(failed_subsys_field(subsys, status))
continue
report_present = check_file_exists(info.subsys_output_dir(subsys), 'report.json')
if not report_present:
continue
try:
report = read_json(info.subsys_output_dir(subsys), 'report.json')
reports.append(build_field(
title=report['title'],
value=report['value']))
except Exception:
failed_reports.append(subsys_name)
attachments = []
if reports:
attachments.append(build_attachment(
title='Reports',
fields=reports))
if failed_reports or failed_subsys:
failure_text = ''
if failed_reports:
failure_text = 'Failed to parse reports: {}'.format(', '.join(failed_reports))
attachments.append(build_attachment(
title='Errors',
text=failure_text,
color='#fa0000',
fields=failed_subsys))
if not attachments:
write_status(output_dir, True, 'Nothing to report')
return 0
success, msg = post_message(
webhook,
build_message(
text='Subsystem Results',
attachments=attachments))
write_status(output_dir, success, msg)
def main():
global config, options
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help="app-id to check for updates")
parser.add_argument("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_argument("--autoonly", action="store_true", default=False,
help="Only process apps with auto-updates")
parser.add_argument("--commit", action="store_true", default=False,
help="Commit changes")
parser.add_argument("--gplay", action="store_true", default=False,
help="Only print differences with the Play Store")
options = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata()
apps = common.read_app_args(options.appid, allapps, False)
if options.gplay:
for app in apps:
version, reason = check_gplay(app)
if version is None:
if reason == '404':
logging.info("{0} is not in the Play Store".format(common.getappname(app)))
else:
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
if version is not None:
stored = app.CurrentVersion
if not stored:
logging.info("{0} has no Current Version but has version {1} on the Play Store"
.format(common.getappname(app), version))
elif LooseVersion(stored) < LooseVersion(version):
logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
.format(common.getappname(app), version, stored))
else:
if stored != version:
logging.info("{0} has version {1} on the Play Store, which differs from {2}"
.format(common.getappname(app), version, stored))
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
return
for appid, app in apps.iteritems():
if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
logging.debug("Nothing to do for {0}...".format(appid))
continue
logging.info("Processing " + appid + '...')
checkupdates_app(app)
logging.info("Finished.")
def main():
global options, config
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-a",
"--all",
action="store_true",
default=False,
help="Install all signed applications available")
(options, args) = parser.parse_args()
if not args and not options.all:
raise OptionError(
"If you really want to install all the signed apps, use --all",
"all")
config = common.read_config(options)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("No signed output directory - nothing to do")
sys.exit(0)
if args:
vercodes = common.read_pkg_args(args, True)
apks = {appid: None for appid in vercodes}
# Get the signed apk with the highest vercode
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
try:
appid, vercode = common.apknameinfo(apkfile)
except FDroidException:
continue
if appid not in apks:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
apks[appid] = apkfile
for appid, apk in apks.iteritems():
if not apk:
raise FDroidException("No signed apk available for %s" % appid)
else:
apks = {
common.apknameinfo(apkfile)[0]: apkfile
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk')))
}
for appid, apk in apks.iteritems():
# Get device list each time to avoid device not found errors
devs = devices()
if not devs:
raise FDroidException("No attached devices found")
logging.info("Installing %s..." % apk)
for dev in devs:
logging.info("Installing %s on %s..." % (apk, dev))
p = SdkToolsPopen(['adb', "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
fail = line[9:-1]
if not fail:
continue
if fail == "INSTALL_FAILED_ALREADY_EXISTS":
logging.warn("%s is already installed on %s." % (apk, dev))
else:
raise FDroidException("Failed to install %s on %s: %s" %
(apk, dev, fail))
logging.info("\nFinished")
def main():
global config, options
# Parse command line...
parser = OptionParser(
usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Read all app and srclib metadata
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, True)
problems = []
build_dir = 'build'
if not os.path.isdir(build_dir):
logging.info("Creating build directory")
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
for appid, app in apps.iteritems():
if app['Disabled']:
logging.info("Skipping %s: disabled" % appid)
continue
if not app['builds']:
logging.info("Skipping %s: no builds specified" % appid)
continue
logging.info("Processing " + appid)
try:
build_dir = 'build/' + appid
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
for thisbuild in app['builds']:
if thisbuild['disable']:
logging.info(
"...skipping version %s - %s" %
(thisbuild['version'],
thisbuild.get('disable', thisbuild['commit'][1:])))
else:
logging.info("...scanning version " + thisbuild['version'])
# Prepare the source code...
root_dir, _ = common.prepare_source(
vcs, app, thisbuild, build_dir, srclib_dir, extlib_dir,
False)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir,
thisbuild)
for problem in buildprobs:
problems.append(problem + ' in ' + appid + ' ' +
thisbuild['version'])
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (appid,
be)
problems.append(msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (appid, vcse)
problems.append(msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (
appid, traceback.format_exc())
problems.append(msg)
logging.info("Finished:")
for problem in problems:
print problem
print str(len(problems)) + ' problems.'
def _check_writeup(writeup_file):
writeup = writeup_file.read()
# Check if writeup.md starts with the challenge name in H1 style.
config = read_config('config.yml')
config_challenge_name = config['name']
h1_pattern = '^%s\n={1,}\n\n' % re.escape(config_challenge_name)
try:
writeup_challenge_name = re.search('^.*\n', writeup).group()[:-1]
assert writeup_challenge_name == config_challenge_name
re.search(h1_pattern, writeup).group()
except AssertionError:
logging.warning(
'The challenge name in writeup.md (%s) and config.yml (%s) should be the same.'
% (writeup_challenge_name, config_challenge_name))
except Exception:
logging.warning(
'The challenge names in writeup.md should be in H1 style. For example:\n\n'
'Challenge name\n==============\n')
# Check if costs and H2 sections are correct in writeup.md
cost_pattern = '\nCost: [0-9]{1,2}%\n'
costs = [
int(re.search(r'[0-9]{1,2}', cost).group())
for cost in re.findall(cost_pattern, writeup)
]
if sum(costs) != COST_SUM:
counted_error(
'The sum of costs in writeup.md should be %d%%.\n\tPlease make sure you have the following '
'format (take care of the white spaces, starting and ending newlines):\n%s'
% (COST_SUM, cost_pattern))
h2_pattern = r'\n## [A-Z].{5,150}\n'
h2 = re.findall(h2_pattern, writeup)
if len(h2) < MIN_WRITEUP_SECTIONS:
counted_error('There should be at least %d sections in writeup.md' %
MIN_WRITEUP_SECTIONS)
h2_last = h2.pop()
if h2_last != '\n## Complete solution\n':
counted_error(
'The last section should be called "Complete solution" in writeup.md'
)
h2costs = [
re.search(h2_pattern, h2cost).group()
for h2cost in re.findall(h2_pattern + cost_pattern, writeup)
]
missing_costs = set(h2) - set(h2costs)
if len(missing_costs) > 0:
counted_error(
'No cost is defined in writeup.md for section(s): %s\n\tPlease make sure you have the following '
'format (take care of the starting and ending newlines):\n%s' %
(missing_costs, cost_pattern))
reference_style_links_pattern = r'(\[.*?\]\[.*?\])'
if re.search(reference_style_links_pattern, writeup) is not None:
logging.warning(
'The writeup contains reference style links like [this one][0], which might render incorrectly.\n\t'
'Please use inline style ones like [this](http://example.net/)')
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'channel_id' not in config:
write_status(output_dir, False, 'No channel token given')
return 1
success, msg, client = new_client(config)
info = DashboardInfo(home_dir)
if not success:
write_status(output_dir, False, msg)
return 1
slack_channel = config['channel_id']
description = ''
if 'description' in config:
description = config['description']
info = DashboardInfo(home_dir)
inactive_experiments = [] # list of titles
failed_experiments = [] # list of slack fields
successful_experiments = [] # list of slack fields
failed_graphs = [] # list of titles
for exp_name in info.all_present_experiments():
stage_statuses = info.exp_stage_statuses(exp_name)
if not stage_statuses['precheck']['success']:
failed_experiments.append(
failed_experiment_field(exp_name, stage_statuses, 'precheck'))
continue
exp_conf = info.read_exp_config(exp_name)
exp_status = info.exp_status_dir(exp_name)
run_status = validate_json(exp_status,
'time_delta',
filename='run.json')
exp_title = exp_name if 'title' not in exp_conf else exp_conf['title']
notify = exp_conf['notify']
if not exp_conf['active']:
inactive_experiments.append(exp_title)
continue
failure = False
for stage in ['setup', 'run', 'analysis', 'summary']:
if stage not in stage_statuses:
# setup is the only stage that's optional
assert stage == 'setup'
continue
if not stage_statuses[stage]['success']:
failed_experiments.append(
failed_experiment_field(
exp_title,
stage_statuses,
stage,
duration=run_status.get('time_delta'),
notify=notify))
failure = True
break
if failure:
continue
# failure to visualize is not as big a deal as failing to
# run or analyze the experiment, so we only report it but
# don't fail to report the summary
if not stage_statuses['visualization']['success']:
failed_graphs.append(exp_title)
summary = info.read_exp_summary(exp_name)
successful_experiments.append(
build_field(
summary['title'],
attach_duration(summary['value'],
run_status.get('time_delta'))))
# produce messages
attachments = []
if successful_experiments:
attachments.append(
build_attachment(title='Successful benchmarks',
fields=successful_experiments))
if failed_experiments:
attachments.append(
build_attachment(color='#fa0000',
title='Failed benchmarks',
fields=failed_experiments))
if inactive_experiments:
attachments.append(
build_attachment(color='#616161',
title='Inactive benchmarks',
text=', '.join(inactive_experiments)))
if failed_graphs:
attachments.append(
build_attachment(color='#fa0000',
title='Failed to Visualize',
text=', '.join(failed_graphs)))
success, _, report = post_message(
client, slack_channel,
build_message(text='*Dashboard Results*{}'.format(
'\n' + description if description != '' else ''),
attachments=attachments))
if config.get('report_images', False):
success, msg = send_graphs(config, info, client, output_dir)
if not success:
write_status(output_dir, False, msg)
return 1
write_status(output_dir, success, report)
def check_config(config: dict, is_static):
invalid_keys = set(config.keys()) - set(CONFIG_KEYS)
if len(invalid_keys) > 0:
counted_error('Invalid key(s) found in config.yml: %s' % invalid_keys)
if config['version'][:1] != 'v':
counted_error(
'Invalid version. The version number must start with the letter v')
elif config['version'] == 'v1':
counted_error('This version is deprecated, please use v2.0.0')
elif config['version'] != 'v2.0.0':
counted_error(
'Invalid version. The supplied config version is not supported')
# Difficulty
try:
assert DIFFICULTY_RANGE['min'] <= int(
config['difficulty']) <= DIFFICULTY_RANGE['max']
except Exception:
counted_error('Invalid difficulty in config.yml. '
'Valid values: %d - %d' %
(DIFFICULTY_RANGE['min'], DIFFICULTY_RANGE['max']))
# Name
try:
assert NAME_RANGE['min'] <= len(config['name']) <= NAME_RANGE['max']
except Exception:
counted_error('Invalid challenge name in config.yml. '
'Name should be a string between %d - %d characters.' %
(NAME_RANGE['min'], NAME_RANGE['max']))
for template_config in glob(
os.path.join(TOOLBOX_PATH, 'templates', '*', 'config.yml')):
if config['name'] == read_config(template_config).get('name'):
counted_error('Please, set the challenge name in the config file.')
# Skills
if not isinstance(config['skills'], list):
counted_error(
'Invalid skills in config.yml. Skills should be placed into a list.\n'
'\tValid skills are listed here: \n'
'\thttps://platform.avatao.com/api-explorer/#/api/core/skills/')
# Recommendations
if not isinstance(config['recommendations'], dict):
counted_error(
'Invalid recommendations in config.yml. Recommendations should be added in the following '
'format:\n\n'
'recommendations:\n'
'\twww.example.com: \'Example webpage\'\n'
'\thttp://www.example2.com: \'Example2 webpage\''
'\thttp://example3.com: \'Example3 webpage\'')
url_re = re.compile(
r'(?i)\b((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)'
r'(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]'
r'+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'".,<>?«»“”‘’]))')
for item in config['recommendations'].items():
if url_re.fullmatch(item[0]) is None:
counted_error('Invalid recommended URL (%s) found in config.yml' %
item[0])
if not isinstance(item[1], str):
counted_error(
'The name of recommended url (%s) should be a string in config.yml'
% item[1])
# Owners
if not isinstance(config['owners'], list):
counted_error(
'Challenge owners (%s) should be placed into a list in config.yml'
% config['owners'])
email_re = re.compile(
r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
for owner in config.get('owners', []):
if email_re.fullmatch(owner) is None:
counted_error(
'Invalid owner email (%s) found in config.yml. '
'Make sure you list the email addresses of the owners.' %
owner)
if not is_static:
controller_found = False
for item in config['crp_config'].values():
if not isinstance(item, dict):
counted_error('Items of crp_config must be dictionaries.')
if 'image' in item and item['image'].find('/') < 0:
counted_error(
'If the image is explicitly defined, it must be relative '
'to the registry - e.g. challenge:solvable.')
if 'capabilities' in item:
invalid_caps = set(item['capabilities']) - CAPABILITIES
if len(invalid_caps) > 0:
counted_error(
'Forbidden capabilities: %s\n\tAllowed capabilities: %s',
invalid_caps, CAPABILITIES)
if 'kernel_params' in item:
invalid_parameters = set(
item['kernel_params']) - KERNEL_PARAMETERS
if len(invalid_parameters) > 0:
counted_error(
'Forbidden kernel parameters: %s\n\tAllowed parameters: %s',
invalid_parameters, KERNEL_PARAMETERS)
if 'mem_limit' in item:
if not isinstance(item['mem_limit'], str):
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string like: 100M'
)
if item['mem_limit'][-1] not in "M":
counted_error(
'Invalid mem_limit value: %s, The mem_limit should be a string ending with '
'M (megabytes). No other unit is allowed.')
try:
mem_limit_number_part = int(item['mem_limit'][:-1])
if mem_limit_number_part > 999:
counted_error(
'Invalid mem_limit value: %s, The mem_limit can not be greater than 999M.'
)
except Exception:
counted_error(
'Invalid mem_limit value: %s, mem_limit must start with a number and end with '
'M (megabytes). No other unit is allowed.')
for port in item.get('ports', []):
try:
port, protocol = port.split('/', 1)
try:
port = int(port)
except Exception:
counted_error(
'Invalid port. The port should be a number between 1 and 65535.'
)
if PORT_RANGE['min'] > port or PORT_RANGE['max'] < port:
counted_error(
'Invalid port. The port should be a number between 1 and 65535'
)
if protocol not in PROTOCOLS:
counted_error(
'Invalid protocol in config.yml (crp_config): %s. Valid values: %s',
protocol, PROTOCOLS)
elif protocol == CONTROLLER_PROTOCOL:
controller_found = True
except Exception:
counted_error('Invalid port format. [port/protocol]')
if not config.get('flag') and not controller_found:
counted_error(
'Missing controller port [5555/%s] for a dynamic challenge.' %
CONTROLLER_PROTOCOL)
if str(config.get('enable_flag_input')).lower() not in ('true', 'false',
'1', '0'):
counted_error('Invalid enable_flag_input. Should be a boolean.')
if is_static:
try:
assert isinstance(config['flag'], str)
except AssertionError:
counted_error('Invalid flag. Should be a string.')
except KeyError:
counted_error(
'Missing flag. Static challenges must have a static flag set.')
if 'flag' in config and config['flag'][0:6] == 'regex:':
try:
re.compile(config['flag'][6:])
except Exception:
counted_error('Failed to compile regex flag.')
argument and the script does the rest.
Python dependencies:
- PyYAML (http://pyyaml.org/) or simply `pip3 install PyYAML`
(on Ubuntu you additionally need `apt-get install python3-yaml`)
"""
init_logger()
repo_path, repo_name = get_sys_args()
os.chdir(repo_path)
atexit.register(remove_containers)
proc_list = []
first = None
if 'crp_config' not in read_config(repo_path):
logging.error(
"There is no crp_config in the config.yml,"
" if this is a static challenge you don't need to run it.")
sys.exit(0)
for short_name, crp_config_item in get_crp_config(repo_path,
repo_name).items():
proc, container = run_container(short_name,
crp_config_item,
share_with=first)
proc_list.append(proc)
if first is None:
first = container
logging.info(
def main(config_dir, home_dir, output_dir):
config = read_config(config_dir)
if 'webhook_url' not in config:
write_status(output_dir, False, 'No webhook URL given')
return 1
webhook = config['webhook_url']
description = ''
if 'description' in config:
description = config['description']
info = DashboardInfo(home_dir)
inactive_experiments = [] # list of titles
failed_experiments = [] # list of slack fields
successful_experiments = [] # list of slack fields
failed_graphs = [] # list of titles
for exp_name in info.all_present_experiments():
stage_statuses = info.exp_stage_statuses(exp_name)
if not stage_statuses['precheck']['success']:
failed_experiments.append(
failed_experiment_field(exp_name, stage_statuses, 'precheck'))
continue
exp_conf = info.read_exp_config(exp_name)
exp_title = exp_name if 'title' not in exp_conf else exp_conf['title']
notify = exp_conf['notify']
if not exp_conf['active']:
inactive_experiments.append(exp_title)
continue
failure = False
for stage in ['setup', 'run', 'analysis', 'summary']:
if stage not in stage_statuses:
# setup is the only stage that's optional
assert stage == 'setup'
continue
if not stage_statuses[stage]['success']:
failed_experiments.append(
failed_experiment_field(exp_title, stage_statuses,
stage, notify))
failure = True
break
if failure:
continue
# failure to visualize is not as big a deal as failing to
# run or analyze the experiment, so we only report it but
# don't fail to report the summary
if not stage_statuses['visualization']['success']:
failed_graphs.append(exp_title)
summary = info.read_exp_summary(exp_name)
successful_experiments.append(
build_field(summary['title'], summary['value']))
# produce messages
attachments = []
if successful_experiments:
attachments.append(
build_attachment(
title='Successful benchmarks',
pretext=description,
fields=successful_experiments))
if failed_experiments:
attachments.append(
build_attachment(
color='#fa0000',
title='Failed benchmarks',
fields=failed_experiments))
if inactive_experiments:
attachments.append(
build_attachment(
color='#616161',
title='Inactive benchmarks',
text=', '.join(inactive_experiments)))
if failed_graphs:
attachments.append(
build_attachment(
color='#fa0000',
title='Failed to Visualize',
text=', '.join(failed_graphs)))
success, report = post_message(
webhook,
build_message(
text='Dashboard Results',
attachments=attachments))
write_status(output_dir, success, report)
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option(
"-i",
"--identity-file",
default=None,
help="Specify an identity file to provide to SSH for rsyncing")
parser.add_option("--local-copy-dir",
default=None,
help="Specify a local folder to sync the repo to")
parser.add_option(
"--sync-from-local-copy-dir",
action="store_true",
default=False,
help="Before uploading to servers, sync from local copy dir")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("--no-checksum",
action="store_true",
default=False,
help="Don't use rsync checksums")
(options, args) = parser.parse_args()
config = common.read_config(options)
if len(args) != 1:
logging.critical("Specify a single command")
sys.exit(1)
if args[0] != 'init' and args[0] != 'update':
logging.critical(
"The only commands currently supported are 'init' and 'update'")
sys.exit(1)
if config.get('nonstandardwebroot') is True:
standardwebroot = False
else:
standardwebroot = True
for serverwebroot in config.get('serverwebroot', []):
# this supports both an ssh host:path and just a path
s = serverwebroot.rstrip('/').split(':')
if len(s) == 1:
fdroiddir = s[0]
elif len(s) == 2:
host, fdroiddir = s
else:
logging.error('Malformed serverwebroot line: ' + serverwebroot)
sys.exit(1)
repobase = os.path.basename(fdroiddir)
if standardwebroot and repobase != 'fdroid':
logging.error('serverwebroot path does not end with "fdroid", ' +
'perhaps you meant one of these:\n\t' +
serverwebroot.rstrip('/') + '/fdroid\n\t' +
serverwebroot.rstrip('/').rstrip(repobase) +
'fdroid')
sys.exit(1)
if options.local_copy_dir is not None:
local_copy_dir = options.local_copy_dir
elif config.get('local_copy_dir'):
local_copy_dir = config['local_copy_dir']
else:
local_copy_dir = None
if local_copy_dir is not None:
fdroiddir = local_copy_dir.rstrip('/')
if os.path.exists(fdroiddir) and not os.path.isdir(fdroiddir):
logging.error('local_copy_dir must be directory, not a file!')
sys.exit(1)
if not os.path.exists(os.path.dirname(fdroiddir)):
logging.error('The root dir for local_copy_dir "' +
os.path.dirname(fdroiddir) + '" does not exist!')
sys.exit(1)
if not os.path.isabs(fdroiddir):
logging.error('local_copy_dir must be an absolute path!')
sys.exit(1)
repobase = os.path.basename(fdroiddir)
if standardwebroot and repobase != 'fdroid':
logging.error('local_copy_dir does not end with "fdroid", ' +
'perhaps you meant: ' + fdroiddir + '/fdroid')
sys.exit(1)
if local_copy_dir[-1] != '/':
local_copy_dir += '/'
local_copy_dir = local_copy_dir.replace('//', '/')
if not os.path.exists(fdroiddir):
os.mkdir(fdroiddir)
if not config.get('awsbucket') \
and not config.get('serverwebroot') \
and local_copy_dir is None:
logging.warn('No serverwebroot, local_copy_dir, or awsbucket set!' +
'Edit your config.py to set at least one.')
sys.exit(1)
repo_sections = ['repo']
if config['archive_older'] != 0:
repo_sections.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
if config['per_app_repos']:
repo_sections += common.get_per_app_repos()
if args[0] == 'init':
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
for serverwebroot in config.get('serverwebroot', []):
sshstr, remotepath = serverwebroot.rstrip('/').split(':')
if sshstr.find('@') >= 0:
username, hostname = sshstr.split('@')
else:
username = pwd.getpwuid(os.getuid())[0] # get effective uid
hostname = sshstr
ssh.connect(hostname, username=username)
sftp = ssh.open_sftp()
if os.path.basename(remotepath) \
not in sftp.listdir(os.path.dirname(remotepath)):
sftp.mkdir(remotepath, mode=0755)
for repo_section in repo_sections:
repo_path = os.path.join(remotepath, repo_section)
if os.path.basename(repo_path) \
not in sftp.listdir(remotepath):
sftp.mkdir(repo_path, mode=0755)
sftp.close()
ssh.close()
elif args[0] == 'update':
for repo_section in repo_sections:
if local_copy_dir is not None:
if config['sync_from_local_copy_dir'] and os.path.exists(
repo_section):
sync_from_localcopy(repo_section, local_copy_dir)
else:
update_localcopy(repo_section, local_copy_dir)
for serverwebroot in config.get('serverwebroot', []):
update_serverwebroot(serverwebroot, repo_section)
if config.get('awsbucket'):
update_awsbucket(repo_section)
sys.exit(0)
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-d", "--distinguished-name", default=None,
help="X.509 'Distiguished Name' used when generating keys")
parser.add_option("--keystore", default=None,
help="Path to the keystore for the repo signing key")
parser.add_option("--repo-keyalias", default=None,
help="Alias of the repo signing key in the keystore")
parser.add_option("--android-home", default=None,
help="Path to the Android SDK (sometimes set in ANDROID_HOME)")
parser.add_option("--no-prompt", action="store_true", default=False,
help="Do not prompt for Android SDK path, just fail")
(options, args) = parser.parse_args()
# find root install prefix
tmp = os.path.dirname(sys.argv[0])
if os.path.basename(tmp) == 'bin':
prefix = os.path.dirname(tmp)
examplesdir = prefix + '/share/doc/fdroidserver/examples'
else:
# we're running straight out of the git repo
prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
examplesdir = prefix + '/examples'
fdroiddir = os.getcwd()
test_config = common.get_default_config()
# track down where the Android SDK is, the default is to use the path set
# in ANDROID_HOME if that exists, otherwise None
if options.android_home is not None:
test_config['sdk_path'] = options.android_home
elif not common.test_sdk_exists(test_config):
# if neither --android-home nor the default sdk_path exist, prompt the user
default_sdk_path = '/opt/android-sdk'
while not options.no_prompt:
try:
s = raw_input('Enter the path to the Android SDK ('
+ default_sdk_path + ') here:\n> ')
except KeyboardInterrupt:
print('')
sys.exit(1)
if re.match('^\s*$', s) is not None:
test_config['sdk_path'] = default_sdk_path
else:
test_config['sdk_path'] = s
if common.test_sdk_exists(test_config):
break
if not common.test_sdk_exists(test_config):
sys.exit(3)
if not os.path.exists('config.py'):
# 'metadata' and 'tmp' are created in fdroid
if not os.path.exists('repo'):
os.mkdir('repo')
shutil.copy(os.path.join(examplesdir, 'fdroid-icon.png'), fdroiddir)
shutil.copyfile(os.path.join(examplesdir, 'config.py'), 'config.py')
os.chmod('config.py', 0o0600)
write_to_config('sdk_path', test_config['sdk_path'])
else:
logging.warn('Looks like this is already an F-Droid repo, cowardly refusing to overwrite it...')
logging.info('Try running `fdroid init` in an empty directory.')
sys.exit()
# try to find a working aapt, in all the recent possible paths
build_tools = os.path.join(test_config['sdk_path'], 'build-tools')
aaptdirs = []
aaptdirs.append(os.path.join(build_tools, test_config['build_tools']))
aaptdirs.append(build_tools)
for f in os.listdir(build_tools):
if os.path.isdir(os.path.join(build_tools, f)):
aaptdirs.append(os.path.join(build_tools, f))
for d in sorted(aaptdirs, reverse=True):
if os.path.isfile(os.path.join(d, 'aapt')):
aapt = os.path.join(d, 'aapt')
break
if os.path.isfile(aapt):
dirname = os.path.basename(os.path.dirname(aapt))
if dirname == 'build-tools':
# this is the old layout, before versioned build-tools
test_config['build_tools'] = ''
else:
test_config['build_tools'] = dirname
write_to_config('build_tools', test_config['build_tools'])
if not common.test_build_tools_exists(test_config):
sys.exit(3)
# now that we have a local config.py, read configuration...
config = common.read_config(options)
# track down where the Android NDK is
ndk_path = '/opt/android-ndk'
if os.path.isdir(config['ndk_path']):
ndk_path = config['ndk_path']
elif 'ANDROID_NDK' in os.environ.keys():
logging.info('using ANDROID_NDK')
ndk_path = os.environ['ANDROID_NDK']
if os.path.isdir(ndk_path):
write_to_config('ndk_path', ndk_path)
# the NDK is optional so we don't prompt the user for it if its not found
# find or generate the keystore for the repo signing key. First try the
# path written in the default config.py. Then check if the user has
# specified a path from the command line, which will trump all others.
# Otherwise, create ~/.local/share/fdroidserver and stick it in there. If
# keystore is set to NONE, that means that Java will look for keys in a
# Hardware Security Module aka Smartcard.
keystore = config['keystore']
if options.keystore:
keystore = os.path.abspath(options.keystore)
if options.keystore == 'NONE':
keystore = options.keystore
else:
keystore = os.path.abspath(options.keystore)
if not os.path.exists(keystore):
logging.info('"' + keystore
+ '" does not exist, creating a new keystore there.')
write_to_config('keystore', keystore)
repo_keyalias = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
write_to_config('repo_keyalias', repo_keyalias)
if options.distinguished_name:
keydname = options.distinguished_name
write_to_config('keydname', keydname)
if keystore == 'NONE': # we're using a smartcard
write_to_config('repo_keyalias', '1') # seems to be the default
disable_in_config('keypass', 'never used with smartcard')
write_to_config('smartcardoptions',
('-storetype PKCS11 -providerName SunPKCS11-OpenSC '
+ '-providerClass sun.security.pkcs11.SunPKCS11 '
+ '-providerArg opensc-fdroid.cfg'))
# find opensc-pkcs11.so
if not os.path.exists('opensc-fdroid.cfg'):
if os.path.exists('/usr/lib/opensc-pkcs11.so'):
opensc_so = '/usr/lib/opensc-pkcs11.so'
elif os.path.exists('/usr/lib64/opensc-pkcs11.so'):
opensc_so = '/usr/lib64/opensc-pkcs11.so'
else:
files = glob.glob('/usr/lib/' + os.uname()[4] + '-*-gnu/opensc-pkcs11.so')
if len(files) > 0:
opensc_so = files[0]
else:
opensc_so = '/usr/lib/opensc-pkcs11.so'
logging.warn('No OpenSC PKCS#11 module found, ' +
'install OpenSC then edit "opensc-fdroid.cfg"!')
with open(os.path.join(examplesdir, 'opensc-fdroid.cfg'), 'r') as f:
opensc_fdroid = f.read()
opensc_fdroid = re.sub('^library.*', 'library = ' + opensc_so, opensc_fdroid,
flags=re.MULTILINE)
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
elif not os.path.exists(keystore):
# no existing or specified keystore, generate the whole thing
keystoredir = os.path.dirname(keystore)
if not os.path.exists(keystoredir):
os.makedirs(keystoredir, mode=0o700)
password = genpassword()
write_to_config('keystorepass', password)
write_to_config('keypass', password)
if options.repo_keyalias is None:
repo_keyalias = socket.getfqdn()
write_to_config('repo_keyalias', repo_keyalias)
if not options.distinguished_name:
keydname = 'CN=' + repo_keyalias + ', OU=F-Droid'
write_to_config('keydname', keydname)
genkey(keystore, repo_keyalias, password, keydname)
logging.info('Built repo based in "' + fdroiddir + '"')
logging.info('with this config:')
logging.info(' Android SDK:\t\t\t' + config['sdk_path'])
logging.info(' Android SDK Build Tools:\t' + os.path.dirname(aapt))
logging.info(' Android NDK (optional):\t' + ndk_path)
logging.info(' Keystore for signing key:\t' + keystore)
if repo_keyalias is not None:
logging.info(' Alias for key in store:\t' + repo_keyalias)
logging.info('\nTo complete the setup, add your APKs to "' +
os.path.join(fdroiddir, 'repo') + '"' + '''
then run "fdroid update -c; fdroid update". You might also want to edit
"config.py" to set the URL, repo name, and more. You should also set up
a signing key (a temporary one might have been automatically generated).
For more info: https://f-droid.org/manual/fdroid.html#Simple-Binary-Repository
and https://f-droid.org/manual/fdroid.html#Signing
''')
def main():
# parse options
(options, args) = parse_options()
if os.isatty(sys.stdin.fileno()):
raise RuntimeError('Need configuration in stdin.')
config = common.read_config(sys.stdin)
conn = common.connect(config.s3)
bucket = None
try:
# setup
real_stdout = sys.stdout
sys.stdout = sys.stderr
# verify all required config items are present
if 'roundtrip' not in config:
raise RuntimeError('roundtrip section not found in config')
for item in ['readers', 'writers', 'duration', 'files', 'bucket']:
if item not in config.roundtrip:
raise RuntimeError("Missing roundtrip config item: {item}".format(item=item))
for item in ['num', 'size', 'stddev']:
if item not in config.roundtrip.files:
raise RuntimeError("Missing roundtrip config item: files.{item}".format(item=item))
seeds = dict(config.roundtrip.get('random_seed', {}))
seeds.setdefault('main', random.randrange(2**32))
rand = random.Random(seeds['main'])
for name in ['names', 'contents', 'writer', 'reader']:
seeds.setdefault(name, rand.randrange(2**32))
print 'Using random seeds: {seeds}'.format(seeds=seeds)
# setup bucket and other objects
bucket_name = common.choose_bucket_prefix(config.roundtrip.bucket, max_len=30)
bucket = conn.create_bucket(bucket_name)
print "Created bucket: {name}".format(name=bucket.name)
objnames = realistic.names(
mean=15,
stddev=4,
seed=seeds['names'],
)
objnames = itertools.islice(objnames, config.roundtrip.files.num)
objnames = list(objnames)
files = realistic.files(
mean=1024 * config.roundtrip.files.size,
stddev=1024 * config.roundtrip.files.stddev,
seed=seeds['contents'],
)
q = gevent.queue.Queue()
logger_g = gevent.spawn(yaml.safe_dump_all, q, stream=real_stdout)
print "Writing {num} objects with {w} workers...".format(
num=config.roundtrip.files.num,
w=config.roundtrip.writers,
)
pool = gevent.pool.Pool(size=config.roundtrip.writers)
start = time.time()
for objname in objnames:
fp = next(files)
pool.spawn(
writer,
bucket=bucket,
objname=objname,
fp=fp,
queue=q,
)
pool.join()
stop = time.time()
elapsed = stop - start
q.put(dict(
type='write_done',
duration=int(round(elapsed * NANOSECOND)),
))
print "Reading {num} objects with {w} workers...".format(
num=config.roundtrip.files.num,
w=config.roundtrip.readers,
)
# avoid accessing them in the same order as the writing
rand.shuffle(objnames)
pool = gevent.pool.Pool(size=config.roundtrip.readers)
start = time.time()
for objname in objnames:
pool.spawn(
reader,
bucket=bucket,
objname=objname,
queue=q,
)
pool.join()
stop = time.time()
elapsed = stop - start
q.put(dict(
type='read_done',
duration=int(round(elapsed * NANOSECOND)),
))
q.put(StopIteration)
logger_g.get()
finally:
# cleanup
if options.cleanup:
if bucket is not None:
common.nuke_bucket(bucket)
def main():
global options, config
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
parser.add_argument("--recalc", action="store_true", default=False,
help="Recalculate aggregate stats - use when changes "
"have been made that would invalidate old cached data.")
parser.add_argument("--nologs", action="store_true", default=False,
help="Don't do anything logs-related")
options = parser.parse_args()
config = common.read_config(options)
if not config['update_stats']:
logging.info("Stats are disabled - set \"update_stats = True\" in your config.py")
sys.exit(1)
# Get all metadata-defined apps...
allmetaapps = [app for app in metadata.read_metadata().itervalues()]
metaapps = [app for app in allmetaapps if not app.Disabled]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
logging.info('Retrieving logs')
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(config['stats_server'], username=config['stats_user'],
timeout=10, key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
logging.info("...connected")
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
logging.debug("...retrieving " + f)
ftp.get(f, destpath)
except Exception:
traceback.print_exc()
sys.exit(1)
finally:
# Disconnect
if ftp is not None:
ftp.close()
if ssh is not None:
ssh.close()
knownapks = common.KnownApks()
unknownapks = []
if not options.nologs:
# Process logs
logging.info('Processing logs...')
appscount = Counter()
appsvercount = Counter()
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
logging.debug('...' + logfile)
# Get the date for this log - e.g. 2012-02-28
thisdate = os.path.basename(logfile)[7:-7]
agg_path = os.path.join(datadir, thisdate + '.json')
if not options.recalc and os.path.exists(agg_path):
# Use previously calculated aggregate data
with open(agg_path, 'r') as f:
today = json.load(f)
else:
# Calculate from logs...
today = {
'apps': Counter(),
'appsver': Counter(),
'unknown': []
}
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if not match:
continue
if match.group('statuscode') != '200':
continue
if match.group('ip') in config['stats_ignore']:
continue
uri = match.group('uri')
if not uri.endswith('.apk'):
continue
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
today['apps'][appid] += 1
# Strip the '.apk' from apkname
appver = apkname[:-4]
today['appsver'][appver] += 1
else:
if apkname not in today['unknown']:
today['unknown'].append(apkname)
# Save calculated aggregate data for today to cache
with open(agg_path, 'w') as f:
json.dump(today, f)
# Add today's stats (whether cached or recalculated) to the total
for appid in today['apps']:
appscount[appid] += today['apps'][appid]
for appid in today['appsver']:
appsvercount[appid] += today['appsver'][appid]
for uk in today['unknown']:
if uk not in unknownapks:
unknownapks.append(uk)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for appid in appscount:
count = appscount[appid]
lst.append(appid + " " + str(count))
if config['stats_to_carbon']:
carbon_send('fdroid.download.' + appid.replace('.', '_'),
count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
with open(os.path.join(statsdir, 'total_downloads_app.txt'), 'w') as f:
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
lst = []
for appver in appsvercount:
count = appsvercount[appver]
lst.append(appver + " " + str(count))
with open(os.path.join(statsdir, 'total_downloads_app_version.txt'), 'w') as f:
f.write('# Total downloads by application and version, '
'since October 2011\n')
for line in sorted(lst):
f.write(line + "\n")
# Calculate and write stats for repo types...
logging.info("Processing repo types...")
repotypes = Counter()
for app in metaapps:
rtype = app.RepoType or 'none'
if rtype == 'srclib':
rtype = common.getsrclibvcs(app.Repo)
repotypes[rtype] += 1
with open(os.path.join(statsdir, 'repotypes.txt'), 'w') as f:
for rtype, count in most_common_stable(repotypes):
f.write(rtype + ' ' + str(count) + '\n')
# Calculate and write stats for update check modes...
logging.info("Processing update check modes...")
ucms = Counter()
for app in metaapps:
checkmode = app.UpdateCheckMode
if checkmode.startswith('RepoManifest/'):
checkmode = checkmode[:12]
if checkmode.startswith('Tags '):
checkmode = checkmode[:4]
ucms[checkmode] += 1
with open(os.path.join(statsdir, 'update_check_modes.txt'), 'w') as f:
for checkmode, count in most_common_stable(ucms):
f.write(checkmode + ' ' + str(count) + '\n')
logging.info("Processing categories...")
ctgs = Counter()
for app in metaapps:
for category in app.Categories:
ctgs[category] += 1
with open(os.path.join(statsdir, 'categories.txt'), 'w') as f:
for category, count in most_common_stable(ctgs):
f.write(category + ' ' + str(count) + '\n')
logging.info("Processing antifeatures...")
afs = Counter()
for app in metaapps:
if app.AntiFeatures is None:
continue
for antifeature in app.AntiFeatures:
afs[antifeature] += 1
with open(os.path.join(statsdir, 'antifeatures.txt'), 'w') as f:
for antifeature, count in most_common_stable(afs):
f.write(antifeature + ' ' + str(count) + '\n')
# Calculate and write stats for licenses...
logging.info("Processing licenses...")
licenses = Counter()
for app in metaapps:
license = app.License
licenses[license] += 1
with open(os.path.join(statsdir, 'licenses.txt'), 'w') as f:
for license, count in most_common_stable(licenses):
f.write(license + ' ' + str(count) + '\n')
# Write list of disabled apps...
logging.info("Processing disabled apps...")
disabled = [app.id for app in allmetaapps if app.Disabled]
with open(os.path.join(statsdir, 'disabled_apps.txt'), 'w') as f:
for appid in sorted(disabled):
f.write(appid + '\n')
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
with open(os.path.join(statsdir, 'latestapps.txt'), 'w') as f:
for appid in latest:
f.write(appid + '\n')
if unknownapks:
logging.info('\nUnknown apks:')
for apk in unknownapks:
logging.info(apk)
logging.info("Finished.")
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Verify only the specified package")
(options, args) = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
print "No unsigned directory - nothing to do"
sys.exit(0)
verified = 0
notverified = 0
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
i = apkfilename.rfind('_')
if i == -1:
raise BuildException("Invalid apk name")
appid = apkfilename[:i]
if not options.package or options.package == appid:
try:
print "Processing " + apkfilename
remoteapk = os.path.join(tmp_dir, apkfilename)
if os.path.exists(remoteapk):
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
print "...retrieving " + url
p = subprocess.Popen(['wget', url],
cwd=tmp_dir,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out = p.communicate()[0]
if p.returncode != 0:
raise Exception("Failed to get " + apkfilename)
thisdir = os.path.join(tmp_dir, 'this_apk')
thatdir = os.path.join(tmp_dir, 'that_apk')
for d in [thisdir, thatdir]:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d)
if subprocess.call(['jar', 'xf',
os.path.join("..", "..", unsigned_dir, apkfilename)],
cwd=thisdir) != 0:
raise Exception("Failed to unpack local build of " + apkfilename)
if subprocess.call(['jar', 'xf', os.path.join("..", "..", remoteapk)],
cwd=thatdir) != 0:
raise Exception("Failed to unpack remote build of " + apkfilename)
p = subprocess.Popen(['diff', '-r', 'this_apk', 'that_apk'],
cwd=tmp_dir, stdout=subprocess.PIPE)
out = p.communicate()[0]
lines = out.splitlines()
if len(lines) != 1 or lines[0].find('META-INF') == -1:
raise Exception("Unexpected diff output - " + out)
print "...successfully verified"
verified += 1
except Exception, e:
print "...NOT verified - {0}".format(e)
notverified += 1
def main():
global options, config
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
common.setup_global_opts(parser)
parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
parser.add_argument("-a", "--all", action="store_true", default=False,
help="Install all signed applications available")
options = parser.parse_args()
if not options.appid and not options.all:
parser.error("option %s: If you really want to install all the signed apps, use --all" % "all")
config = common.read_config(options)
output_dir = 'repo'
if not os.path.isdir(output_dir):
logging.info("No signed output directory - nothing to do")
sys.exit(0)
if options.appid:
vercodes = common.read_pkg_args(options.appid, True)
apks = {appid: None for appid in vercodes}
# Get the signed apk with the highest vercode
for apkfile in sorted(glob.glob(os.path.join(output_dir, '*.apk'))):
try:
appid, vercode = common.apknameinfo(apkfile)
except FDroidException:
continue
if appid not in apks:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
apks[appid] = apkfile
for appid, apk in apks.iteritems():
if not apk:
raise FDroidException("No signed apk available for %s" % appid)
else:
apks = {common.apknameinfo(apkfile)[0]: apkfile for apkfile in
sorted(glob.glob(os.path.join(output_dir, '*.apk')))}
for appid, apk in apks.iteritems():
# Get device list each time to avoid device not found errors
devs = devices()
if not devs:
raise FDroidException("No attached devices found")
logging.info("Installing %s..." % apk)
for dev in devs:
logging.info("Installing %s on %s..." % (apk, dev))
p = SdkToolsPopen(['adb', "-s", dev, "install", apk])
fail = ""
for line in p.output.splitlines():
if line.startswith("Failure"):
fail = line[9:-1]
if not fail:
continue
if fail == "INSTALL_FAILED_ALREADY_EXISTS":
logging.warn("%s is already installed on %s." % (apk, dev))
else:
raise FDroidException("Failed to install %s on %s: %s" % (
apk, dev, fail))
logging.info("\nFinished")
if __name__ == "__main__":
print("\nStart train XGB model\n")
parser = argparse.ArgumentParser(description="Train XGB model")
parser.add_argument("config_filepath",
metavar="<config_filepath>",
help="Config file")
args = parser.parse_args()
config_filepath = args.config_filepath
assert os.path.exists(
config_filepath), "Config file is not found at %s" % config_filepath
config = read_config(config_filepath)
now = datetime.now()
config_filename = os.path.basename(config_filepath)
config_filename = config_filename.replace(".json", "")
output_path = os.path.abspath(
os.path.join(
"..", "output",
"%s_training_%s" % (now.strftime("%Y%m%d_%H%M"), config_filename)))
logs_path = output_path
if not os.path.exists(logs_path):
os.makedirs(logs_path)
assert "dataset" in config
train_df = get_train_df(config["dataset"])
def read_exp_config(self, exp_name):
return read_config(self.exp_config_dir(exp_name))
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-u",
"--url",
default=None,
help="Project URL to import from.")
parser.add_option(
"-s",
"--subdir",
default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option(
"--rev",
default=None,
help=
"Allows a different revision (or git branch) to be specified for the initial import"
)
(options, args) = parser.parse_args()
config = common.read_config(options)
if not options.url:
logging.error("Specify project url.")
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
# Get all apps...
apps = metadata.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url # by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
elif url.startswith('https://github.com'):
projecttype = 'github'
repo = url
repotype = 'git'
sourcecode = url
issuetracker = url + '/issues'
website = ""
elif url.startswith('https://gitlab.com/'):
projecttype = 'gitlab'
repo = url
repotype = 'git'
sourcecode = url + '/tree/HEAD'
issuetracker = url + '/issues'
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
logging.error("Unable to determine vcs type. " + repo)
sys.exit(1)
if not projecttype:
logging.error("Unable to determine the project type.")
logging.error(
"The URL you supplied was not in one of the supported formats. Please consult"
)
logging.error(
"the manual for a list of supported formats, and supply one of those."
)
sys.exit(1)
# Ensure we have a sensible-looking repo address at this point. If not, we
# might have got a page format we weren't expecting. (Note that we
# specifically don't want git@...)
if ((repotype != 'bzr' and
(not repo.startswith('http://') and not repo.startswith('https://')
and not repo.startswith('git://'))) or ' ' in repo):
logging.error(
"Repo address '{0}' does not seem to be valid".format(repo))
sys.exit(1)
# Get a copy of the source so we can extract some info...
logging.info('Getting source from ' + repotype + ' repo at ' + repo)
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir)
vcs.gotorevision(options.rev)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Extract some information...
paths = common.manifest_paths(root_dir, [])
if paths:
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
logging.error("Couldn't find package ID")
sys.exit(1)
if not version:
logging.warn("Couldn't find latest version name")
if not vercode:
logging.warn("Couldn't find latest version code")
else:
spec = os.path.join(root_dir, 'buildozer.spec')
if os.path.exists(spec):
defaults = {
'orientation': 'landscape',
'icon': '',
'permissions': '',
'android.api': "18"
}
bconfig = ConfigParser(defaults, allow_no_value=True)
bconfig.read(spec)
package = bconfig.get('app', 'package.domain') + '.' + bconfig.get(
'app', 'package.name')
version = bconfig.get('app', 'version')
vercode = None
else:
logging.error(
"No android or kivy project could be found. Specify --subdir?")
sys.exit(1)
# Make sure it's actually new...
if package in apps:
logging.error("Package " + package + " already exists")
sys.exit(1)
# Construct the metadata...
app = metadata.parse_metadata(None)[1]
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
app['Update Check Mode'] = "Tags"
# Create a build line...
build = {}
build['version'] = version or '?'
build['vercode'] = vercode or '?'
build['commit'] = '?'
build[
'disable'] = 'Generated by import.py - check/set version fields and commit id'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = ['yes']
for flag, value in metadata.flag_defaults.iteritems():
if flag in build:
continue
build[flag] = value
app['builds'].append(build)
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
with open('build/.fdroidvcs-' + package, 'w') as f:
f.write(repotype + ' ' + repo)
metafile = os.path.join('metadata', package + '.txt')
metadata.write_metadata(metafile, app)
logging.info("Wrote " + metafile)
def read_subsys_config(self, subsys_name):
return read_config(self.subsys_config_dir(subsys_name))
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_option("--autoonly", action="store_true", default=False,
help="Only process apps with auto-updates")
parser.add_option("--commit", action="store_true", default=False,
help="Commit changes")
parser.add_option("--gplay", action="store_true", default=False,
help="Only print differences with the Play Store")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, False)
if options.gplay:
for app in apps:
version, reason = check_gplay(app)
if version is None:
if reason == '404':
logging.info("{0} is not in the Play Store".format(common.getappname(app)))
else:
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
if version is not None:
stored = app['Current Version']
if not stored:
logging.info("{0} has no Current Version but has version {1} on the Play Store"
.format(common.getappname(app), version))
elif LooseVersion(stored) < LooseVersion(version):
logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
.format(common.getappname(app), version, stored))
else:
if stored != version:
logging.info("{0} has version {1} on the Play Store, which differs from {2}"
.format(common.getappname(app), version, stored))
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
return
for appid, app in apps.iteritems():
if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
logging.debug("Nothing to do for {0}...".format(appid))
continue
logging.info("Processing " + appid + '...')
checkupdates_app(app)
logging.info("Finished.")
def main():
global options, config
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
(options, args) = parser.parse_args()
config = common.read_config(options)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
logging.info("Creating temporary directory")
os.makedirs(tmp_dir)
unsigned_dir = 'unsigned'
if not os.path.isdir(unsigned_dir):
logging.error("No unsigned directory - nothing to do")
sys.exit(0)
verified = 0
notverified = 0
vercodes = common.read_pkg_args(args, True)
for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
apkfilename = os.path.basename(apkfile)
appid, vercode = common.apknameinfo(apkfile)
if vercodes and appid not in vercodes:
continue
if vercodes[appid] and vercode not in vercodes[appid]:
continue
try:
logging.info("Processing " + apkfilename)
remoteapk = os.path.join(tmp_dir, apkfilename)
if os.path.exists(remoteapk):
os.remove(remoteapk)
url = 'https://f-droid.org/repo/' + apkfilename
logging.info("...retrieving " + url)
p = FDroidPopen(['wget', url], cwd=tmp_dir)
if p.returncode != 0:
raise FDroidException("Failed to get " + apkfilename)
thisdir = os.path.join(tmp_dir, 'this_apk')
thatdir = os.path.join(tmp_dir, 'that_apk')
for d in [thisdir, thatdir]:
if os.path.exists(d):
shutil.rmtree(d)
os.mkdir(d)
if subprocess.call(['jar', 'xf',
os.path.join("..", "..", unsigned_dir, apkfilename)],
cwd=thisdir) != 0:
raise FDroidException("Failed to unpack local build of " + apkfilename)
if subprocess.call(['jar', 'xf',
os.path.join("..", "..", remoteapk)],
cwd=thatdir) != 0:
raise FDroidException("Failed to unpack remote build of " + apkfilename)
p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir)
lines = p.output.splitlines()
if len(lines) != 1 or 'META-INF' not in lines[0]:
raise FDroidException("Unexpected diff output - " + p.output)
logging.info("...successfully verified")
verified += 1
except FDroidException, e:
logging.info("...NOT verified - {0}".format(e))
notverified += 1
def test_read_config(self):
# Try to read a non-existent config. Config is vital
# so this should blow up.
with self.assertRaises(Exception):
common.read_config('')
def main(home_dir):
global_conf = read_config(sanitize_path(home_dir))
print(sanitize_path(global_conf['setup_dir']))
def find_and_load_config(fn):
fn = conditionally_prepend_section_path(fn)
config = read_config(fn)
return fn, config
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
parser.add_option("--recalc", action="store_true", default=False,
help="Recalculate aggregate stats - use when changes "
"have been made that would invalidate old cached data.")
parser.add_option("--nologs", action="store_true", default=False,
help="Don't do anything logs-related")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not config['update_stats']:
logging.info("Stats are disabled - check your configuration")
sys.exit(1)
# Get all metadata-defined apps...
metaapps = [a for a in metadata.read_metadata().itervalues() if not a['Disabled']]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
logging.info('Retrieving logs')
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='******', timeout=10,
key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
logging.info("...connected")
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
logging.debug("...retrieving " + f)
ftp.get(f, destpath)
except Exception:
traceback.print_exc()
sys.exit(1)
finally:
# Disconnect
if ftp is not None:
ftp.close()
if ssh is not None:
ssh.close()
knownapks = common.KnownApks()
unknownapks = []
if not options.nologs:
# Process logs
logging.info('Processing logs...')
appscount = Counter()
appsvercount = Counter()
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
logging.debug('...' + logfile)
# Get the date for this log - e.g. 2012-02-28
thisdate = os.path.basename(logfile)[7:-7]
agg_path = os.path.join(datadir, thisdate + '.json')
if not options.recalc and os.path.exists(agg_path):
# Use previously calculated aggregate data
with open(agg_path, 'r') as f:
today = json.load(f)
else:
# Calculate from logs...
today = {
'apps': Counter(),
'appsver': Counter(),
'unknown': []
}
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if not match:
continue
if match.group('statuscode') != '200':
continue
if match.group('ip') in config['stats_ignore']:
continue
uri = match.group('uri')
if not uri.endswith('.apk'):
continue
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
today['apps'][appid] += 1
# Strip the '.apk' from apkname
appver = apkname[:-4]
today['appsver'][appver] += 1
else:
if apkname not in today['unknown']:
today['unknown'].append(apkname)
# Save calculated aggregate data for today to cache
with open(agg_path, 'w') as f:
json.dump(today, f)
# Add today's stats (whether cached or recalculated) to the total
for appid in today['apps']:
appscount[appid] += today['apps'][appid]
for appid in today['appsver']:
appsvercount[appid] += today['appsver'][appid]
for uk in today['unknown']:
if uk not in unknownapks:
unknownapks.append(uk)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for appid in appscount:
count = appscount[appid]
lst.append(appid + " " + str(count))
if config['stats_to_carbon']:
carbon_send('fdroid.download.' + appid.replace('.', '_'),
count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
f = open('stats/total_downloads_app_version.txt', 'w')
f.write('# Total downloads by application and version, '
'since October 2011\n')
lst = []
for appver in appsvercount:
count = appsvercount[appver]
lst.append(appver + " " + str(count))
for line in sorted(lst):
f.write(line + "\n")
f.close()
# Calculate and write stats for repo types...
logging.info("Processing repo types...")
repotypes = Counter()
for app in metaapps:
rtype = app['Repo Type'] or 'none'
if rtype == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
repotypes[rtype] += 1
f = open('stats/repotypes.txt', 'w')
for rtype in repotypes:
count = repotypes[rtype]
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
logging.info("Processing update check modes...")
ucms = Counter()
for app in metaapps:
checkmode = app['Update Check Mode']
if checkmode.startswith('RepoManifest/'):
checkmode = checkmode[:12]
if checkmode.startswith('Tags '):
checkmode = checkmode[:4]
ucms[checkmode] += 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode in ucms:
count = ucms[checkmode]
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
logging.info("Processing categories...")
ctgs = Counter()
for app in metaapps:
for category in app['Categories']:
ctgs[category] += 1
f = open('stats/categories.txt', 'w')
for category in ctgs:
count = ctgs[category]
f.write(category + ' ' + str(count) + '\n')
f.close()
logging.info("Processing antifeatures...")
afs = Counter()
for app in metaapps:
if app['AntiFeatures'] is None:
continue
antifeatures = [a.strip() for a in app['AntiFeatures'].split(',')]
for antifeature in antifeatures:
afs[antifeature] += 1
f = open('stats/antifeatures.txt', 'w')
for antifeature in afs:
count = afs[antifeature]
f.write(antifeature + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
logging.info("Processing licenses...")
licenses = Counter()
for app in metaapps:
license = app['License']
licenses[license] += 1
f = open('stats/licenses.txt', 'w')
for license in licenses:
count = licenses[license]
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if unknownapks:
logging.info('\nUnknown apks:')
for apk in unknownapks:
logging.info(apk)
logging.info("Finished.")
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="No output, except for warnings and errors")
parser.add_option("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status")
parser.add_option("-i", "--interactive", default=False, action="store_true",
help="Interactively ask about things that need updating.")
parser.add_option("-I", "--icons", action="store_true", default=False,
help="Resize all the icons exceeding the max pixel size and exit")
parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default "+
"is /etc/alternatives/editor")
parser.add_option("-w", "--wiki", default=False, action="store_true",
help="Update the wiki")
parser.add_option("", "--pretty", action="store_true", default=False,
help="Produce human-readable index.xml")
parser.add_option("--clean", action="store_true", default=False,
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
if options.icons:
resize_all_icons(repodirs)
sys.exit(0)
# Get all apps...
apps = metadata.read_metadata()
# Generate a list of categories...
categories = []
for app in apps:
cats = app['Categories'].split(',')
for cat in cats:
if cat not in categories:
categories.append(cat)
# Read known apks data (will be updated and written back when we've finished)
knownapks = common.KnownApks()
# Gather information about all the apk files in the repo directory, using
# cached data if possible.
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf)
else:
apkcache = {}
cachechanged = False
delete_disabled_builds(apps, apkcache, repodirs)
# Scan all apks in the main repo
apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
if cc:
cachechanged = True
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
if cc:
cachechanged = True
else:
archapks = []
# Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
for app in apps:
bestver = 0
added = None
lastupdated = None
for apk in apks + archapks:
if apk['id'] == app['id']:
if apk['versioncode'] > bestver:
bestver = apk['versioncode']
bestapk = apk
if 'added' in apk:
if not added or apk['added'] < added:
added = apk['added']
if not lastupdated or apk['added'] > lastupdated:
lastupdated = apk['added']
if added:
app['added'] = added
else:
if options.verbose:
print "WARNING: Don't know when " + app['id'] + " was added"
if lastupdated:
app['lastupdated'] = lastupdated
else:
if options.verbose:
print "WARNING: Don't know when " + app['id'] + " was last updated"
if bestver == 0:
if app['Name'] is None:
app['Name'] = app['id']
app['icon'] = None
if options.verbose and app['Disabled'] is None:
print "WARNING: Application " + app['id'] + " has no packages"
else:
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
apps = sorted(apps, key=lambda app: app['Name'].upper())
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
for apk in apks:
found = False
for app in apps:
if app['id'] == apk['id']:
found = True
break
if not found:
if options.createmeta:
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
f.write(".\n")
f.close()
print "Generated skeleton metadata for " + apk['id']
else:
print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
print " " + apk['name'] + " - " + apk['version']
if len(repodirs) > 1:
archive_old_apks(apps, apks, repodirs[0], repodirs[1], config['archive_older'])
# Make the index for the main repo...
make_index(apps, apks, repodirs[0], False, categories)
# If there's an archive repo, make the index for it. We already scanned it
# earlier on.
if len(repodirs) > 1:
make_index(apps, archapks, repodirs[1], True, categories)
if config['update_stats']:
# Update known apks info...
knownapks.writeifchanged()
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip()
data += appid + "\t"
for app in apps:
if app['id'] == appid:
data += app['Name'] + "\t"
if app['icon'] is not None:
data += app['icon'] + "\t"
data += app['License'] + "\n"
break
f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
f.write(data)
f.close()
if cachechanged:
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
# Update the wiki...
if options.wiki:
update_wiki(apps, apks + archapks)
print "Finished."
def main():
global config, options
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_option("--autoonly", action="store_true", default=False,
help="Only process apps with auto-updates")
parser.add_option("--commit", action="store_true", default=False,
help="Commit changes")
parser.add_option("--gplay", action="store_true", default=False,
help="Only print differences with the Play Store")
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata()
apps = common.read_app_args(args, allapps, False)
if options.gplay:
for app in apps:
version, reason = check_gplay(app)
if version is None:
if reason == '404':
logging.info("{0} is not in the Play Store".format(common.getappname(app)))
else:
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
if version is not None:
stored = app['Current Version']
if not stored:
logging.info("{0} has no Current Version but has version {1} on the Play Store"
.format(common.getappname(app), version))
elif LooseVersion(stored) < LooseVersion(version):
logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
.format(common.getappname(app), version, stored))
else:
if stored != version:
logging.info("{0} has version {1} on the Play Store, which differs from {2}"
.format(common.getappname(app), version, stored))
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
return
for appid, app in apps.iteritems():
if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
logging.debug("Nothing to do for {0}...".format(appid))
continue
logging.info("Processing " + appid + '...')
checkupdates_app(app)
logging.info("Finished.")
def main():
global config, options, curid, count
curid = None
count = Counter()
def warn(message):
global curid, count
if curid:
print "%s:" % curid
curid = None
count["app"] += 1
print " %s" % message
count["warn"] += 1
# Parse command line...
parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option(
"-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal"
)
parser.add_option(
"-q", "--quiet", action="store_true", default=False, help="Restrict output to warnings and errors"
)
(options, args) = parser.parse_args()
config = common.read_config(options)
# Get all apps...
allapps = metadata.read_metadata(xref=False)
apps = common.read_app_args(args, allapps, False)
filling_ucms = re.compile("^(Tags.*|RepoManifest.*)")
for appid, app in apps.iteritems():
if app["Disabled"]:
continue
curid = appid
count["app_total"] += 1
# enabled_builds = 0
lowest_vercode = -1
curbuild = None
for build in app["builds"]:
if not build["disable"]:
# enabled_builds += 1
vercode = int(build["vercode"])
if lowest_vercode == -1 or vercode < lowest_vercode:
lowest_vercode = vercode
if not curbuild or int(build["vercode"]) > int(curbuild["vercode"]):
curbuild = build
# Incorrect UCM
if (
curbuild
and curbuild["commit"]
and app["Update Check Mode"] == "RepoManifest"
and not curbuild["commit"].startswith("unknown")
and curbuild["vercode"] == app["Current Version Code"]
and not curbuild["forcevercode"]
and any(s in curbuild["commit"] for s in ".,_-/")
):
warn(
"Last used commit '%s' looks like a tag, but Update Check Mode is '%s'"
% (curbuild["commit"], app["Update Check Mode"])
)
# Summary size limit
summ_chars = len(app["Summary"])
if summ_chars > config["char_limits"]["Summary"]:
warn("Summary of length %s is over the %i char limit" % (summ_chars, config["char_limits"]["Summary"]))
# Redundant info
if app["Web Site"] and app["Source Code"]:
if app["Web Site"].lower() == app["Source Code"].lower():
warn("Website '%s' is just the app's source code link" % app["Web Site"])
if filling_ucms.match(app["Update Check Mode"]):
if all(
app[f] == metadata.app_defaults[f] for f in ["Auto Name", "Current Version", "Current Version Code"]
):
warn("UCM is set but it looks like checkupdates hasn't been run yet")
if app["Update Check Name"] == appid:
warn("Update Check Name is set to the known app id - it can be removed")
cvc = int(app["Current Version Code"])
if cvc > 0 and cvc < lowest_vercode:
warn("Current Version Code is lower than any enabled build")
# Missing or incorrect categories
if not app["Categories"]:
warn("Categories are not set")
for categ in app["Categories"]:
if categ not in categories:
warn("Category '%s' is not valid" % categ)
if app["Name"] and app["Name"] == app["Auto Name"]:
warn("Name '%s' is just the auto name" % app["Name"])
name = app["Name"] or app["Auto Name"]
if app["Summary"] and name:
if app["Summary"].lower() == name.lower():
warn("Summary '%s' is just the app's name" % app["Summary"])
desc = app["Description"]
if app["Summary"] and desc and len(desc) == 1:
if app["Summary"].lower() == desc[0].lower():
warn("Description '%s' is just the app's summary" % app["Summary"])
# Description size limit
desc_charcount = sum(len(l) for l in desc)
if desc_charcount > config["char_limits"]["Description"]:
warn(
"Description of length %s is over the %i char limit"
% (desc_charcount, config["char_limits"]["Description"])
)
if not desc[0] or not desc[-1] or any(not desc[l - 1] and not desc[l] for l in range(1, len(desc))):
warn("Description has an extra empty line")
# Check for lists using the wrong characters
validchars = ["*", "#"]
lchar = ""
lcount = 0
for l in app["Description"]:
if len(l) < 1:
continue
for um in desc_url.finditer(l):
url = um.group(1)
for m, r in http_warnings:
if m.match(url):
warn("URL '%s' in Description: %s" % (url, r))
c = l.decode("utf-8")[0]
if c == lchar:
lcount += 1
if lcount > 3 and lchar not in validchars:
warn("Description has a list (%s) but it isn't bulleted (*) nor numbered (#)" % lchar)
break
else:
lchar = c
lcount = 1
# Regex checks in all kinds of fields
for f in regex_warnings:
for m, r in regex_warnings[f]:
v = app[f]
if type(v) == str:
if v is None:
continue
if m.match(v):
warn("%s '%s': %s" % (f, v, r))
elif type(v) == list:
for l in v:
if m.match(l):
warn("%s at line '%s': %s" % (f, l, r))
# Build warnings
for build in app["builds"]:
if build["disable"]:
continue
for s in ["master", "origin", "HEAD", "default", "trunk"]:
if build["commit"] and build["commit"].startswith(s):
warn("Branch '%s' used as commit in build '%s'" % (s, build["version"]))
for srclib in build["srclibs"]:
ref = srclib.split("@")[1].split("/")[0]
if ref.startswith(s):
warn("Branch '%s' used as commit in srclib '%s'" % (s, srclib))
if not curid:
print
logging.info(
"Found a total of %i warnings in %i apps out of %i total." % (count["warn"], count["app"], count["app_total"])
)
sys.exit(1 if count["warn"] > 0 else 0)
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-c",
"--create-metadata",
action="store_true",
default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("--delete-unknown",
action="store_true",
default=False,
help="Delete APKs without metadata from the repo")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-b",
"--buildreport",
action="store_true",
default=False,
help="Report on build data status")
parser.add_option(
"-i",
"--interactive",
default=False,
action="store_true",
help="Interactively ask about things that need updating.")
parser.add_option(
"-I",
"--icons",
action="store_true",
default=False,
help="Resize all the icons exceeding the max pixel size and exit")
parser.add_option(
"-e",
"--editor",
default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default " +
"is /etc/alternatives/editor")
parser.add_option("-w",
"--wiki",
default=False,
action="store_true",
help="Update the wiki")
parser.add_option("",
"--pretty",
action="store_true",
default=False,
help="Produce human-readable index.xml")
parser.add_option(
"--clean",
action="store_true",
default=False,
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
if options.icons:
resize_all_icons(repodirs)
sys.exit(0)
# check that icons exist now, rather than fail at the end of `fdroid update`
for k in ['repo_icon', 'archive_icon']:
if k in config:
if not os.path.exists(config[k]):
logging.critical(k + ' "' + config[k] +
'" does not exist! Correct it in config.py.')
sys.exit(1)
# Get all apps...
apps = metadata.read_metadata()
# Generate a list of categories...
categories = set()
for app in apps.itervalues():
categories.update(app['Categories'])
# Read known apks data (will be updated and written back when we've finished)
knownapks = common.KnownApks()
# Gather information about all the apk files in the repo directory, using
# cached data if possible.
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf)
else:
apkcache = {}
cachechanged = False
delete_disabled_builds(apps, apkcache, repodirs)
# Scan all apks in the main repo
apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
if cc:
cachechanged = True
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
newmetadata = False
for apk in apks:
if apk['id'] not in apps:
if options.create_metadata:
if 'name' not in apk:
logging.error(apk['id'] +
' does not have a name! Skipping...')
continue
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
f.write(".\n")
f.close()
logging.info("Generated skeleton metadata for " + apk['id'])
newmetadata = True
else:
msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
if options.delete_unknown:
logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
rmf = os.path.join(repodirs[0], apk['apkname'])
if not os.path.exists(rmf):
logging.error(
"Could not find {0} to remove it".format(rmf))
else:
os.remove(rmf)
else:
logging.warn(msg +
"\n\tUse `fdroid update -c` to create it.")
# update the metadata with the newly created ones included
if newmetadata:
apps = metadata.read_metadata()
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
if cc:
cachechanged = True
else:
archapks = []
# Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
for appid, app in apps.iteritems():
bestver = 0
added = None
lastupdated = None
for apk in apks + archapks:
if apk['id'] == appid:
if apk['versioncode'] > bestver:
bestver = apk['versioncode']
bestapk = apk
if 'added' in apk:
if not added or apk['added'] < added:
added = apk['added']
if not lastupdated or apk['added'] > lastupdated:
lastupdated = apk['added']
if added:
app['added'] = added
else:
logging.warn("Don't know when " + appid + " was added")
if lastupdated:
app['lastupdated'] = lastupdated
else:
logging.warn("Don't know when " + appid + " was last updated")
if bestver == 0:
if app['Name'] is None:
app['Name'] = appid
app['icon'] = None
logging.warn("Application " + appid + " has no packages")
else:
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
sortedids = sorted(apps.iterkeys(),
key=lambda appid: apps[appid]['Name'].upper())
if len(repodirs) > 1:
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1],
config['archive_older'])
# Make the index for the main repo...
make_index(apps, sortedids, apks, repodirs[0], False, categories)
# If there's an archive repo, make the index for it. We already scanned it
# earlier on.
if len(repodirs) > 1:
make_index(apps, sortedids, archapks, repodirs[1], True, categories)
if config['update_stats']:
# Update known apks info...
knownapks.writeifchanged()
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip()
data += appid + "\t"
app = apps[appid]
data += app['Name'] + "\t"
if app['icon'] is not None:
data += app['icon'] + "\t"
data += app['License'] + "\n"
f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
f.write(data)
f.close()
if cachechanged:
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
# Update the wiki...
if options.wiki:
update_wiki(apps, sortedids, apks + archapks)
logging.info("Finished.")
