def main():
#Read configuration...
execfile('config.py', globals())
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Build only the specified package")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(options.verbose)
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
for app in apps:
print "Writing " + app['id']
common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
print "Finished."
def main():
# Read configuration...
execfile('config.py', globals())
import common
# Parse command line...
parser = OptionParser()
parser.add_option("-u", "--url", default=None,
help="Project URL to import from.")
parser.add_option("-s", "--subdir", default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option("-r", "--repo", default=None,
help="Allows a different repo to be specified for a multi-repo google code project")
(options, args) = parser.parse_args()
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = common.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
if url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/';
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir, sdk_path)
vcs.gotorevision(None)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Check AndroidManiifest.xml exists...
if not os.path.exists(root_dir + '/AndroidManifest.xml'):
print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1)
# Extract some information...
version, vercode, package = common.parse_androidmanifest(root_dir)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "Couldn't find latest version name"
sys.exit(1)
if not vercode:
print "Couldn't find latest version code"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = common.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
# Create a build line...
build = {}
build['version'] = version
build['vercode'] = vercode
build['commit'] = '?'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
app['comments'].append(('build:' + version,
"#Generated by import.py - check this is the right version, and find the right commit!"))
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
metafile = os.path.join('metadata', package + '.txt')
common.write_metadata(metafile, app)
print "Wrote " + metafile
def main():
global options
# Read configuration...
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
globals()['archive_older'] = 0
execfile('config.py', globals())
options, args = parse_commandline()
if build_server_always:
options.server = True
if options.resetserver and not options.server:
print "Using --resetserver without --server makes no sense"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose, xref=not options.onserver)
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
if archive_older != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Filter apps and build versions according to command-line options, etc...
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
apps = [
app for app in apps
if (options.force or not app['Disabled']) and app['builds']
and len(app['Repo Type']) > 0 and len(app['builds']) > 0
]
if len(apps) == 0:
print "Nothing to do - all apps are disabled or have no builds defined."
sys.exit(1)
if options.vercode:
for app in apps:
app['builds'] = [
b for b in app['builds']
if str(b['vercode']) == options.vercode
]
elif options.latest:
for app in apps:
m = max([i['vercode'] for i in app['builds']], key=int)
app['builds'] = [b for b in app['builds'] if b['vercode'] == m]
if options.wiki:
import mwclient
site = mwclient.Site(wiki_server, path=wiki_path)
site.login(wiki_user, wiki_password)
# Build applications...
failed_apps = {}
build_succeeded = []
for app in apps:
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib',
app['Repo'])
else:
build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code...
if options.verbose:
print "Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'],
build_dir, sdk_path)
first = False
if options.verbose:
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir,
also_check_dir, srclib_dir, extlib_dir, tmp_dir,
repo_dir, vcs, options.test, options.server,
options.install, options.force, options.verbose,
options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print "Could not build app %s due to BuildException: %s" % (
app['id'], be)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = vcse
wikilog = str(vcse)
except Exception as e:
print "Could not build app %s due to unknown error: %s" % (
app['id'], traceback.format_exc())
if options.stop:
sys.exit(1)
failed_apps[app['id']] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
newpage = site.Pages[app['id'] + '/lastbuild']
txt = wikilog
if len(txt) > 8192:
txt = txt[-8192:]
txt = "Build completed at " + time.strftime(
"%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(wikilog, summary='Build log')
except:
print "Error while attempting to publish build log"
for app in build_succeeded:
print "success: %s" % (app['id'])
for fa in failed_apps:
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
print "Finished."
if len(build_succeeded) > 0:
print str(len(build_succeeded)) + ' builds succeeded'
if len(failed_apps) > 0:
print str(len(failed_apps)) + ' builds failed'
sys.exit(0)
def main():
# Read configuration...
global update_stats, stats_to_carbon
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats:
print "Stats are disabled - check your configuration"
sys.exit(1)
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
(options, args) = parser.parse_args()
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
print 'Retrieving logs'
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='******', timeout=10,
key_filename=webserver_keyfile)
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
print "...connected"
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e:
traceback.print_exc()
sys.exit(1)
finally:
#Disconnect
if ftp != None:
ftp.close()
if ssh != None:
ssh.close()
# Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
apps = {}
unknownapks = []
knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if match and match.group('statuscode') == '200':
uri = match.group('uri')
if uri.endswith('.apk'):
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
if appid in apps:
apps[appid] += 1
else:
apps[appid] = 1
else:
if not apkname in unknownapks:
unknownapks.append(apkname)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for app, count in apps.iteritems():
lst.append(app + " " + str(count))
if stats_to_carbon:
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
# Calculate and write stats for repo types...
repotypes = {}
for app in metaapps:
if len(app['Repo Type']) == 0:
rtype = 'none'
else:
if app['Repo Type'] == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
else:
rtype = app['Repo Type']
if rtype in repotypes:
repotypes[rtype] += 1;
else:
repotypes[rtype] = 1
f = open('stats/repotypes.txt', 'w')
for rtype, count in repotypes.iteritems():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
ucms = {}
for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0]
if checkmode in ucms:
ucms[checkmode] += 1;
else:
ucms[checkmode] = 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode, count in ucms.iteritems():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
licenses = {}
for app in metaapps:
license = app['License']
if license in licenses:
licenses[license] += 1;
else:
licenses[license] = 1
f = open('stats/licenses.txt', 'w')
for license, count in licenses.iteritems():
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if len(unknownapks) > 0:
print '\nUnknown apks:'
for apk in unknownapks:
print apk
print "Finished."
def main():
# Read configuration...
global update_stats, archive_older
update_stats = False
archive_older = 0
execfile('config.py', globals())
# Parse command line...
global options
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="No output, except for warnings and errors")
parser.add_option("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status")
parser.add_option("-i", "--interactive", default=False, action="store_true",
help="Interactively ask about things that need updating.")
parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default "+
"is /etc/alternatives/editor")
parser.add_option("-w", "--wiki", default=False, action="store_true",
help="Update the wiki")
parser.add_option("", "--pretty", action="store_true", default=False,
help="Produce human-readable index.xml")
parser.add_option("--clean", action="store_true", default=False,
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(verbose=options.verbose)
# Generate a list of categories...
categories = []
for app in apps:
cats = app['Category'].split(';')
for cat in cats:
if cat not in categories:
categories.append(cat)
# Read known apks data (will be updated and written back when we've finished)
knownapks = common.KnownApks()
# Gather information about all the apk files in the repo directory, using
# cached data if possible.
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf)
else:
apkcache = {}
cachechanged = False
repodirs = ['repo']
if archive_older != 0:
repodirs.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
delete_disabled_builds(apps, apkcache, repodirs)
apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
if cc:
cachechanged = True
# Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
for app in apps:
bestver = 0
added = None
lastupdated = None
for apk in apks:
if apk['id'] == app['id']:
if apk['versioncode'] > bestver:
bestver = apk['versioncode']
bestapk = apk
if 'added' in apk:
if not added or apk['added'] < added:
added = apk['added']
if not lastupdated or apk['added'] > lastupdated:
lastupdated = apk['added']
if added:
app['added'] = added
else:
print "WARNING: Don't know when " + app['id'] + " was added"
if lastupdated:
app['lastupdated'] = lastupdated
else:
print "WARNING: Don't know when " + app['id'] + " was last updated"
if bestver == 0:
if app['Name'] is None:
app['Name'] = app['id']
app['icon'] = ''
if app['Disabled'] is None:
print "WARNING: Application " + app['id'] + " has no packages"
else:
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon']
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
apps = sorted(apps, key=lambda app: app['Name'].upper())
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
for apk in apks:
found = False
for app in apps:
if app['id'] == apk['id']:
found = True
break
if not found:
if options.createmeta:
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
f.write(".\n")
f.close()
print "Generated skeleton metadata for " + apk['id']
else:
print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
print " " + apk['name'] + " - " + apk['version']
if len(repodirs) > 1:
archive_old_apks(apps, apks, repodirs[0], repodirs[1], archive_older)
# Make the index for the main repo...
make_index(apps, apks, repodirs[0], False, categories)
# If there's an archive repo, scan the apks for that and make the index...
archapks = None
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
if cc:
cachechanged = True
make_index(apps, archapks, repodirs[1], True, categories)
if update_stats:
# Update known apks info...
knownapks.writeifchanged()
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip()
data += appid + "\t"
for app in apps:
if app['id'] == appid:
data += app['Name'] + "\t"
data += app['icon'] + "\t"
data += app['License'] + "\n"
break
f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
f.write(data)
f.close()
if cachechanged:
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
# Update the wiki...
if options.wiki:
if archapks:
apks.extend(archapks)
update_wiki(apps, apks, options.verbose)
print "Finished."
def main():
# Read configuration...
execfile('config.py', globals())
import common
# Parse command line...
parser = OptionParser()
parser.add_option("-u",
"--url",
default=None,
help="Project URL to import from.")
parser.add_option(
"-s",
"--subdir",
default=None,
help="Path to main android project subdirectory, if not in root.")
parser.add_option(
"-r",
"--repo",
default=None,
help=
"Allows a different repo to be specified for a multi-repo google code project"
)
(options, args) = parser.parse_args()
if not options.url:
print "Specify project url."
sys.exit(1)
url = options.url
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
# Get all apps...
apps = common.read_metadata()
# Figure out what kind of project it is...
projecttype = None
issuetracker = None
license = None
website = url #by default, we might override it
if url.startswith('git://'):
projecttype = 'git'
repo = url
repotype = 'git'
sourcecode = ""
website = ""
if url.startswith('https://github.com'):
if url.endswith('/'):
url = url[:-1]
if url.endswith('.git'):
print "A github URL should point to the project, not the git repo"
sys.exit(1)
projecttype = 'github'
repo = url + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://gitorious.org/'):
projecttype = 'gitorious'
repo = 'https://git.gitorious.org/' + url[22:] + '.git'
repotype = 'git'
sourcecode = url
elif url.startswith('https://bitbucket.org/'):
if url.endswith('/'):
url = url[:-1]
projecttype = 'bitbucket'
sourcecode = url + '/src'
issuetracker = url + '/issues'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
elif url.startswith('http://code.google.com/p/'):
if not url.endswith('/'):
url += '/'
projecttype = 'googlecode'
sourcecode = url + 'source/checkout'
if options.repo:
sourcecode += "?repo=" + options.repo
issuetracker = url + 'issues/list'
# Figure out the repo type and adddress...
repotype, repo = getrepofrompage(sourcecode)
if not repotype:
print "Unable to determine vcs type. " + repo
sys.exit(1)
# Figure out the license...
req = urllib.urlopen(url)
if req.getcode() != 200:
print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(
req.getcode())
sys.exit(1)
page = req.read()
index = page.find('Code license')
if index == -1:
print "Couldn't find license data"
sys.exit(1)
ltext = page[index:]
lprefix = 'rel="nofollow">'
index = ltext.find(lprefix)
if index == -1:
print "Couldn't find license text"
sys.exit(1)
ltext = ltext[index + len(lprefix):]
index = ltext.find('<')
if index == -1:
print "License text not formatted as expected"
sys.exit(1)
ltext = ltext[:index]
if ltext == 'GNU GPL v3':
license = 'GPLv3'
elif ltext == 'GNU GPL v2':
license = 'GPLv2'
elif ltext == 'Apache License 2.0':
license = 'Apache2'
elif ltext == 'MIT License':
license = 'MIT'
elif ltext == 'GNU Lesser GPL':
license = 'LGPL'
elif ltext == 'Mozilla Public License 1.1':
license = 'MPL'
elif ltext == 'New BSD License':
license = 'NewBSD'
else:
print "License " + ltext + " is not recognised"
sys.exit(1)
if not projecttype:
print "Unable to determine the project type."
print "The URL you supplied was not in one of the supported formats. Please consult"
print "the manual for a list of supported formats, and supply one of those."
sys.exit(1)
# Get a copy of the source so we can extract some info...
print 'Getting source from ' + repotype + ' repo at ' + repo
src_dir = os.path.join(tmp_dir, 'importer')
if os.path.exists(src_dir):
shutil.rmtree(src_dir)
vcs = common.getvcs(repotype, repo, src_dir, sdk_path)
vcs.gotorevision(None)
if options.subdir:
root_dir = os.path.join(src_dir, options.subdir)
else:
root_dir = src_dir
# Check AndroidManiifest.xml exists...
if not os.path.exists(root_dir + '/AndroidManifest.xml'):
print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
sys.exit(1)
# Extract some information...
version, vercode, package = common.parse_androidmanifest(root_dir)
if not package:
print "Couldn't find package ID"
sys.exit(1)
if not version:
print "Couldn't find latest version name"
sys.exit(1)
if not vercode:
print "Couldn't find latest version code"
sys.exit(1)
# Make sure it's actually new...
for app in apps:
if app['id'] == package:
print "Package " + package + " already exists"
sys.exit(1)
# Construct the metadata...
app = common.parse_metadata(None)
app['id'] = package
app['Web Site'] = website
app['Source Code'] = sourcecode
if issuetracker:
app['Issue Tracker'] = issuetracker
if license:
app['License'] = license
app['Repo Type'] = repotype
app['Repo'] = repo
# Create a build line...
build = {}
build['version'] = version
build['vercode'] = vercode
build['commit'] = '?'
if options.subdir:
build['subdir'] = options.subdir
if os.path.exists(os.path.join(root_dir, 'jni')):
build['buildjni'] = 'yes'
app['builds'].append(build)
app['comments'].append((
'build:' + version,
"#Generated by import.py - check this is the right version, and find the right commit!"
))
# Keep the repo directory to save bandwidth...
if not os.path.exists('build'):
os.mkdir('build')
shutil.move(src_dir, os.path.join('build', package))
metafile = os.path.join('metadata', package + '.txt')
common.write_metadata(metafile, app)
print "Wrote " + metafile
def main():
global options
# Read configuration...
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
globals()['archive_older'] = 0
execfile('config.py', globals())
options, args = parse_commandline()
if build_server_always:
options.server = True
if options.resetserver and not options.server:
print "Using --resetserver without --server makes no sense"
sys.exit(1)
# Get all apps...
apps = common.read_metadata(options.verbose, xref=not options.onserver)
log_dir = 'logs'
if not os.path.isdir(log_dir):
print "Creating log directory"
os.makedirs(log_dir)
tmp_dir = 'tmp'
if not os.path.isdir(tmp_dir):
print "Creating temporary directory"
os.makedirs(tmp_dir)
if options.test:
output_dir = tmp_dir
else:
output_dir = 'unsigned'
if not os.path.isdir(output_dir):
print "Creating output directory"
os.makedirs(output_dir)
if archive_older != 0:
also_check_dir = 'archive'
else:
also_check_dir = None
repo_dir = 'repo'
build_dir = 'build'
if not os.path.isdir(build_dir):
print "Creating build directory"
os.makedirs(build_dir)
srclib_dir = os.path.join(build_dir, 'srclib')
extlib_dir = os.path.join(build_dir, 'extlib')
# Filter apps and build versions according to command-line options, etc...
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
apps = [app for app in apps if (options.force or not app['Disabled']) and
app['builds'] and len(app['Repo Type']) > 0 and len(app['builds']) > 0]
if len(apps) == 0:
print "Nothing to do - all apps are disabled or have no builds defined."
sys.exit(1)
if options.vercode:
for app in apps:
app['builds'] = [b for b in app['builds']
if str(b['vercode']) == options.vercode]
elif options.latest:
for app in apps:
m = max([i['vercode'] for i in app['builds']], key=int)
app['builds'] = [b for b in app['builds'] if b['vercode'] == m]
if options.wiki:
import mwclient
site = mwclient.Site(wiki_server, path=wiki_path)
site.login(wiki_user, wiki_password)
# Build applications...
failed_apps = {}
build_succeeded = []
for app in apps:
first = True
for thisbuild in app['builds']:
wikilog = None
try:
# For the first build of a particular app, we need to set up
# the source repo. We can reuse it on subsequent builds, if
# there are any.
if first:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
else:
build_dir = os.path.join('build', app['id'])
# Set up vcs interface and make sure we have the latest code...
if options.verbose:
print "Getting {0} vcs interface for {1}".format(
app['Repo Type'], app['Repo'])
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
first = False
if options.verbose:
print "Checking " + thisbuild['version']
if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir,
srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test,
options.server, options.install, options.force,
options.verbose, options.onserver):
build_succeeded.append(app)
wikilog = "Build succeeded"
except BuildException as be:
logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+')
logfile.write(str(be))
logfile.close()
print "Could not build app %s due to BuildException: %s" % (app['id'], be)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = be
wikilog = be.get_wikitext()
except VCSException as vcse:
print "VCS error while building app %s: %s" % (app['id'], vcse)
if options.stop:
sys.exit(1)
failed_apps[app['id']] = vcse
wikilog = str(vcse)
except Exception as e:
print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
if options.stop:
sys.exit(1)
failed_apps[app['id']] = e
wikilog = str(e)
if options.wiki and wikilog:
try:
newpage = site.Pages[app['id'] + '/lastbuild']
txt = wikilog
if len(txt) > 8192:
txt = txt[-8192:]
txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt
newpage.save(wikilog, summary='Build log')
except:
print "Error while attempting to publish build log"
for app in build_succeeded:
print "success: %s" % (app['id'])
for fa in failed_apps:
print "Build for app %s failed:\n%s" % (fa, failed_apps[fa])
print "Finished."
if len(build_succeeded) > 0:
print str(len(build_succeeded)) + ' builds succeeded'
if len(failed_apps) > 0:
print str(len(failed_apps)) + ' builds failed'
sys.exit(0)
def main():
# Read configuration...
global update_stats, stats_to_carbon
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats:
print "Stats are disabled - check your configuration"
sys.exit(1)
# Parse command line...
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-d",
"--download",
action="store_true",
default=False,
help="Download logs we don't have")
(options, args) = parser.parse_args()
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
print 'Retrieving logs'
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org',
username='******',
timeout=10,
key_filename=webserver_keyfile)
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
print "...connected"
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath)
or os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e:
traceback.print_exc()
sys.exit(1)
finally:
#Disconnect
if ftp != None:
ftp.close()
if ssh != None:
ssh.close()
# Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
apps = {}
unknownapks = []
knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if match and match.group('statuscode') == '200':
uri = match.group('uri')
if uri.endswith('.apk'):
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
if appid in apps:
apps[appid] += 1
else:
apps[appid] = 1
else:
if not apkname in unknownapks:
unknownapks.append(apkname)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for app, count in apps.iteritems():
lst.append(app + " " + str(count))
if stats_to_carbon:
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
# Calculate and write stats for repo types...
repotypes = {}
for app in metaapps:
if len(app['Repo Type']) == 0:
rtype = 'none'
else:
if app['Repo Type'] == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
else:
rtype = app['Repo Type']
if rtype in repotypes:
repotypes[rtype] += 1
else:
repotypes[rtype] = 1
f = open('stats/repotypes.txt', 'w')
for rtype, count in repotypes.iteritems():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
ucms = {}
for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0]
if checkmode in ucms:
ucms[checkmode] += 1
else:
ucms[checkmode] = 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode, count in ucms.iteritems():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
licenses = {}
for app in metaapps:
license = app['License']
if license in licenses:
licenses[license] += 1
else:
licenses[license] = 1
f = open('stats/licenses.txt', 'w')
for license, count in licenses.iteritems():
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if len(unknownapks) > 0:
print '\nUnknown apks:'
for apk in unknownapks:
print apk
print "Finished."
def main():
#Read configuration...
execfile('config.py', globals())
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Check only the specified package")
parser.add_option("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_option("--commit", action="store_true", default=False,
help="Commit changes")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(options.verbose)
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
for app in apps:
print "Processing " + app['id'] + '...'
writeit = False
logmsg = None
mode = app['Update Check Mode']
if mode == 'Market':
(version, vercode) = check_market(app)
elif mode == 'Tags':
(version, vercode) = check_tags(app, sdk_path)
elif mode == 'RepoManifest':
(version, vercode) = check_repomanifest(app, sdk_path)
elif mode.startswith('RepoManifest/'):
(version, vercode) = check_repomanifest(app, sdk_path, mode[13:])
elif mode == 'Static':
version = None
vercode = 'Checking disabled'
elif mode == 'None':
version = None
vercode = 'Checking disabled'
else:
version = None
vercode = 'Invalid update check method'
if not version:
print "..." + vercode
elif vercode == app['Current Version Code'] and version == app['Current Version']:
print "...up to date"
else:
print '...updating to version:' + version + ' vercode:' + vercode
app['Current Version'] = version
app['Current Version Code'] = str(int(vercode))
writeit = True
logmsg = "Update current version of " + app['id'] + " to " + version
if options.auto:
mode = app['Auto Update Mode']
if mode == 'None':
pass
elif mode.startswith('Version '):
pattern = mode[8:]
if pattern.startswith('+'):
o = pattern.find(' ')
suffix = pattern[1:o]
pattern = pattern[o + 1:]
else:
suffix = ''
gotcur = False
latest = None
for build in app['builds']:
if build['vercode'] == app['Current Version Code']:
gotcur = True
if not latest or int(build['vercode']) > int(latest['vercode']):
latest = build
if not gotcur:
newbuild = latest.copy()
del newbuild['origlines']
newbuild['vercode'] = app['Current Version Code']
newbuild['version'] = app['Current Version'] + suffix
print "...auto-generating build for " + newbuild['version']
commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit
app['builds'].append(newbuild)
writeit = True
logmsg = "Update " + app['id'] + " to " + newbuild['version']
else:
print 'Invalid auto update mode'
if writeit:
metafile = os.path.join('metadata', app['id'] + '.txt')
common.write_metadata(metafile, app)
if options.commit:
if subprocess.call("git add " + metafile, shell=True) != 0:
print "Git add failed"
sys.exit(1)
if subprocess.call("git commit -m '" + logmsg.replace("'", "\\'") + "'", shell=True) != 0:
print "Git commit failed"
sys.exit(1)
print "Finished."
def main():
# Read configuration...
global build_server_always, mvn3
globals()['build_server_always'] = False
globals()['mvn3'] = "mvn3"
execfile('config.py', globals())
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-p", "--package", default=None,
help="Scan only the specified package")
parser.add_option("--nosvn", action="store_true", default=False,
help="Skip svn repositories - for test purposes, because they are too slow.")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(options.verbose)
# Filter apps according to command-line options
if options.package:
apps = [app for app in apps if app['id'] == options.package]
if len(apps) == 0:
print "No such package"
sys.exit(1)
html_parser = HTMLParser.HTMLParser()
problems = []
extlib_dir = os.path.join('build', 'extlib')
for app in apps:
skip = False
if app['Disabled']:
print "Skipping %s: disabled" % app['id']
skip = True
elif not app['builds']:
print "Skipping %s: no builds specified" % app['id']
skip = True
elif options.nosvn and app['Repo Type'] == 'svn':
skip = True
if not skip:
print "Processing " + app['id']
try:
build_dir = 'build/' + app['id']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
for thisbuild in app['builds']:
if thisbuild['commit'].startswith('!'):
print ("..skipping version " + thisbuild['version'] + " - " +
thisbuild['commit'][1:])
else:
print "..scanning version " + thisbuild['version']
# Prepare the source code...
root_dir, _ = common.prepare_source(vcs, app, thisbuild,
build_dir, extlib_dir, sdk_path, ndk_path, javacc_path, mvn3, options.verbose)
# Do the scan...
buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
for problem in buildprobs:
problems.append(problem +
' in ' + app['id'] + ' ' + thisbuild['version'])
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
problems.append(msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
problems.append(msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
problems.append(msg)
print "Finished:"
for problem in problems:
print problem
print str(len(problems)) + ' problems.'