def check_repotrunk(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('svn', 'git-svn'):
return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def check_repotrunk(app, branch=None):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
if repotype not in ('git-svn', ):
return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
def check_repotrunk(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git-svn', ):
return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def check_tags(app):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavour = None
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if 'gradle' in app['builds'][-1]:
flavour = app['builds'][-1]['gradle']
htag = None
hver = None
hcode = "0"
for tag in vcs.gettags():
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavour)
version, vercode, package = common.parse_androidmanifests(paths)
if package and package == app['id'] and version and vercode:
print "Manifest exists. Found version %s (%s)" % (
version, vercode)
if int(vercode) > int(hcode):
htag = tag
hcode = str(int(vercode))
hver = version
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg, None)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg, None)
def check_tags(app, sdk_path):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib')
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn'):
return (None, 'Tags update mode only works for git and git-svn repositories currently')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
if app['Repo Type'] == 'srclib':
build_dir = os.path.join(build_dir, app['Repo'])
vcs.gotorevision(None)
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
hver = None
hcode = "0"
for tag in vcs.gettags():
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
if os.path.exists(os.path.join(build_dir, 'AndroidManifest.xml')):
version, vercode, package = common.parse_androidmanifest(build_dir)
if package and package == app['id'] and version and vercode:
if int(vercode) > int(hcode):
hcode = str(int(vercode))
hver = version
if hver:
return (hver, hcode)
return (None, "Couldn't find any version information")
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def check_repomanifest(app, branch=None):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
last_build = metadata.Build()
if len(app.builds) > 0:
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
hver = None
hcode = "0"
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
if int(vercode) > int(hcode):
hpak = package
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID")
if hver:
return (hver, hcode)
return (None, "Couldn't find any version information")
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
def check_tags(app, pattern):
try:
appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app['Repo']:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
if repotype in ('git',):
tags = vcs.latesttags(tags, 5)
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package or package != appid or not version or not vercode:
continue
logging.debug("Manifest exists. Found version {0} ({1})"
.format(version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg, None)
def check_repomanifest(app, branch=None):
try:
appid = app['Update Check Name'] or app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
if package != appid:
return (None, "Package ID mismatch - expected {0}, got {1}"
.format(appid, package))
if not version:
return (None, "Couldn't find latest version name")
if not vercode:
if "Ignore" == version:
return (None, "Latest version is ignored")
return (None, "Couldn't find latest version code")
vercode = str(int(vercode))
logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
return (version, vercode)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def main():
# Read configuration...
global update_stats, stats_to_carbon
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats:
print "Stats are disabled - check your configuration"
sys.exit(1)
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
(options, args) = parser.parse_args()
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
print 'Retrieving logs'
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='******', timeout=10,
key_filename=webserver_keyfile)
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
print "...connected"
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e:
traceback.print_exc()
sys.exit(1)
finally:
#Disconnect
if ftp != None:
ftp.close()
if ssh != None:
ssh.close()
# Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
apps = {}
unknownapks = []
knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir,'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout = subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if match and match.group('statuscode') == '200':
uri = match.group('uri')
if uri.endswith('.apk'):
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
if appid in apps:
apps[appid] += 1
else:
apps[appid] = 1
else:
if not apkname in unknownapks:
unknownapks.append(apkname)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for app, count in apps.iteritems():
lst.append(app + " " + str(count))
if stats_to_carbon:
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
# Calculate and write stats for repo types...
repotypes = {}
for app in metaapps:
if len(app['Repo Type']) == 0:
rtype = 'none'
else:
if app['Repo Type'] == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
else:
rtype = app['Repo Type']
if rtype in repotypes:
repotypes[rtype] += 1;
else:
repotypes[rtype] = 1
f = open('stats/repotypes.txt', 'w')
for rtype, count in repotypes.iteritems():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
ucms = {}
for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0]
if checkmode in ucms:
ucms[checkmode] += 1;
else:
ucms[checkmode] = 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode, count in ucms.iteritems():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
licenses = {}
for app in metaapps:
license = app['License']
if license in licenses:
licenses[license] += 1;
else:
licenses[license] = 1
f = open('stats/licenses.txt', 'w')
for license, count in licenses.iteritems():
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if len(unknownapks) > 0:
print '\nUnknown apks:'
for apk in unknownapks:
print apk
print "Finished."
def check_tags(app, pattern):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app['Repo']:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
logging.debug("Matching tags: " + ','.join(tags))
if repotype in ('git',):
tags = vcs.latesttags(tags, 5)
logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not app_matches_packagename(app, package) or not version or not vercode:
continue
logging.debug("Manifest exists. Found version {0} ({1})"
.format(version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg, None)
def check_repomanifest(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavours = []
if len(app['builds']) > 0:
if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if app['builds'][-1]['gradle']:
flavours = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavours)
version, vercode, package = \
common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
if not app_matches_packagename(app, package):
return (None, "Package ID mismatch - got {0}".format(package))
if not version:
return (None, "Couldn't find latest version name")
if not vercode:
if "Ignore" == version:
return (None, "Latest version is ignored")
return (None, "Couldn't find latest version code")
vercode = str(int(vercode))
logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
return (version, vercode)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
def main():
# Read configuration...
global update_stats, stats_to_carbon
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats:
print "Stats are disabled - check your configuration"
sys.exit(1)
# Parse command line...
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-d",
"--download",
action="store_true",
default=False,
help="Download logs we don't have")
(options, args) = parser.parse_args()
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
print 'Retrieving logs'
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org',
username='******',
timeout=10,
key_filename=webserver_keyfile)
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
print "...connected"
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath)
or os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e:
traceback.print_exc()
sys.exit(1)
finally:
#Disconnect
if ftp != None:
ftp.close()
if ssh != None:
ssh.close()
# Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
apps = {}
unknownapks = []
knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if match and match.group('statuscode') == '200':
uri = match.group('uri')
if uri.endswith('.apk'):
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
if appid in apps:
apps[appid] += 1
else:
apps[appid] = 1
else:
if not apkname in unknownapks:
unknownapks.append(apkname)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for app, count in apps.iteritems():
lst.append(app + " " + str(count))
if stats_to_carbon:
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
# Calculate and write stats for repo types...
repotypes = {}
for app in metaapps:
if len(app['Repo Type']) == 0:
rtype = 'none'
else:
if app['Repo Type'] == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
else:
rtype = app['Repo Type']
if rtype in repotypes:
repotypes[rtype] += 1
else:
repotypes[rtype] = 1
f = open('stats/repotypes.txt', 'w')
for rtype, count in repotypes.iteritems():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
ucms = {}
for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0]
if checkmode in ucms:
ucms[checkmode] += 1
else:
ucms[checkmode] = 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode, count in ucms.iteritems():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
licenses = {}
for app in metaapps:
license = app['License']
if license in licenses:
licenses[license] += 1
else:
licenses[license] = 1
f = open('stats/licenses.txt', 'w')
for license, count in licenses.iteritems():
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if len(unknownapks) > 0:
print '\nUnknown apks:'
for apk in unknownapks:
print apk
print "Finished."
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
parser.add_option("--recalc", action="store_true", default=False,
help="Recalculate aggregate stats - use when changes "
"have been made that would invalidate old cached data.")
parser.add_option("--nologs", action="store_true", default=False,
help="Don't do anything logs-related")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not config['update_stats']:
logging.info("Stats are disabled - check your configuration")
sys.exit(1)
# Get all metadata-defined apps...
metaapps = [a for a in metadata.read_metadata().itervalues() if not a['Disabled']]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
logging.info('Retrieving logs')
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='******', timeout=10,
key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
logging.info("...connected")
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
logging.debug("...retrieving " + f)
ftp.get(f, destpath)
except Exception:
traceback.print_exc()
sys.exit(1)
finally:
# Disconnect
if ftp is not None:
ftp.close()
if ssh is not None:
ssh.close()
knownapks = common.KnownApks()
unknownapks = []
if not options.nologs:
# Process logs
logging.info('Processing logs...')
appscount = Counter()
appsvercount = Counter()
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
logging.debug('...' + logfile)
# Get the date for this log - e.g. 2012-02-28
thisdate = os.path.basename(logfile)[7:-7]
agg_path = os.path.join(datadir, thisdate + '.json')
if not options.recalc and os.path.exists(agg_path):
# Use previously calculated aggregate data
with open(agg_path, 'r') as f:
today = json.load(f)
else:
# Calculate from logs...
today = {
'apps': Counter(),
'appsver': Counter(),
'unknown': []
}
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if not match:
continue
if match.group('statuscode') != '200':
continue
if match.group('ip') in config['stats_ignore']:
continue
uri = match.group('uri')
if not uri.endswith('.apk'):
continue
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
today['apps'][appid] += 1
# Strip the '.apk' from apkname
appver = apkname[:-4]
today['appsver'][appver] += 1
else:
if apkname not in today['unknown']:
today['unknown'].append(apkname)
# Save calculated aggregate data for today to cache
with open(agg_path, 'w') as f:
json.dump(today, f)
# Add today's stats (whether cached or recalculated) to the total
for appid in today['apps']:
appscount[appid] += today['apps'][appid]
for appid in today['appsver']:
appsvercount[appid] += today['appsver'][appid]
for uk in today['unknown']:
if uk not in unknownapks:
unknownapks.append(uk)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for appid in appscount:
count = appscount[appid]
lst.append(appid + " " + str(count))
if config['stats_to_carbon']:
carbon_send('fdroid.download.' + appid.replace('.', '_'),
count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
f = open('stats/total_downloads_app_version.txt', 'w')
f.write('# Total downloads by application and version, '
'since October 2011\n')
lst = []
for appver in appsvercount:
count = appsvercount[appver]
lst.append(appver + " " + str(count))
for line in sorted(lst):
f.write(line + "\n")
f.close()
# Calculate and write stats for repo types...
logging.info("Processing repo types...")
repotypes = Counter()
for app in metaapps:
rtype = app['Repo Type'] or 'none'
if rtype == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
repotypes[rtype] += 1
f = open('stats/repotypes.txt', 'w')
for rtype in repotypes:
count = repotypes[rtype]
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
logging.info("Processing update check modes...")
ucms = Counter()
for app in metaapps:
checkmode = app['Update Check Mode']
if checkmode.startswith('RepoManifest/'):
checkmode = checkmode[:12]
if checkmode.startswith('Tags '):
checkmode = checkmode[:4]
ucms[checkmode] += 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode in ucms:
count = ucms[checkmode]
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
logging.info("Processing categories...")
ctgs = Counter()
for app in metaapps:
for category in app['Categories']:
ctgs[category] += 1
f = open('stats/categories.txt', 'w')
for category in ctgs:
count = ctgs[category]
f.write(category + ' ' + str(count) + '\n')
f.close()
logging.info("Processing antifeatures...")
afs = Counter()
for app in metaapps:
if app['AntiFeatures'] is None:
continue
antifeatures = [a.strip() for a in app['AntiFeatures'].split(',')]
for antifeature in antifeatures:
afs[antifeature] += 1
f = open('stats/antifeatures.txt', 'w')
for antifeature in afs:
count = afs[antifeature]
f.write(antifeature + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
logging.info("Processing licenses...")
licenses = Counter()
for app in metaapps:
license = app['License']
licenses[license] += 1
f = open('stats/licenses.txt', 'w')
for license in licenses:
count = licenses[license]
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if unknownapks:
logging.info('\nUnknown apks:')
for apk in unknownapks:
logging.info(apk)
logging.info("Finished.")
def check_repomanifest(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
branch = 'origin/'+branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
elif repotype == 'svn':
vcs.gotorevision(None)
elif repotype == 'hg':
vcs.gotorevision(branch)
elif repotype == 'bzr':
vcs.gotorevision(None)
flavour = None
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
if 'gradle' in app['builds'][-1]:
flavour = app['builds'][-1]['gradle']
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavour)
version, vercode, package = common.parse_androidmanifests(paths)
if not package:
return (None, "Couldn't find package ID")
if package != app['id']:
return (None, "Package ID mismatch")
if not version:
return (None,"Couldn't find latest version name")
if not vercode:
return (None,"Couldn't find latest version code")
return (version, str(int(vercode)))
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
def check_tags(app, pattern):
try:
if app.RepoType == 'srclib':
build_dir = os.path.join('build', 'srclib', app.Repo)
repotype = common.getsrclibvcs(app.Repo)
else:
build_dir = os.path.join('build', app.id)
repotype = app.RepoType
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
if repotype == 'git-svn' and ';' not in app.Repo:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
last_build = metadata.Build()
if len(app.builds) > 0:
last_build = app.builds[-1]
if last_build.submodules:
vcs.initsubmodules()
hpak = None
htag = None
hver = None
hcode = "0"
tags = vcs.gettags()
if not tags:
return (None, "No tags found", None)
logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
if not tags:
return (None, "No matching tags found", None)
logging.debug("Matching tags: " + ','.join(tags))
if len(tags) > 5 and repotype in ('git',):
tags = vcs.latesttags(tags, 5)
logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
for subdir in possible_subdirs(app):
if subdir == '.':
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
paths = common.manifest_paths(root_dir, last_build.gradle)
version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
if int(vercode) > int(hcode):
hpak = package
htag = tag
hcode = str(int(vercode))
hver = version
if not hpak:
return (None, "Couldn't find package ID", None)
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg, None)
except Exception:
msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg, None)
def main():
global options, config
# Parse command line...
parser = ArgumentParser()
common.setup_global_opts(parser)
parser.add_argument("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
parser.add_argument("--recalc", action="store_true", default=False,
help="Recalculate aggregate stats - use when changes "
"have been made that would invalidate old cached data.")
parser.add_argument("--nologs", action="store_true", default=False,
help="Don't do anything logs-related")
options = parser.parse_args()
config = common.read_config(options)
if not config['update_stats']:
logging.info("Stats are disabled - set \"update_stats = True\" in your config.py")
sys.exit(1)
# Get all metadata-defined apps...
allmetaapps = [app for app in metadata.read_metadata().itervalues()]
metaapps = [app for app in allmetaapps if not app.Disabled]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
logging.info('Retrieving logs')
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(config['stats_server'], username=config['stats_user'],
timeout=10, key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
logging.info("...connected")
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
logging.debug("...retrieving " + f)
ftp.get(f, destpath)
except Exception:
traceback.print_exc()
sys.exit(1)
finally:
# Disconnect
if ftp is not None:
ftp.close()
if ssh is not None:
ssh.close()
knownapks = common.KnownApks()
unknownapks = []
if not options.nologs:
# Process logs
logging.info('Processing logs...')
appscount = Counter()
appsvercount = Counter()
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
logging.debug('...' + logfile)
# Get the date for this log - e.g. 2012-02-28
thisdate = os.path.basename(logfile)[7:-7]
agg_path = os.path.join(datadir, thisdate + '.json')
if not options.recalc and os.path.exists(agg_path):
# Use previously calculated aggregate data
with open(agg_path, 'r') as f:
today = json.load(f)
else:
# Calculate from logs...
today = {
'apps': Counter(),
'appsver': Counter(),
'unknown': []
}
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if not match:
continue
if match.group('statuscode') != '200':
continue
if match.group('ip') in config['stats_ignore']:
continue
uri = match.group('uri')
if not uri.endswith('.apk'):
continue
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
today['apps'][appid] += 1
# Strip the '.apk' from apkname
appver = apkname[:-4]
today['appsver'][appver] += 1
else:
if apkname not in today['unknown']:
today['unknown'].append(apkname)
# Save calculated aggregate data for today to cache
with open(agg_path, 'w') as f:
json.dump(today, f)
# Add today's stats (whether cached or recalculated) to the total
for appid in today['apps']:
appscount[appid] += today['apps'][appid]
for appid in today['appsver']:
appsvercount[appid] += today['appsver'][appid]
for uk in today['unknown']:
if uk not in unknownapks:
unknownapks.append(uk)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for appid in appscount:
count = appscount[appid]
lst.append(appid + " " + str(count))
if config['stats_to_carbon']:
carbon_send('fdroid.download.' + appid.replace('.', '_'),
count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
with open(os.path.join(statsdir, 'total_downloads_app.txt'), 'w') as f:
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
lst = []
for appver in appsvercount:
count = appsvercount[appver]
lst.append(appver + " " + str(count))
with open(os.path.join(statsdir, 'total_downloads_app_version.txt'), 'w') as f:
f.write('# Total downloads by application and version, '
'since October 2011\n')
for line in sorted(lst):
f.write(line + "\n")
# Calculate and write stats for repo types...
logging.info("Processing repo types...")
repotypes = Counter()
for app in metaapps:
rtype = app.RepoType or 'none'
if rtype == 'srclib':
rtype = common.getsrclibvcs(app.Repo)
repotypes[rtype] += 1
with open(os.path.join(statsdir, 'repotypes.txt'), 'w') as f:
for rtype, count in most_common_stable(repotypes):
f.write(rtype + ' ' + str(count) + '\n')
# Calculate and write stats for update check modes...
logging.info("Processing update check modes...")
ucms = Counter()
for app in metaapps:
checkmode = app.UpdateCheckMode
if checkmode.startswith('RepoManifest/'):
checkmode = checkmode[:12]
if checkmode.startswith('Tags '):
checkmode = checkmode[:4]
ucms[checkmode] += 1
with open(os.path.join(statsdir, 'update_check_modes.txt'), 'w') as f:
for checkmode, count in most_common_stable(ucms):
f.write(checkmode + ' ' + str(count) + '\n')
logging.info("Processing categories...")
ctgs = Counter()
for app in metaapps:
for category in app.Categories:
ctgs[category] += 1
with open(os.path.join(statsdir, 'categories.txt'), 'w') as f:
for category, count in most_common_stable(ctgs):
f.write(category + ' ' + str(count) + '\n')
logging.info("Processing antifeatures...")
afs = Counter()
for app in metaapps:
if app.AntiFeatures is None:
continue
for antifeature in app.AntiFeatures:
afs[antifeature] += 1
with open(os.path.join(statsdir, 'antifeatures.txt'), 'w') as f:
for antifeature, count in most_common_stable(afs):
f.write(antifeature + ' ' + str(count) + '\n')
# Calculate and write stats for licenses...
logging.info("Processing licenses...")
licenses = Counter()
for app in metaapps:
license = app.License
licenses[license] += 1
with open(os.path.join(statsdir, 'licenses.txt'), 'w') as f:
for license, count in most_common_stable(licenses):
f.write(license + ' ' + str(count) + '\n')
# Write list of disabled apps...
logging.info("Processing disabled apps...")
disabled = [app.id for app in allmetaapps if app.Disabled]
with open(os.path.join(statsdir, 'disabled_apps.txt'), 'w') as f:
for appid in sorted(disabled):
f.write(appid + '\n')
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
with open(os.path.join(statsdir, 'latestapps.txt'), 'w') as f:
for appid in latest:
f.write(appid + '\n')
if unknownapks:
logging.info('\nUnknown apks:')
for apk in unknownapks:
logging.info(apk)
logging.info("Finished.")
def check_repomanifest(app, sdk_path, branch=None):
try:
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib')
repotype = common.getsrclibvcs(app['Repo'])
else:
build_dir = os.path.join('build/', app['id'])
repotype = app['Repo Type']
if repotype == 'bzr':
return (None, 'RepoManifest update mode has not been ported to bzr repositories yet')
# Set up vcs interface and make sure we have the latest code...
vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
if app['Repo Type'] == 'srclib':
build_dir = os.path.join(build_dir, app['Repo'])
if vcs.repotype() == 'git':
if branch:
vcs.gotorevision('origin/'+branch)
else:
vcs.gotorevision('origin/master')
pass
elif vcs.repotype() == 'git-svn':
if branch:
vcs.gotorevision(branch)
else:
vcs.gotorevision(None)
elif vcs.repotype() == 'svn':
vcs.gotorevision(None)
elif vcs.repotype() == 'hg':
if branch:
vcs.gotorevision(branch)
else:
vcs.gotorevision('default')
if len(app['builds']) > 0:
if 'subdir' in app['builds'][-1]:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
version, vercode, package = common.parse_androidmanifest(build_dir)
if not package:
return (None, "Couldn't find package ID")
if package != app['id']:
return (None, "Package ID mismatch")
if not version:
return (None,"Couldn't find latest version name")
if not vercode:
return (None,"Couldn't find latest version code")
return (version, str(int(vercode)))
except BuildException as be:
msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
return (None, msg)
except VCSException as vcse:
msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
return (None, msg)
except Exception:
msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
return (None, msg)
