def main():
# Read configuration...
global update_stats, archive_older
update_stats = False
archive_older = 0
execfile('config.py', globals())
# Parse command line...
global options
parser = OptionParser()
parser.add_option("-c", "--createmeta", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="No output, except for warnings and errors")
parser.add_option("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status")
parser.add_option("-i", "--interactive", default=False, action="store_true",
help="Interactively ask about things that need updating.")
parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default "+
"is /etc/alternatives/editor")
parser.add_option("-w", "--wiki", default=False, action="store_true",
help="Update the wiki")
parser.add_option("", "--pretty", action="store_true", default=False,
help="Produce human-readable index.xml")
parser.add_option("--clean", action="store_true", default=False,
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
# Get all apps...
apps = common.read_metadata(verbose=options.verbose)
# Generate a list of categories...
categories = []
for app in apps:
cats = app['Category'].split(';')
for cat in cats:
if cat not in categories:
categories.append(cat)
# Read known apks data (will be updated and written back when we've finished)
knownapks = common.KnownApks()
# Gather information about all the apk files in the repo directory, using
# cached data if possible.
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf)
else:
apkcache = {}
cachechanged = False
repodirs = ['repo']
if archive_older != 0:
repodirs.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
delete_disabled_builds(apps, apkcache, repodirs)
apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
if cc:
cachechanged = True
# Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
for app in apps:
bestver = 0
added = None
lastupdated = None
for apk in apks:
if apk['id'] == app['id']:
if apk['versioncode'] > bestver:
bestver = apk['versioncode']
bestapk = apk
if 'added' in apk:
if not added or apk['added'] < added:
added = apk['added']
if not lastupdated or apk['added'] > lastupdated:
lastupdated = apk['added']
if added:
app['added'] = added
else:
print "WARNING: Don't know when " + app['id'] + " was added"
if lastupdated:
app['lastupdated'] = lastupdated
else:
print "WARNING: Don't know when " + app['id'] + " was last updated"
if bestver == 0:
if app['Name'] is None:
app['Name'] = app['id']
app['icon'] = ''
if app['Disabled'] is None:
print "WARNING: Application " + app['id'] + " has no packages"
else:
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon']
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
apps = sorted(apps, key=lambda app: app['Name'].upper())
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
for apk in apks:
found = False
for app in apps:
if app['id'] == apk['id']:
found = True
break
if not found:
if options.createmeta:
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
f.write(".\n")
f.close()
print "Generated skeleton metadata for " + apk['id']
else:
print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
print " " + apk['name'] + " - " + apk['version']
if len(repodirs) > 1:
archive_old_apks(apps, apks, repodirs[0], repodirs[1], archive_older)
# Make the index for the main repo...
make_index(apps, apks, repodirs[0], False, categories)
# If there's an archive repo, scan the apks for that and make the index...
archapks = None
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
if cc:
cachechanged = True
make_index(apps, archapks, repodirs[1], True, categories)
if update_stats:
# Update known apks info...
knownapks.writeifchanged()
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip()
data += appid + "\t"
for app in apps:
if app['id'] == appid:
data += app['Name'] + "\t"
data += app['icon'] + "\t"
data += app['License'] + "\n"
break
f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
f.write(data)
f.close()
if cachechanged:
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
# Update the wiki...
if options.wiki:
if archapks:
apks.extend(archapks)
update_wiki(apps, apks, options.verbose)
print "Finished."
def main():
global options, config
# Parse command line...
parser = OptionParser()
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
parser.add_option("-q", "--quiet", action="store_true", default=False,
help="Restrict output to warnings and errors")
parser.add_option("-d", "--download", action="store_true", default=False,
help="Download logs we don't have")
parser.add_option("--recalc", action="store_true", default=False,
help="Recalculate aggregate stats - use when changes "
"have been made that would invalidate old cached data.")
parser.add_option("--nologs", action="store_true", default=False,
help="Don't do anything logs-related")
(options, args) = parser.parse_args()
config = common.read_config(options)
if not config['update_stats']:
logging.info("Stats are disabled - check your configuration")
sys.exit(1)
# Get all metadata-defined apps...
metaapps = [a for a in metadata.read_metadata().itervalues() if not a['Disabled']]
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
logging.info('Retrieving logs')
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org', username='******', timeout=10,
key_filename=config['webserver_keyfile'])
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
logging.info("...connected")
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath) or
os.path.getsize(destpath) != destsize):
logging.debug("...retrieving " + f)
ftp.get(f, destpath)
except Exception:
traceback.print_exc()
sys.exit(1)
finally:
# Disconnect
if ftp is not None:
ftp.close()
if ssh is not None:
ssh.close()
knownapks = common.KnownApks()
unknownapks = []
if not options.nologs:
# Process logs
logging.info('Processing logs...')
appscount = Counter()
appsvercount = Counter()
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] ' + \
'"GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) ' + \
'\d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
logging.debug('...' + logfile)
# Get the date for this log - e.g. 2012-02-28
thisdate = os.path.basename(logfile)[7:-7]
agg_path = os.path.join(datadir, thisdate + '.json')
if not options.recalc and os.path.exists(agg_path):
# Use previously calculated aggregate data
with open(agg_path, 'r') as f:
today = json.load(f)
else:
# Calculate from logs...
today = {
'apps': Counter(),
'appsver': Counter(),
'unknown': []
}
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if not match:
continue
if match.group('statuscode') != '200':
continue
if match.group('ip') in config['stats_ignore']:
continue
uri = match.group('uri')
if not uri.endswith('.apk'):
continue
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
today['apps'][appid] += 1
# Strip the '.apk' from apkname
appver = apkname[:-4]
today['appsver'][appver] += 1
else:
if apkname not in today['unknown']:
today['unknown'].append(apkname)
# Save calculated aggregate data for today to cache
with open(agg_path, 'w') as f:
json.dump(today, f)
# Add today's stats (whether cached or recalculated) to the total
for appid in today['apps']:
appscount[appid] += today['apps'][appid]
for appid in today['appsver']:
appsvercount[appid] += today['appsver'][appid]
for uk in today['unknown']:
if uk not in unknownapks:
unknownapks.append(uk)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for appid in appscount:
count = appscount[appid]
lst.append(appid + " " + str(count))
if config['stats_to_carbon']:
carbon_send('fdroid.download.' + appid.replace('.', '_'),
count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
f = open('stats/total_downloads_app_version.txt', 'w')
f.write('# Total downloads by application and version, '
'since October 2011\n')
lst = []
for appver in appsvercount:
count = appsvercount[appver]
lst.append(appver + " " + str(count))
for line in sorted(lst):
f.write(line + "\n")
f.close()
# Calculate and write stats for repo types...
logging.info("Processing repo types...")
repotypes = Counter()
for app in metaapps:
rtype = app['Repo Type'] or 'none'
if rtype == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
repotypes[rtype] += 1
f = open('stats/repotypes.txt', 'w')
for rtype in repotypes:
count = repotypes[rtype]
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
logging.info("Processing update check modes...")
ucms = Counter()
for app in metaapps:
checkmode = app['Update Check Mode']
if checkmode.startswith('RepoManifest/'):
checkmode = checkmode[:12]
if checkmode.startswith('Tags '):
checkmode = checkmode[:4]
ucms[checkmode] += 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode in ucms:
count = ucms[checkmode]
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
logging.info("Processing categories...")
ctgs = Counter()
for app in metaapps:
for category in app['Categories']:
ctgs[category] += 1
f = open('stats/categories.txt', 'w')
for category in ctgs:
count = ctgs[category]
f.write(category + ' ' + str(count) + '\n')
f.close()
logging.info("Processing antifeatures...")
afs = Counter()
for app in metaapps:
if app['AntiFeatures'] is None:
continue
antifeatures = [a.strip() for a in app['AntiFeatures'].split(',')]
for antifeature in antifeatures:
afs[antifeature] += 1
f = open('stats/antifeatures.txt', 'w')
for antifeature in afs:
count = afs[antifeature]
f.write(antifeature + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
logging.info("Processing licenses...")
licenses = Counter()
for app in metaapps:
license = app['License']
licenses[license] += 1
f = open('stats/licenses.txt', 'w')
for license in licenses:
count = licenses[license]
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
logging.info("Processing latest apps...")
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if unknownapks:
logging.info('\nUnknown apks:')
for apk in unknownapks:
logging.info(apk)
logging.info("Finished.")
def main():
# Read configuration...
global update_stats, stats_to_carbon
update_stats = False
stats_to_carbon = False
execfile('config.py', globals())
if not update_stats:
print "Stats are disabled - check your configuration"
sys.exit(1)
# Parse command line...
parser = OptionParser()
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-d",
"--download",
action="store_true",
default=False,
help="Download logs we don't have")
(options, args) = parser.parse_args()
# Get all metadata-defined apps...
metaapps = common.read_metadata(options.verbose)
statsdir = 'stats'
logsdir = os.path.join(statsdir, 'logs')
logsarchivedir = os.path.join(logsdir, 'archive')
datadir = os.path.join(statsdir, 'data')
if not os.path.exists(statsdir):
os.mkdir(statsdir)
if not os.path.exists(logsdir):
os.mkdir(logsdir)
if not os.path.exists(datadir):
os.mkdir(datadir)
if options.download:
# Get any access logs we don't have...
ssh = None
ftp = None
try:
print 'Retrieving logs'
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect('f-droid.org',
username='******',
timeout=10,
key_filename=webserver_keyfile)
ftp = ssh.open_sftp()
ftp.get_channel().settimeout(60)
print "...connected"
ftp.chdir('logs')
files = ftp.listdir()
for f in files:
if f.startswith('access-') and f.endswith('.log.gz'):
destpath = os.path.join(logsdir, f)
destsize = ftp.stat(f).st_size
if (not os.path.exists(destpath)
or os.path.getsize(destpath) != destsize):
print "...retrieving " + f
ftp.get(f, destpath)
except Exception as e:
traceback.print_exc()
sys.exit(1)
finally:
#Disconnect
if ftp != None:
ftp.close()
if ssh != None:
ssh.close()
# Process logs
if options.verbose:
print 'Processing logs...'
logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
logsearch = re.compile(logexpr).search
apps = {}
unknownapks = []
knownapks = common.KnownApks()
for logfile in glob.glob(os.path.join(logsdir, 'access-*.log.gz')):
if options.verbose:
print '...' + logfile
logdate = logfile[len(logsdir) + 1 + len('access-'):-7]
p = subprocess.Popen(["zcat", logfile], stdout=subprocess.PIPE)
matches = (logsearch(line) for line in p.stdout)
for match in matches:
if match and match.group('statuscode') == '200':
uri = match.group('uri')
if uri.endswith('.apk'):
_, apkname = os.path.split(uri)
app = knownapks.getapp(apkname)
if app:
appid, _ = app
if appid in apps:
apps[appid] += 1
else:
apps[appid] = 1
else:
if not apkname in unknownapks:
unknownapks.append(apkname)
# Calculate and write stats for total downloads...
lst = []
alldownloads = 0
for app, count in apps.iteritems():
lst.append(app + " " + str(count))
if stats_to_carbon:
carbon_send('fdroid.download.' + app.replace('.', '_'), count)
alldownloads += count
lst.append("ALL " + str(alldownloads))
f = open('stats/total_downloads_app.txt', 'w')
f.write('# Total downloads by application, since October 2011\n')
for line in sorted(lst):
f.write(line + '\n')
f.close()
# Calculate and write stats for repo types...
repotypes = {}
for app in metaapps:
if len(app['Repo Type']) == 0:
rtype = 'none'
else:
if app['Repo Type'] == 'srclib':
rtype = common.getsrclibvcs(app['Repo'])
else:
rtype = app['Repo Type']
if rtype in repotypes:
repotypes[rtype] += 1
else:
repotypes[rtype] = 1
f = open('stats/repotypes.txt', 'w')
for rtype, count in repotypes.iteritems():
f.write(rtype + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for update check modes...
ucms = {}
for app in metaapps:
checkmode = app['Update Check Mode'].split('/')[0]
if checkmode in ucms:
ucms[checkmode] += 1
else:
ucms[checkmode] = 1
f = open('stats/update_check_modes.txt', 'w')
for checkmode, count in ucms.iteritems():
f.write(checkmode + ' ' + str(count) + '\n')
f.close()
# Calculate and write stats for licenses...
licenses = {}
for app in metaapps:
license = app['License']
if license in licenses:
licenses[license] += 1
else:
licenses[license] = 1
f = open('stats/licenses.txt', 'w')
for license, count in licenses.iteritems():
f.write(license + ' ' + str(count) + '\n')
f.close()
# Write list of latest apps added to the repo...
latest = knownapks.getlatest(10)
f = open('stats/latestapps.txt', 'w')
for app in latest:
f.write(app + '\n')
f.close()
if len(unknownapks) > 0:
print '\nUnknown apks:'
for apk in unknownapks:
print apk
print "Finished."
def main():
global config, options
# Parse command line...
parser = OptionParser()
parser.add_option("-c",
"--create-metadata",
action="store_true",
default=False,
help="Create skeleton metadata files that are missing")
parser.add_option("--delete-unknown",
action="store_true",
default=False,
help="Delete APKs without metadata from the repo")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="Spew out even more information than normal")
parser.add_option("-q",
"--quiet",
action="store_true",
default=False,
help="Restrict output to warnings and errors")
parser.add_option("-b",
"--buildreport",
action="store_true",
default=False,
help="Report on build data status")
parser.add_option(
"-i",
"--interactive",
default=False,
action="store_true",
help="Interactively ask about things that need updating.")
parser.add_option(
"-I",
"--icons",
action="store_true",
default=False,
help="Resize all the icons exceeding the max pixel size and exit")
parser.add_option(
"-e",
"--editor",
default="/etc/alternatives/editor",
help="Specify editor to use in interactive mode. Default " +
"is /etc/alternatives/editor")
parser.add_option("-w",
"--wiki",
default=False,
action="store_true",
help="Update the wiki")
parser.add_option("",
"--pretty",
action="store_true",
default=False,
help="Produce human-readable index.xml")
parser.add_option(
"--clean",
action="store_true",
default=False,
help="Clean update - don't uses caches, reprocess all apks")
(options, args) = parser.parse_args()
config = common.read_config(options)
repodirs = ['repo']
if config['archive_older'] != 0:
repodirs.append('archive')
if not os.path.exists('archive'):
os.mkdir('archive')
if options.icons:
resize_all_icons(repodirs)
sys.exit(0)
# check that icons exist now, rather than fail at the end of `fdroid update`
for k in ['repo_icon', 'archive_icon']:
if k in config:
if not os.path.exists(config[k]):
logging.critical(k + ' "' + config[k] +
'" does not exist! Correct it in config.py.')
sys.exit(1)
# Get all apps...
apps = metadata.read_metadata()
# Generate a list of categories...
categories = set()
for app in apps.itervalues():
categories.update(app['Categories'])
# Read known apks data (will be updated and written back when we've finished)
knownapks = common.KnownApks()
# Gather information about all the apk files in the repo directory, using
# cached data if possible.
apkcachefile = os.path.join('tmp', 'apkcache')
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf)
else:
apkcache = {}
cachechanged = False
delete_disabled_builds(apps, apkcache, repodirs)
# Scan all apks in the main repo
apks, cc = scan_apks(apps, apkcache, repodirs[0], knownapks)
if cc:
cachechanged = True
# Generate warnings for apk's with no metadata (or create skeleton
# metadata files, if requested on the command line)
newmetadata = False
for apk in apks:
if apk['id'] not in apps:
if options.create_metadata:
if 'name' not in apk:
logging.error(apk['id'] +
' does not have a name! Skipping...')
continue
f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
f.write("License:Unknown\n")
f.write("Web Site:\n")
f.write("Source Code:\n")
f.write("Issue Tracker:\n")
f.write("Summary:" + apk['name'] + "\n")
f.write("Description:\n")
f.write(apk['name'] + "\n")
f.write(".\n")
f.close()
logging.info("Generated skeleton metadata for " + apk['id'])
newmetadata = True
else:
msg = apk['apkname'] + " (" + apk['id'] + ") has no metadata!"
if options.delete_unknown:
logging.warn(msg + "\n\tdeleting: repo/" + apk['apkname'])
rmf = os.path.join(repodirs[0], apk['apkname'])
if not os.path.exists(rmf):
logging.error(
"Could not find {0} to remove it".format(rmf))
else:
os.remove(rmf)
else:
logging.warn(msg +
"\n\tUse `fdroid update -c` to create it.")
# update the metadata with the newly created ones included
if newmetadata:
apps = metadata.read_metadata()
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks)
if cc:
cachechanged = True
else:
archapks = []
# Some information from the apks needs to be applied up to the application
# level. When doing this, we use the info from the most recent version's apk.
# We deal with figuring out when the app was added and last updated at the
# same time.
for appid, app in apps.iteritems():
bestver = 0
added = None
lastupdated = None
for apk in apks + archapks:
if apk['id'] == appid:
if apk['versioncode'] > bestver:
bestver = apk['versioncode']
bestapk = apk
if 'added' in apk:
if not added or apk['added'] < added:
added = apk['added']
if not lastupdated or apk['added'] > lastupdated:
lastupdated = apk['added']
if added:
app['added'] = added
else:
logging.warn("Don't know when " + appid + " was added")
if lastupdated:
app['lastupdated'] = lastupdated
else:
logging.warn("Don't know when " + appid + " was last updated")
if bestver == 0:
if app['Name'] is None:
app['Name'] = appid
app['icon'] = None
logging.warn("Application " + appid + " has no packages")
else:
if app['Name'] is None:
app['Name'] = bestapk['name']
app['icon'] = bestapk['icon'] if 'icon' in bestapk else None
# Sort the app list by name, then the web site doesn't have to by default.
# (we had to wait until we'd scanned the apks to do this, because mostly the
# name comes from there!)
sortedids = sorted(apps.iterkeys(),
key=lambda appid: apps[appid]['Name'].upper())
if len(repodirs) > 1:
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1],
config['archive_older'])
# Make the index for the main repo...
make_index(apps, sortedids, apks, repodirs[0], False, categories)
# If there's an archive repo, make the index for it. We already scanned it
# earlier on.
if len(repodirs) > 1:
make_index(apps, sortedids, archapks, repodirs[1], True, categories)
if config['update_stats']:
# Update known apks info...
knownapks.writeifchanged()
# Generate latest apps data for widget
if os.path.exists(os.path.join('stats', 'latestapps.txt')):
data = ''
for line in file(os.path.join('stats', 'latestapps.txt')):
appid = line.rstrip()
data += appid + "\t"
app = apps[appid]
data += app['Name'] + "\t"
if app['icon'] is not None:
data += app['icon'] + "\t"
data += app['License'] + "\n"
f = open(os.path.join(repodirs[0], 'latestapps.dat'), 'w')
f.write(data)
f.close()
if cachechanged:
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
# Update the wiki...
if options.wiki:
update_wiki(apps, sortedids, apks + archapks)
logging.info("Finished.")