def updateDbs(apps):
"""Used to create .pkl files for any apps or plugins the are declared
supported but don't have an up-to-date pkl file.
Takes a list of appnames.
"""
for app in apps:
print(config.getDbPath(app))
if not os.access(config.getDbPath(app), os.F_OK):
print("No db file available for app %s. Creating it from %s..." % (app, config.getAppPath(app)))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPath(app), config.APP_CONFIG[app]["versionDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app), pathNodes, versionNodes, allversions)
else:
print("Found db file for app %s" % app, end=' ')
#print "(path is ", config.getDbPath(app), ")"
pathNodes, versionNodes, allversions = dt.loadTables(config.getDbPath(app), False)
versInDb = len(allversions)
versOnDisk = len([entry for entry in os.listdir(config.getAppPath(app)) if re.match(config.APP_CONFIG[app]["versionDirectoryRegex"], entry)])
#versInDb = set(ver.vstring for ver in allversions)
#versOnDisk = set([entry for entry in os.listdir(config.getAppPath(app)) if re.match(config.APP_CONFIG[app]["versionDirectoryRegex"], entry)])
#verDifferences = versInDb ^ versOnDisk
#print "DB: %s\nDisk: %s\nDiffs: %s" % (versInDb, versOnDisk, verDifferences)
if versInDb != versOnDisk:
print("but it is out of date (%s versions in db, %s versions on disk). Recreating it from %s... " % (versInDb, versOnDisk, config.getAppPath(app)))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPath(app), config.APP_CONFIG[app]["versionDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app), pathNodes, versionNodes, allversions)
else:
print(".")
if os.access(config.getAppPluginPath(app), os.F_OK):
for plugin in [p for p in sorted(os.listdir(config.getAppPluginPath(app))) if os.path.isdir(config.getAppPluginPath(app, p))]:
if not os.access(config.getDbPath(app, plugin), os.F_OK):
print("No db file available for %s plugin %s. Creating it from %s..." % (app, plugin, config.getAppPluginPath(app, plugin)))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPluginPath(app, plugin), plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], "none", config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app, plugin), pathNodes, versionNodes, allversions)
else:
print("Found db file for %s plugin %s" % (app, plugin), end=' ')
pathNodes, versionNodes, allversions = dt.loadTables(config.getDbPath(app, plugin), False)
versInDb = len(allversions)
versOnDisk = len([entry for entry in os.listdir(config.getAppPluginPath(app, plugin)) if re.match(plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], entry)])
if versInDb != versOnDisk:
print("but it is out of date (%s versions in db, %s versions on disk). Recreating it from %s... " % (versInDb, versOnDisk, config.getAppPluginPath(app, plugin)))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPluginPath(app, plugin), plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app, plugin), pathNodes, versionNodes, allversions)
else:
print(".")
parser.add_option(
"-u",
"--updateDB",
action="store_true",
help=
"Pull latest DB files from blindelephant.sourceforge.net repo (Equivalent to svn update on blindelephant/dbs/). May require root if blindelephant was installed with root."
)
(options, args) = parser.parse_args()
if options.list:
print("Currently configured web apps:",
len(list(wac.APP_CONFIG.keys())),
file=wac.DEFAULT_LOGFILE)
for app in sorted(wac.APP_CONFIG.keys()):
pluginsDir = wac.getDbDir(app)
plugins = os.listdir(pluginsDir) if os.access(pluginsDir,
os.F_OK) else []
plugins = [p for p in plugins if p.endswith(wac.DB_EXTENSION)]
print("%s with %d plugins" % (app, len(plugins)),
file=wac.DEFAULT_LOGFILE)
for p in sorted(plugins):
print(" -",
p[:-(len(wac.DB_EXTENSION))],
file=wac.DEFAULT_LOGFILE)
quit()
if options.updateDB:
"""Added at the request of backbox.org; pretty hacky. May formalize this in the future if there's demand"""
import tempfile
import urllib.request, urllib.parse, urllib.error
def updateDbs(apps):
"""Used to create .pkl files for any apps or plugins the are declared
supported but don't have an up-to-date pkl file.
Takes a list of appnames.
"""
for app in apps:
if not os.access(config.getDbPath(app), os.F_OK):
print "No db file available for app %s. Creating it from %s..." % (app, config.getAppPath(app))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPath(app), config.APP_CONFIG[app]["versionDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app), pathNodes, versionNodes, allversions)
else:
print "Found db file for app %s" % app,
#print "(path is ", config.getDbPath(app), ")"
pathNodes, versionNodes, allversions = dt.loadTables(config.getDbPath(app), False)
versInDb = len(allversions)
versOnDisk = len(filter(lambda entry: re.match(config.APP_CONFIG[app]["versionDirectoryRegex"], entry),os.listdir(config.getAppPath(app))))
#versInDb = set(ver.vstring for ver in allversions)
#versOnDisk = set([entry for entry in os.listdir(config.getAppPath(app)) if re.match(config.APP_CONFIG[app]["versionDirectoryRegex"], entry)])
#verDifferences = versInDb ^ versOnDisk
#print "DB: %s\nDisk: %s\nDiffs: %s" % (versInDb, versOnDisk, verDifferences)
if versInDb != versOnDisk:
print "but it is out of date (%s versions in db, %s versions on disk). Recreating it from %s... " % (versInDb, versOnDisk, config.getAppPath(app))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPath(app), config.APP_CONFIG[app]["versionDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app), pathNodes, versionNodes, allversions)
else:
print "."
if os.access(config.getAppPluginPath(app), os.F_OK):
for plugin in filter(lambda p: os.path.isdir(config.getAppPluginPath(app, p)), sorted(os.listdir(config.getAppPluginPath(app)))):
if not os.access(config.getDbPath(app, plugin), os.F_OK):
print "No db file available for %s plugin %s. Creating it from %s..." % (app, plugin, config.getAppPluginPath(app, plugin))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPluginPath(app, plugin), plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], "none", config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app, plugin), pathNodes, versionNodes, allversions)
else:
print "Found db file for %s plugin %s" % (app, plugin),
pathNodes, versionNodes, allversions = dt.loadTables(config.getDbPath(app, plugin), False)
versInDb = len(allversions)
versOnDisk = len(filter(lambda entry: re.match(plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], entry),os.listdir(config.getAppPluginPath(app, plugin))))
if versInDb != versOnDisk:
print "but it is out of date (%s versions in db, %s versions on disk). Recreating it from %s... " % (versInDb, versOnDisk, config.getAppPluginPath(app, plugin))
pathNodes, versionNodes, allversions = dt.computeTables(config.getAppPluginPath(app, plugin), plugin+config.APP_CONFIG[app]["pluginsDirectoryRegex"], config.APP_CONFIG[app]["directoryExcludeRegex"], config.APP_CONFIG[app]["fileExcludeRegex"])
dt.saveTables(config.getDbPath(app, plugin), pathNodes, versionNodes, allversions)
else:
print "."
parser = OptionParser(usage=USAGE, epilog=EPILOGUE)
parser.add_option("-p", "--pluginName", help="Fingerprint version of plugin (should apply to web app given in appname)")
parser.add_option("-s", "--skip", action="store_true", help="Skip fingerprinting webpp, just fingerprint plugin")
parser.add_option("-n", "--numProbes", type='int', help="Number of files to fetch (more may increase accuracy). Default: %default", default=15)
parser.add_option("-w", "--winnow", action="store_true", help="If more than one version are returned, use winnowing to attempt to narrow it down (up to numProbes additional requests).")
parser.add_option("-l", "--list", action="store_true", help="List supported webapps and plugins")
parser.add_option("-u", "--updateDB", action="store_true", help="Pull latest DB files from blindelephant.sourceforge.net repo (Equivalent to svn update on blindelephant/dbs/). May require root if blindelephant was installed with root.")
(options, args) = parser.parse_args()
if options.list:
print >> wac.DEFAULT_LOGFILE, "Currently configured web apps:", len(wac.APP_CONFIG.keys())
for app in sorted(wac.APP_CONFIG.keys()):
pluginsDir = wac.getDbDir(app)
plugins = os.listdir(pluginsDir) if os.access(pluginsDir, os.F_OK) else []
plugins = [p for p in plugins if p.endswith(wac.DB_EXTENSION)]
print >> wac.DEFAULT_LOGFILE, "%s with %d plugins" % (app, len(plugins))
for p in sorted(plugins):
print >> wac.DEFAULT_LOGFILE, " -", p[:-(len(wac.DB_EXTENSION))]
quit()
if options.updateDB:
"""Added at the request of backbox.org; pretty hacky. May formalize this in the future if there's demand"""
import tempfile
import urllib
import os
import tarfile
dbtar_url = "http://blindelephant.svn.sourceforge.net/viewvc/blindelephant/trunk/src/blindelephant/dbs/?view=tar"
untar_dir = os.path.join(wac.getDbDir(), os.path.pardir); #so that dbs/ in tar overlays existing dbs/
