class Index(Minimal, Advanced_API):
#############
# Variables #
#############
def __init__(self):
# TODO: make auth handler and plugin manager singletons
Advanced_API.__init__(self)
Minimal.__init__(self)
self.minimal = False
self.auth_handler = AuthenticationHandler()
self.plugManager = PluginManager()
self.login_manager = LoginManager()
self.plugManager.loadPlugins()
self.login_manager.init_app(self.app)
self.login_manager.user_loader(self.load_user)
self.redisdb = Configuration.getRedisVendorConnection()
self.defaultFilters.update({
'blacklistSelect': 'off',
'whitelistSelect': 'on',
'unlistedSelect': 'off',
})
self.args.update({'minimal': False})
self.pluginArgs = {
"current_user": current_user,
"plugin_manager": self.plugManager
}
routes = [{
'r': '/cve/<cveid>',
'm': ['GET'],
'f': self.cve
}, {
'r': '/_get_plugins',
'm': ['GET'],
'f': self._get_plugins
}, {
'r': '/plugin/_get_cve_actions',
'm': ['GET'],
'f': self._get_cve_actions
}, {
'r': '/plugin/<plugin>',
'm': ['GET'],
'f': self.openPlugin
}, {
'r': '/plugin/<plugin>/subpage/<page>',
'm': ['GET'],
'f': self.openPluginSubpage
}, {
'r': '/plugin/<plugin>/_cve_action/<action>',
'm': ['GET'],
'f': self._jsonCVEAction
}, {
'r': '/login',
'm': ['POST'],
'f': self.login_check
}, {
'r': '/logout',
'm': ['GET'],
'f': self.logout
}, {
'r': '/admin',
'm': ['GET'],
'f': self.admin
}, {
'r': '/admin/',
'm': ['GET'],
'f': self.admin
}, {
'r': '/admin/change_pass',
'm': ['GET'],
'f': self.change_pass
}, {
'r': '/admin/request_token',
'm': ['GET'],
'f': self.request_token
}, {
'r': '/admin/updatedb',
'm': ['GET'],
'f': self.updatedb
}, {
'r': '/admin/enableupdatedb',
'm': ['GET'],
'f': self.enableupdatedb
}, {
'r': '/admin/enableupdatedb/status',
'm': ['GET'],
'f': self.enableupdatedbstatus
}, {
'r': '/admin/whitelist/import',
'm': ['POST'],
'f': self.listImport
}, {
'r': '/admin/blacklist/import',
'm': ['POST'],
'f': self.listImport
}, {
'r': '/admin/whitelist/export',
'm': ['GET'],
'f': self.listExport
}, {
'r': '/admin/blacklist/export',
'm': ['GET'],
'f': self.listExport
}, {
'r': '/admin/whitelist/drop',
'm': ['GET'],
'f': self.listDrop
}, {
'r': '/admin/blacklist/drop',
'm': ['GET'],
'f': self.listDrop
}, {
'r': '/admin/whitelist',
'm': ['GET'],
'f': self.listView
}, {
'r': '/admin/blacklist',
'm': ['GET'],
'f': self.listView
}, {
'r': '/admin/addToList',
'm': ['GET'],
'f': self.listAdd
}, {
'r': '/admin/removeFromList',
'm': ['GET'],
'f': self.listRemove
}, {
'r': '/admin/editInList',
'm': ['GET'],
'f': self.listEdit
}, {
'r': '/admin/listmanagement',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/<vendor>',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/<vendor>/<product>',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/add',
'm': ['GET'],
'f': self.listManagementAdd
}, {
'r': '/login',
'm': ['POST'],
'f': self.login_check
}]
for route in routes:
self.addRoute(route)
#############
# Functions #
#############
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
if current_user.is_authenticated():
if f['blacklistSelect'] == "on":
regexes = db.getRules('blacklist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['whitelistSelect'] == "hide":
regexes = db.getRules('whitelist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['unlistedSelect'] == "hide":
wlregexes = tk.compile(db.getRules('whitelist'))
blregexes = tk.compile(db.getRules('blacklist'))
query.append({
'$or': [{
'vulnerable_configuration': {
'$in': wlregexes
}
}, {
'vulnerable_configuration': {
'$in': blregexes
}
}]
})
return query
def markCPEs(self, cve):
blacklist = tk.compile(db.getRules('blacklist'))
whitelist = tk.compile(db.getRules('whitelist'))
for conf in cve['vulnerable_configuration']:
conf['list'] = 'none'
conf['match'] = 'none'
for w in whitelist:
if w.match(conf['id']):
conf['list'] = 'white'
conf['match'] = w
for b in blacklist:
if b.match(conf['id']):
conf['list'] = 'black'
conf['match'] = b
return cve
def filter_logic(self, filters, skip, limit=None):
query = self.generate_full_query(filters)
limit = limit if limit else self.args['pageLength']
cve = db.getCVEs(limit=limit, skip=skip, query=query)
# marking relevant records
if current_user.is_authenticated():
user = current_user.get_id()
if filters['whitelistSelect'] == "on":
cve = self.list_mark('white', cve, user=user)
if filters['blacklistSelect'] == "mark":
cve = self.list_mark('black', cve, user=user)
self.plugManager.mark(cve, **self.pluginArgs)
cve = list(cve)
return cve
def addCPEToList(self,
cpe,
listType,
cpeType=None,
isglobal=True,
user=None):
def addCPE(cpe, cpeType, funct, isglobal, user):
return True if funct(cpe, cpeType, isglobal, user) else False
if not cpeType: cpeType = 'cpe'
if listType.lower() in ("blacklist", "black", "b", "bl"):
return addCPE(cpe, cpeType, bl.insertBlacklist, isglobal, user)
if listType.lower() in ("whitelist", "white", "w", "wl"):
return addCPE(cpe, cpeType, wl.insertWhitelist, isglobal, user)
#TODO: we need only to filer list that we own or it is marker as a global
def list_mark(self, listed, cveList, user=None):
if listed not in ['white', 'black']: return list(cves)
items = tk.compile(db.getRules(listed + 'list'))
# check the cpes (full or partially) in the black/whitelist
for i, cve in enumerate(
list(cveList)
): # the list() is to ensure we don't have a pymongo cursor object
for c in cve['vulnerable_configuration']:
if any(regex.match(c) for regex in items):
cveList[i][listed + 'listed'] = 'yes'
return cveList
def filterUpdateField(self, data):
if not data: return data
returnvalue = []
for line in data.split("\n"):
if (not line.startswith("[+]Success to create index")
and not line == "Not modified"
and not line.startswith("Starting")):
returnvalue.append(line)
return "\n".join(returnvalue)
def adminInfo(self, output=None):
return {
'stats': db.getDBStats(True),
'plugins': self.plugManager.getPlugins(),
'updateOutput': self.filterUpdateField(output),
'token': str(db.getToken(current_user.id))
}
# user management
def load_user(self, id):
return User.get(id, self.auth_handler)
##########
# ROUTES #
##########
# /cve/<cveid>
def cve(self, cveid):
cveid = cveid.upper()
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True,
subscorelookup=True)
cve = cvesp.getcve(cveid=cveid)
if cve is None:
return render_template('error.html',
status={
'except': 'cve-not-found',
'info': {
'cve': cveid
}
})
cve = self.markCPEs(cve)
self.plugManager.onCVEOpen(cveid, **self.pluginArgs)
pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs)
return render_template('cve.html', cve=cve, plugins=pluginData)
# /_get_plugins
def _get_plugins(self):
if not current_user.is_authenticated(
): # Don't show plugins requiring auth if not authenticated
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs) if not x.requiresAuth]
else:
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs)]
return jsonify({"plugins": plugins})
# /plugin/_get_cve_actions
def _get_cve_actions(self):
cve = request.args.get('cve', type=str)
if not current_user.is_authenticated(
): # Don't show actions requiring auth if not authenticated
actions = [
x for x in self.plugManager.getCVEActions(
cve, **self.pluginArgs) if not x['auth']
]
else:
actions = self.plugManager.getCVEActions(cve, **self.pluginArgs)
return jsonify({"actions": actions})
# /plugin/<plugin>
def openPlugin(self, plugin):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openPage(plugin, **self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
else:
abort(404)
# /plugin/<plugin>/subpage/<page>
def openPluginSubpage(self, plugin, page):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openSubpage(plugin, page,
**self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
else:
abort(404)
# /plugin/<plugin>/_cve_action/<action>
def _jsonCVEAction(self, plugin, action):
cve = request.args.get('cve', type=str)
response = self.plugManager.onCVEAction(cve,
plugin,
action,
fields=dict(request.args),
**self.pluginArgs)
if type(response) is bool and response is True:
return jsonify({'status': 'plugin_action_complete'})
elif type(response) is bool and response is False or response is None:
return jsonify({'status': 'plugin_action_failed'})
elif type(response) is dict:
return jsonify(response)
# /admin
# /admin/
def admin(self):
if Configuration.loginRequired():
if not current_user.is_authenticated():
return render_template('login.html')
else:
person = User.get("_dummy_", self.auth_handler)
login_user(person)
output = None
master = db.isMasterAccount(current_user.get_id())
checked = ct.checkCronJobExists('cve_search')
if os.path.isfile(Configuration.getUpdateLogFile()):
with open(Configuration.getUpdateLogFile()) as updateFile:
separator = "==========================\n"
output = updateFile.read().split(separator)[-2:]
output = separator + separator.join(output)
return render_template('admin.html',
status="default",
master=master,
checked=checked,
**self.adminInfo(output))
# /admin/change_pass
@login_required
def change_pass(self):
current_pass = request.args.get('current_pass')
new_pass = request.args.get('new_pass')
if current_user.authenticate(current_pass):
if new_pass:
db.changePassword(current_user.id, new_pass)
return jsonify({"status": "password_changed"})
return jsonify({"status": "no_password"})
else:
return jsonify({"status": "wrong_user_pass"})
# /admin/request_token
@login_required
def request_token(self):
return jsonify({"token": db.generateToken(current_user.id)})
# /admin/updatedb
@login_required
def updatedb(self):
process = subprocess.Popen([
sys.executable,
os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str(
err, 'utf-8')) if err else str(out, 'utf-8')
return jsonify({"updateOutput": output, "status": "db_updated"})
# /admin/enableupdatedb
@login_required
def enableupdatedb(self):
status = request.args['checked']
if status == 'true':
retval = ct.createCrontab()
return jsonify({
"status": "crontab_created",
"crontabupdate": "cve_search"
})
else:
retval = ct.removeCronTab()
return jsonify({
"status": "crontab_removed",
"crontabupdate": "cve_search"
})
# /admin/enableupdatedb/status
@login_required
def enableupdatedbstatus(self):
logging.info('Retriving information about cron')
# /admin/whitelist
# /admin/blacklist
@login_required
def listView(self):
if request.url_rule.rule.split('/')[2].lower() == 'whitelist':
return render_template('list.html',
rules=db.getWhitelist(
current_user.get_id()),
user=current_user.get_id(),
listType="Whitelist")
else:
return render_template('list.html',
rules=db.getBlacklist(
current_user.get_id()),
user=current_user.get_id(),
listType="Blacklist")
# /admin/whitelist/import
# /admin/blacklist/import
@login_required
def listImport(self, force=None, path=None):
_list = request.url_rule.split('/')[2]
file = request.files['file']
force = request.form.get('force')
count = wl.countWhitelist(
) if _list.lower == 'whitelist' else bl.countBlacklist()
if (count == 0) | (not count) | (force == "f"):
if _list.lower == 'whitelist':
wl.dropWhitelist()
wl.importWhitelist(TextIOWrapper(file.stream))
else:
bl.dropBlacklist()
bl.importBlacklist(TextIOWrapper(file.stream))
status = _list[0] + "l_imported"
else:
status = _list[0] + "l_already_filled"
return render_template('admin.html', status=status, **self.adminInfo())
# /admin/whitelist/export
# /admin/blacklist/export
@login_required
def listExport(self, force=None, path=None):
_list = request.url_rule.rule.split('/')[2]
bytIO = BytesIO()
data = wl.exportWhitelist(
) if _list.lower == 'whitelist' else bl.exportBlacklist()
bytIO.write(bytes(data, "utf-8"))
bytIO.seek(0)
return send_file(bytIO,
as_attachment=True,
attachment_filename=_list + ".txt")
#TODO: we want that only user can drop list that he owns
# /admin/whitelist/drop
# /admin/blacklist/drop
@login_required
def listDrop(self):
_list = str(request.url_rule).split('/')[2].lower()
logging.info('User {0} is droping {1}'.format(current_user.get_id(),
_list))
if _list == 'whitelist':
wl.dropWhitelist(current_user.get_id())
else:
bl.dropBlacklist(current_user.get_id())
return jsonify({"status": _list[0] + "l_dropped"})
# /admin/addToList
@login_required
def listAdd(self):
cpe = request.args.get('cpe')
cpeType = request.args.get('type')
lst = request.args.get('list')
isglobal = False
if db.isMasterAccount(current_user.get_id()):
isglobal = True
logging.info(
"CPE:{0} cpeType:{1} lst:{2} isglobal:{3} user:{4}".format(
cpe, cpeType, lst, isglobal, current_user.get_id()))
if cpe and cpeType and lst:
status = "added_to_list" if self.addCPEToList(
cpe, lst, cpeType, isglobal,
current_user.get_id()) else "already_exists_in_list"
print(status)
returnList = db.getWhitelist(user=current_user.get_id(
)) if lst == "Whitelist" else db.getBlacklist(
user=current_user.get_id())
pprint(returnList)
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
else:
return jsonify({"status": "could_not_add_to_list"})
# /admin/removeFromList
@login_required
def listRemove(self):
cpe = request.args.get('cpe', type=str)
cpe = urllib.parse.quote_plus(cpe).lower()
user = current_user.get_id()
cpe = cpe.replace("%3a", ":")
cpe = cpe.replace("%2f", "/")
lst = request.args.get('list', type=str)
if cpe and lst:
result = wl.removeWhitelist(
cpe,
user) if lst.lower() == "whitelist" else bl.removeBlacklist(
cpe, user)
status = "removed_from_list" if (
result > 0) else "already_removed_from_list"
else:
status = "invalid_cpe"
returnList = db.getWhitelist(user=current_user.get_id(
)) if lst == "Whitelist" else db.getBlacklist(
user=current_user.get_id())
pprint("Returned list {0}".format(returnList))
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
# /admin/editInList
@login_required
def listEdit(self):
old = request.args.get('oldCPE')
new = request.args.get('cpe')
lst = request.args.get('list')
CPEType = request.args.get('type')
if old and new:
result = wl.updateWhitelist(
old, new,
CPEType) if lst == "whitelist" else bl.updateBlacklist(
old, new, CPEType)
status = "cpelist_updated" if (result) else "cpelist_update_failed"
else:
status = "invalid_cpe"
returnList = list(db.getWhitelist()) if lst == "whitelist" else list(
db.getBlacklist())
return jsonify({
"rules": returnList,
"status": status,
"listType": lst
})
# /admin/listmanagement/<vendor>/<product>
# /admin/listmanagement/<vendor>
# /admin/listmanagement
@login_required
def listManagement(self, vendor=None, product=None):
try:
if product is None:
# no product selected yet, so same function as /browse can be used
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
browseList = query.getBrowseList(vendor)
vendor = browseList["vendor"]
product = browseList["product"]
version = None
else:
# product selected, product versions required
version = query.getVersionsOfProduct(
urllib.parse.quote_plus(product).lower())
return render_template('listmanagement.html',
vendor=vendor,
product=product,
version=version)
except redisExceptions.ConnectionError:
return render_template('error.html',
status={
'except': 'redis-connection',
'info': {
'host':
Configuration.getRedisHost(),
'port':
Configuration.getRedisPort()
}
})
# /admin/listmanagement/add
@login_required
def listManagementAdd(self):
# retrieve the separate item parts
item = request.args.get('item', type=str)
pprint("item0 {0}".format(item))
listType = request.args.get('list', type=str)
isadmin = db.isMasterAccount(current_user.get_id())
pattern = re.compile('^[a-z:0-9.~_%-]+$')
if pattern.match(item):
item = item.split(":")
added = False
if len(item) == 1:
# only vendor, so a check on cpe type is needed
logging.info(
"listManagementAdd: Adding from level 1:{0}".format(
item[0]))
if self.redisdb.sismember("t:/o", item[0]):
if self.addCPEToList("cpe:/o:" + item[0],
listType,
isglobal=isadmin,
user=current_user.get_id()):
added = True
if self.redisdb.sismember("t:/a", item[0]):
if self.addCPEToList("cpe:/a:" + item[0],
listType,
isglobal=isadmin,
user=current_user.get_id()):
added = True
if self.redisdb.sismember("t:/h", item[0]):
if self.addCPEToList("cpe:/h:" + item[0],
listType,
isglobal=isadmin,
user=current_user.get_id()):
added = True
elif 4 > len(item) > 1:
logging.info(
"size is bigger than, look for item[1]: {0}".format(
item[1]))
# cpe type can be found with a mongo regex query
result = db.getCVEs(query={'cpe_2_2': {
'$regex': item[1]
}},
collection="cpe")
if len(result) != 0:
prefix = ((result[0])['cpe_2_2'])[:7]
logging.info(
"listManagementAdd: Adding from level 2:{0}{1}{2}".
format(prefix, item[0], item[1]))
if len(item) == 2:
if self.addCPEToList(prefix + item[0] + ":" + item[1],
listType,
isglobal=isadmin,
user=current_user.get_id()):
added = True
if len(item) == 3:
if self.addCPEToList(prefix + item[0] + ":" + item[1] +
":" + item[2],
listType,
isglobal=isadmin,
user=current_user.get_id()):
added = True
status = "added_to_list" if added else "could_not_add_to_list"
else:
status = "invalid_cpe"
j = {"status": status, "listType": listType}
return jsonify(j)
# /login
def login_check(self):
# validate username and password
username = request.form.get('username')
password = request.form.get('password')
person = User.get(username, self.auth_handler)
try:
if person and person.authenticate(password):
login_user(person)
return redirect('admin')
else:
return render_template('login.html', status="wrong_user_pass")
except Exception as e:
print(e)
return render_template('login.html', status="outdated_database")
# /logout
@login_required
def logout(self):
logout_user()
return redirect("/")
class Index(Minimal, Advanced_API):
""" Redis Functions """
def vendors(self, r):
return r.sunion('t:/a', 't:/h', 't:/o')
def vendor_products(self, r, vendor):
allproduct = []
products = r.smembers('v:' + vendor)
for product in products:
allproduct.append(product)
return allproduct
def get_vendor(self, r, product):
result = []
allvendors = r.sunion('t:/a', 't:/h', 't:/o')
for vendor in allvendors:
if product in self.vendor_products(r, vendor):
result.append(vendor)
return result
def product_versions(self, r, product):
allversion = []
versions = r.smembers('p:' + product)
for version in versions:
allversion.append(version)
return allversion
def search_vendor(self, r, search):
result = []
for vendor in self.vendors(r):
if search in vendor:
result.append(vendor)
return result
def search_product(self, r, search):
result = []
for vendor in self.vendors(r):
for product in vendor_products(r, vendor):
if search in product:
result.append(product)
return result
def search_vendor_product(self, r, search, vendor):
result = []
for product in self.vendor_products(r, vendor):
if search in product:
result.append(product)
return result
#############
# Variables #
#############
def __init__(self):
# TODO: make auth handler and plugin manager singletons
Advanced_API.__init__(self)
Minimal.__init__(self)
self.minimal = False
self.auth_handler = AuthenticationHandler()
self.plugManager = PluginManager()
self.login_manager = LoginManager()
self.plugManager.loadPlugins()
self.login_manager.init_app(self.app)
self.login_manager.user_loader(self.load_user)
self.redisdb = Configuration.getRedisVendorConnection()
self.defaultFilters.update({
'blacklistSelect': 'on',
'whitelistSelect': 'on',
'unlistedSelect': 'show',
})
self.args.update({'minimal': False})
self.pluginArgs = {
"current_user": current_user,
"plugin_manager": self.plugManager
}
routes = [{
'r': '/cve/<cveid>',
'm': ['GET'],
'f': self.cve
}, {
'r': '/_get_plugins',
'm': ['GET'],
'f': self._get_plugins
}, {
'r': '/plugin/_get_cve_actions',
'm': ['GET'],
'f': self._get_cve_actions
}, {
'r': '/plugin/<plugin>',
'm': ['GET'],
'f': self.openPlugin
}, {
'r': '/plugin/<plugin>/subpage/<page>',
'm': ['GET'],
'f': self.openPluginSubpage
}, {
'r': '/plugin/<plugin>/_cve_action/<action>',
'm': ['GET'],
'f': self._jsonCVEAction
}, {
'r': '/login',
'm': ['POST'],
'f': self.login_check
}, {
'r': '/logout',
'm': ['GET'],
'f': self.logout
}, {
'r': '/admin',
'm': ['GET'],
'f': self.admin
}, {
'r': '/admin/',
'm': ['GET'],
'f': self.admin
}, {
'r': '/admin/change_pass',
'm': ['GET'],
'f': self.change_pass
}, {
'r': '/admin/request_token',
'm': ['GET'],
'f': self.request_token
}, {
'r': '/admin/updatedb',
'm': ['GET'],
'f': self.updatedb
}, {
'r': '/admin/whitelist/import',
'm': ['POST'],
'f': self.listImport
}, {
'r': '/admin/blacklist/import',
'm': ['POST'],
'f': self.listImport
}, {
'r': '/admin/whitelist/export',
'm': ['GET'],
'f': self.listExport
}, {
'r': '/admin/blacklist/export',
'm': ['GET'],
'f': self.listExport
}, {
'r': '/admin/whitelist/drop',
'm': ['POST'],
'f': self.listDrop
}, {
'r': '/admin/blacklist/drop',
'm': ['POST'],
'f': self.listDrop
}, {
'r': '/admin/whitelist',
'm': ['GET'],
'f': self.listView
}, {
'r': '/admin/blacklist',
'm': ['GET'],
'f': self.listView
}, {
'r': '/admin/addToList',
'm': ['GET'],
'f': self.listAdd
}, {
'r': '/admin/removeFromList',
'm': ['GET'],
'f': self.listRemove
}, {
'r': '/admin/editInList',
'm': ['GET'],
'f': self.listEdit
}, {
'r': '/admin/listmanagement',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/<vendor>',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/<vendor>/<product>',
'm': ['GET'],
'f': self.listManagement
}, {
'r': '/admin/listmanagement/add',
'm': ['GET'],
'f': self.listManagementAdd
}, {
'r': '/login',
'm': ['POST'],
'f': self.login_check
}, {
'r': '/notifications',
'm': ['GET'],
'f': self.notifications
}, {
'r': '/notifications',
'm': ['POST'],
'f': self.notifications_add
}, {
'r': '/notifications',
'm': ['DELETE'],
'f': self.notifications_delete
}, {
'r': '/notiftab',
'm': ['GET'],
'f': self.notiftab
}, {
'r': '/notifjson',
'm': ['POST'],
'f': self.notifjson
}]
for route in routes:
self.addRoute(route)
#############
# Functions #
#############
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
if current_user.is_authenticated():
if f['blacklistSelect'] == "on":
regexes = db.getRules('blacklist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['whitelistSelect'] == "hide":
regexes = db.getRules('whitelist')
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
'$or': [{
'vulnerable_configuration': re.compile(exp)
}, {
'vulnerable_configuration': {
'$exists': False
}
}, {
'vulnerable_configuration': []
}]
})
if f['unlistedSelect'] == "hide":
wlregexes = tk.compile(db.getRules('whitelist'))
blregexes = tk.compile(db.getRules('blacklist'))
query.append({
'$or': [{
'vulnerable_configuration': {
'$in': wlregexes
}
}, {
'vulnerable_configuration': {
'$in': blregexes
}
}]
})
return query
def markCPEs(self, cve):
blacklist = tk.compile(db.getRules('blacklist'))
whitelist = tk.compile(db.getRules('whitelist'))
for conf in cve['vulnerable_configuration']:
conf['list'] = 'none'
conf['match'] = 'none'
for w in whitelist:
if w.match(conf['id']):
conf['list'] = 'white'
conf['match'] = w
for b in blacklist:
if b.match(conf['id']):
conf['list'] = 'black'
conf['match'] = b
return cve
def filter_logic(self, filters, skip, limit=None):
query = self.generate_full_query(filters)
limit = limit if limit else self.args['pageLength']
cve = db.getCVEs(limit=limit, skip=skip, query=query)
# marking relevant records
if current_user.is_authenticated():
if filters['whitelistSelect'] == "on":
cve['results'] = self.list_mark('white', cve['results'])
if filters['blacklistSelect'] == "mark":
cve['results'] = self.list_mark('black', cve['results'])
self.plugManager.mark(cve, **self.pluginArgs)
cve = list(cve)
return cve
def addCPEToList(self, cpe, listType, cpeType=None):
def addCPE(cpe, cpeType, funct):
return True if funct(cpe, cpeType) else False
if not cpeType: cpeType = 'cpe'
if listType.lower() in ("blacklist", "black", "b", "bl"):
return addCPE(cpe, cpeType, bl.insertBlacklist)
if listType.lower() in ("whitelist", "white", "w", "wl"):
return addCPE(cpe, cpeType, wl.insertWhitelist)
def list_mark(self, listed, cveList):
if listed not in ['white', 'black']: return list(cves)
items = tk.compile(db.getRules(listed + 'list'))
# check the cpes (full or partially) in the black/whitelist
for i, cve in enumerate(
list(cveList)
): # the list() is to ensure we don't have a pymongo cursor object
for c in cve['vulnerable_configuration']:
if any(regex.match(c) for regex in items):
cveList[i][listed + 'listed'] = 'yes'
return cveList
def filterUpdateField(self, data):
if not data: return data
returnvalue = []
for line in data.split("\n"):
if (not line.startswith("[+]Success to create index")
and not line == "Not modified"
and not line.startswith("Starting")):
returnvalue.append(line)
return "\n".join(returnvalue)
def adminInfo(self, output=None):
return {
'stats': db.getDBStats(True),
'plugins': self.plugManager.getPlugins(),
'updateOutput': self.filterUpdateField(output),
'token': db.getToken(current_user.id)
}
# user management
def load_user(self, id):
return User.get(id, self.auth_handler)
##########
# ROUTES #
##########
# /cve/<cveid>
def cve(self, cveid):
cveid = cveid.upper()
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True,
subscorelookup=True)
cve = cvesp.getcve(cveid=cveid)
if cve is None:
return render_template('error.html',
status={
'except': 'cve-not-found',
'info': {
'cve': cveid
}
})
cve = self.markCPEs(cve)
self.plugManager.onCVEOpen(cveid, **self.pluginArgs)
pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs)
return render_template('cve.html', cve=cve, plugins=pluginData)
# /_get_plugins
def _get_plugins(self):
if not current_user.is_authenticated(
): # Don't show plugins requiring auth if not authenticated
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs) if not x.requiresAuth]
else:
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs)]
return jsonify({"plugins": plugins})
# /plugin/_get_cve_actions
def _get_cve_actions(self):
cve = request.args.get('cve', type=str)
if not current_user.is_authenticated(
): # Don't show actions requiring auth if not authenticated
actions = [
x for x in self.plugManager.getCVEActions(
cve, **self.pluginArgs) if not x['auth']
]
else:
actions = self.plugManager.getCVEActions(cve, **self.pluginArgs)
return jsonify({"actions": actions})
# /plugin/<plugin>
def openPlugin(self, plugin):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openPage(plugin, **self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
else:
abort(404)
# /plugin/<plugin>/subpage/<page>
def openPluginSubpage(self, plugin, page):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openSubpage(plugin, page,
**self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
else:
abort(404)
# /plugin/<plugin>/_cve_action/<action>
def _jsonCVEAction(self, plugin, action):
cve = request.args.get('cve', type=str)
response = self.plugManager.onCVEAction(cve,
plugin,
action,
fields=dict(request.args),
**self.pluginArgs)
if type(response) is bool and response is True:
return jsonify({'status': 'plugin_action_complete'})
elif type(response) is bool and response is False or response is None:
return jsonify({'status': 'plugin_action_failed'})
elif type(response) is dict:
return jsonify(response)
# /admin
# /admin/
def admin(self):
if Configuration.loginRequired():
if not current_user.is_authenticated():
return render_template('login.html')
else:
person = User.get("_dummy_", self.auth_handler)
login_user(person)
output = None
if os.path.isfile(Configuration.getUpdateLogFile()):
with open(Configuration.getUpdateLogFile()) as updateFile:
separator = "==========================\n"
output = updateFile.read().split(separator)[-2:]
output = separator + separator.join(output)
return render_template('admin.html',
status="default",
**self.adminInfo(output))
# /admin/change_pass
@login_required
def change_pass(self):
current_pass = request.args.get('current_pass')
new_pass = request.args.get('new_pass')
if current_user.authenticate(current_pass):
if new_pass:
db.changePassword(current_user.id, new_pass)
return jsonify({"status": "password_changed"})
return jsonify({"status": "no_password"})
else:
return jsonify({"status": "wrong_user_pass"})
# /admin/request_token
@login_required
def request_token(self):
return jsonify({"token": db.generateToken(current_user.id)})
# /admin/updatedb
@login_required
def updatedb(self):
process = subprocess.Popen([
sys.executable,
os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str(
err, 'utf-8')) if err else str(out, 'utf-8')
return jsonify({"updateOutput": output, "status": "db_updated"})
# /admin/whitelist
# /admin/blacklist
@login_required
def listView(self):
if request.url_rule.rule.split('/')[2].lower() == 'whitelist':
return render_template('list.html',
rules=db.getWhitelist(),
listType="Whitelist")
else:
return render_template('list.html',
rules=db.getBlacklist(),
listType="Blacklist")
# /admin/whitelist/import
# /admin/blacklist/import
@login_required
def listImport(self, force=None, path=None):
_list = request.url_rule.split('/')[2]
file = request.files['file']
force = request.form.get('force')
count = wl.countWhitelist(
) if _list.lower == 'whitelist' else bl.countBlacklist()
if (count == 0) | (not count) | (force == "f"):
if _list.lower == 'whitelist':
wl.dropWhitelist()
wl.importWhitelist(TextIOWrapper(file.stream))
else:
bl.dropBlacklist()
bl.importBlacklist(TextIOWrapper(file.stream))
status = _list[0] + "l_imported"
else:
status = _list[0] + "l_already_filled"
return render_template('admin.html', status=status, **self.adminInfo())
# /admin/whitelist/export
# /admin/blacklist/export
@login_required
def listExport(self, force=None, path=None):
_list = request.url_rule.rule.split('/')[2]
bytIO = BytesIO()
data = wl.exportWhitelist(
) if _list.lower == 'whitelist' else bl.exportBlacklist()
bytIO.write(bytes(data, "utf-8"))
bytIO.seek(0)
return send_file(bytIO,
as_attachment=True,
attachment_filename=_list + ".txt")
# /admin/whitelist/drop
# /admin/blacklist/drop
@login_required
def listDrop(self):
_list = request.url_rule.split('/')[2].lower()
if _list == 'whitelist':
wl.dropWhitelist()
else:
bl.dropBlacklist()
return jsonify({"status": _list[0] + "l_dropped"})
# /admin/addToList
@login_required
def listAdd(self):
cpe = request.args.get('cpe')
cpeType = request.args.get('type')
lst = request.args.get('list')
if cpe and cpeType and lst:
status = "added_to_list" if self.addCPEToList(
cpe, lst, cpeType) else "already_exists_in_list"
returnList = db.getWhitelist(
) if lst == "whitelist" else db.getBlacklist()
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
else:
return jsonify({"status": "could_not_add_to_list"})
# /admin/removeFromList
@login_required
def listRemove(self):
cpe = request.args.get('cpe', type=str)
cpe = urllib.parse.quote_plus(cpe).lower()
cpe = cpe.replace("%3a", ":")
cpe = cpe.replace("%2f", "/")
lst = request.args.get('list', type=str)
if cpe and lst:
result = wl.removeWhitelist(
cpe) if lst.lower() == "whitelist" else bl.removeBlacklist(cpe)
status = "removed_from_list" if (
result > 0) else "already_removed_from_list"
else:
status = "invalid_cpe"
returnList = db.getWhitelist(
) if lst == "whitelist" else db.getBlacklist()
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
# /admin/editInList
@login_required
def listEdit(self):
old = request.args.get('oldCPE')
new = request.args.get('cpe')
lst = request.args.get('list')
CPEType = request.args.get('type')
if old and new:
result = wl.updateWhitelist(
old, new,
CPEType) if lst == "whitelist" else bl.updateBlacklist(
old, new, CPEType)
status = "cpelist_updated" if (result) else "cpelist_update_failed"
else:
status = "invalid_cpe"
returnList = list(db.getWhitelist()) if lst == "whitelist" else list(
db.getBlacklist())
return jsonify({
"rules": returnList,
"status": status,
"listType": lst
})
# /admin/listmanagement/<vendor>/<product>
# /admin/listmanagement/<vendor>
# /admin/listmanagement
@login_required
def listManagement(self, vendor=None, product=None):
try:
if product is None:
# no product selected yet, so same function as /browse can be used
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
browseList = query.getBrowseList(vendor)
vendor = browseList["vendor"]
product = browseList["product"]
version = None
else:
# product selected, product versions required
version = query.getVersionsOfProduct(
urllib.parse.quote_plus(product).lower())
return render_template('listmanagement.html',
vendor=vendor,
product=product,
version=version)
except redisExceptions.ConnectionError:
return render_template('error.html',
status={
'except': 'redis-connection',
'info': {
'host':
Configuration.getRedisHost(),
'port':
Configuration.getRedisPort()
}
})
# /admin/listmanagement/add
@login_required
def listManagementAdd(self):
# retrieve the separate item parts
item = request.args.get('item', type=str)
listType = request.args.get('list', type=str)
pattern = re.compile('^[a-z:/0-9.~_%-]+$')
if pattern.match(item):
item = item.split(":")
added = False
if len(item) == 1:
# only vendor, so a check on cpe type is needed
if self.redisdb.sismember("t:/o", item[0]):
if self.addCPEToList("cpe:/o:" + item[0], listType):
added = True
if self.redisdb.sismember("t:/a", item[0]):
if self.addCPEToList("cpe:/a:" + item[0], listType):
added = True
if self.redisdb.sismember("t:/h", item[0]):
if self.addCPEToList("cpe:/h:" + item[0], listType):
added = True
elif 4 > len(item) > 1:
# cpe type can be found with a mongo regex query
result = db.getCVEs(query={'cpe_2_2': {
'$regex': item[1]
}})['results']
if result.count() != 0:
prefix = ((result[0])['cpe_2_2'])[:7]
if len(item) == 2:
if self.addCPEToList(prefix + item[0] + ":" + item[1],
listType):
added = True
if len(item) == 3:
if self.addCPEToList(
prefix + item[0] + ":" + item[1] + ":" +
item[2], listType):
added = True
status = "added_to_list" if added else "could_not_add_to_list"
else:
status = "invalid_cpe"
j = {"status": status, "listType": listType}
return jsonify(j)
# /login
def login_check(self):
# validate username and password
username = request.form.get('username')
password = request.form.get('password')
person = User.get(username, self.auth_handler)
try:
if person and person.authenticate(password):
login_user(person)
return render_template('admin.html',
status="logged_in",
**self.adminInfo())
else:
return render_template('login.html', status="wrong_user_pass")
except Exception as e:
print(e)
return render_template('login.html', status="outdated_database")
# /logout
@login_required
def logout(self):
logout_user()
return redirect("/")
# /notifications
@login_required
def notifications(self):
return render_template('notifications.html')
# /notifications POST
@login_required
def notifications_add(self):
self.app.logger.info(request.json['notifemail'])
if request.json["notifemail"]:
email = request.json["notifemail"]
if request.json['allversion'] is True and request.json[
'allproduct'] is False:
notification = self.Notification(
user=current_user.id,
email=email,
vendor=escape(request.json['queryvendor'].lower()),
product=escape(request.json['queryproduct'].lower()),
version='')
elif request.json['allproduct'] is True:
notification = self.Notification(
user=current_user.id,
email=email,
vendor=escape(request.json['queryvendor'].lower()),
product='',
version='')
else:
notification = self.Notification(
user=current_user.id,
email=email,
vendor=escape(request.json['queryvendor'].lower()),
product=escape(request.json['queryproduct'].lower()),
version=escape(request.json['queryversion'].lower()))
# Checking Integrity Before Insert #
if self.Notification.query.filter_by(
user=notification.user,
email=email,
vendor=notification.vendor,
product=notification.product,
version=notification.version).first() is None:
self.session.add(notification)
self.session.commit()
flash('Notification Successfully Created.', 'success')
return "{}", 200
else:
flash('Notification Already existing.', 'warning')
return "{}", 200
else:
return "{}", 400
# /notifications DELETE
@login_required
def notifications_delete(self):
row = self.Notification.query.filter_by(id=request.json).first()
self.session.delete(row)
self.session.commit()
flash('Notification removed ', 'info')
return "{}", 200
# /notiftab
@login_required
def notiftab(self):
jnotif = []
dic = {}
limit = request.args.get('limit')
offset = request.args.get('offset')
sort = request.args.get('sort')
order = request.args.get('order')
if order == 'desc':
notif_list = self.Notification.query.filter_by(
user=current_user.id).order_by(
desc(sort)).limit(limit).offset(offset).all()
else:
notif_list = self.Notification.query.filter_by(
user=current_user.id).order_by(sort).limit(limit).offset(
offset).all()
num = self.Notification.query.filter_by(user=current_user.id).count()
for notif in notif_list:
dnotif = {
'id': notif.id,
'fulltxt': notif.fulltxt,
'vendor': notif.vendor,
'product': notif.product,
'version': notif.version,
'email': notif.email
}
jnotif.append(dnotif)
dic['total'] = num
dic['rows'] = jnotif
return jsonify(dic)
@login_required
def notifjson(self):
vendors = []
products = []
versions = []
if request.json is not None:
if request.json['queryvendor'] != "": # SEARCH BY VENDORS
vendors = self.search_vendor(
self.redisdb, request.json['queryvendor'].lower())
products = self.search_vendor_product(
self.redisdb, request.json['queryproduct'].lower(),
request.json['queryvendor'].lower())
versions = self.product_versions(
self.redisdb, request.json['queryproduct'].lower())
elif request.json['queryproduct'] != "": # SEARCH BY PRODUCTS
products = self.search_product(
self.redisdb, request.json['queryproduct'].lower())
vendors = self.get_vendor(self.redisdb,
request.json['queryproduct'].lower())
versions = self.product_versions(
self.redisdb, request.json['queryproduct'].lower())
return jsonify({
"vendors": vendors,
"products": products,
"versions": versions
})
class Index(Minimal, Advanced_API):
#############
# Variables #
#############
def __init__(self):
# TODO: make auth handler and plugin manager singletons
Advanced_API.__init__(self)
Minimal.__init__(self)
self.minimal = False
self.auth_handler = AuthenticationHandler()
self.plugManager = PluginManager()
self.login_manager = LoginManager()
self.plugManager.loadPlugins()
self.login_manager.init_app(self.app)
self.login_manager.user_loader(self.load_user)
self.redisdb = Configuration.getRedisVendorConnection()
self.defaultFilters.update({
"blacklistSelect": "on",
"whitelistSelect": "on",
"unlistedSelect": "show",
})
self.args.update({"minimal": False})
self.pluginArgs = {
"current_user": current_user,
"plugin_manager": self.plugManager,
}
routes = [
{
"r": "/cve/<cveid>",
"m": ["GET"],
"f": self.cve
},
{
"r": "/_get_plugins",
"m": ["GET"],
"f": self._get_plugins
},
{
"r": "/plugin/_get_cve_actions",
"m": ["GET"],
"f": self._get_cve_actions
},
{
"r": "/plugin/<plugin>",
"m": ["GET"],
"f": self.openPlugin
},
{
"r": "/plugin/<plugin>/subpage/<page>",
"m": ["GET"],
"f": self.openPluginSubpage,
},
{
"r": "/plugin/<plugin>/_cve_action/<action>",
"m": ["GET"],
"f": self._jsonCVEAction,
},
{
"r": "/login",
"m": ["POST"],
"f": self.login_check
},
{
"r": "/logout",
"m": ["GET"],
"f": self.logout
},
{
"r": "/admin",
"m": ["GET"],
"f": self.admin
},
{
"r": "/admin/",
"m": ["GET"],
"f": self.admin
},
{
"r": "/admin/change_pass",
"m": ["GET"],
"f": self.change_pass
},
{
"r": "/admin/request_token",
"m": ["GET"],
"f": self.request_token
},
{
"r": "/admin/updatedb",
"m": ["GET"],
"f": self.updatedb
},
{
"r": "/admin/whitelist/import",
"m": ["POST"],
"f": self.listImport
},
{
"r": "/admin/blacklist/import",
"m": ["POST"],
"f": self.listImport
},
{
"r": "/admin/whitelist/export",
"m": ["GET"],
"f": self.listExport
},
{
"r": "/admin/blacklist/export",
"m": ["GET"],
"f": self.listExport
},
{
"r": "/admin/whitelist/drop",
"m": ["POST"],
"f": self.listDrop
},
{
"r": "/admin/blacklist/drop",
"m": ["POST"],
"f": self.listDrop
},
{
"r": "/admin/whitelist",
"m": ["GET"],
"f": self.listView
},
{
"r": "/admin/blacklist",
"m": ["GET"],
"f": self.listView
},
{
"r": "/admin/addToList",
"m": ["GET"],
"f": self.listAdd
},
{
"r": "/admin/removeFromList",
"m": ["GET"],
"f": self.listRemove
},
{
"r": "/admin/editInList",
"m": ["GET"],
"f": self.listEdit
},
{
"r": "/admin/listmanagement",
"m": ["GET"],
"f": self.listManagement
},
{
"r": "/admin/listmanagement/<vendor>",
"m": ["GET"],
"f": self.listManagement,
},
{
"r": "/admin/listmanagement/<vendor>/<product>",
"m": ["GET"],
"f": self.listManagement,
},
{
"r": "/admin/listmanagement/add",
"m": ["GET"],
"f": self.listManagementAdd,
},
{
"r": "/login",
"m": ["POST"],
"f": self.login_check
},
]
for route in routes:
self.addRoute(route)
#############
# Functions #
#############
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
if current_user.is_authenticated:
if f["blacklistSelect"] == "on":
regexes = getRules("blacklist")
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
"$or": [
{
"vulnerable_configuration": re.compile(exp)
},
{
"vulnerable_configuration": {
"$exists": False
}
},
{
"vulnerable_configuration": []
},
]
})
if f["whitelistSelect"] == "hide":
regexes = getRules("whitelist")
if len(regexes) != 0:
exp = "^(?!" + "|".join(regexes) + ")"
query.append({
"$or": [
{
"vulnerable_configuration": re.compile(exp)
},
{
"vulnerable_configuration": {
"$exists": False
}
},
{
"vulnerable_configuration": []
},
]
})
if f["unlistedSelect"] == "hide":
wlregexes = tk_compile(getRules("whitelist"))
blregexes = tk_compile(getRules("blacklist"))
query.append({
"$or": [
{
"vulnerable_configuration": {
"$in": wlregexes
}
},
{
"vulnerable_configuration": {
"$in": blregexes
}
},
]
})
return query
def markCPEs(self, cve):
blacklist = tk_compile(getRules("blacklist"))
whitelist = tk_compile(getRules("whitelist"))
for conf in cve["vulnerable_configuration"]:
conf["list"] = "none"
conf["match"] = "none"
for w in whitelist:
if w.match(conf["id"]):
conf["list"] = "white"
conf["match"] = w
for b in blacklist:
if b.match(conf["id"]):
conf["list"] = "black"
conf["match"] = b
return cve
def filter_logic(self, filters, skip, limit=None):
query = self.generate_full_query(filters)
limit = limit if limit else self.args["pageLength"]
cve = getCVEs(limit=limit, skip=skip, query=query)
# marking relevant records
if current_user.is_authenticated:
if filters["whitelistSelect"] == "on":
cve["results"] = self.list_mark("white", cve["results"])
if filters["blacklistSelect"] == "mark":
cve["results"] = self.list_mark("black", cve["results"])
self.plugManager.mark(cve, **self.pluginArgs)
return cve
def addCPEToList(self, cpe, listType, cpeType=None):
def addCPE(cpe, cpeType, funct):
return True if funct(cpe, cpeType) else False
if not cpeType:
cpeType = "cpe"
if listType.lower() in ("blacklist", "black", "b", "bl"):
return addCPE(cpe, cpeType, insertBlacklist)
if listType.lower() in ("whitelist", "white", "w", "wl"):
return addCPE(cpe, cpeType, insertWhitelist)
def list_mark(self, listed, cveList):
if listed not in ["white", "black"]:
return list(cveList)
items = tk_compile(getRules(listed + "list"))
# check the cpes (full or partially) in the black/whitelist
for i, cve in enumerate(
list(cveList)
): # the list() is to ensure we don't have a pymongo cursor object
for c in cve["vulnerable_configuration"]:
if any(regex.match(c) for regex in items):
cveList[i][listed + "listed"] = "yes"
return cveList
def filterUpdateField(self, data):
if not data:
return data
returnvalue = []
for line in data.split("\n"):
if (not line.startswith("[+]Success to create index")
and not line == "Not modified"
and not line.startswith("Starting")):
returnvalue.append(line)
return "\n".join(returnvalue)
def adminInfo(self, output=None):
return {
"stats": getDBStats(True),
"plugins": self.plugManager.getPlugins(),
"updateOutput": self.filterUpdateField(output),
"token": getToken(current_user.id),
}
# user management
def load_user(self, id):
return User.get(id, self.auth_handler)
##########
# ROUTES #
##########
# /cve/<cveid>
def cve(self, cveid):
cveid = cveid.upper()
cvesp = CveHandler(
rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True,
subscorelookup=True,
)
cve = cvesp.getcve(cveid=cveid)
if cve is None:
return render_template("error.html",
status={
"except": "cve-not-found",
"info": {
"cve": cveid
}
})
cve = self.markCPEs(cve)
self.plugManager.onCVEOpen(cveid, **self.pluginArgs)
pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs)
return render_template("cve.html", cve=cve, plugins=pluginData)
# /_get_plugins
def _get_plugins(self):
if (not current_user.is_authenticated
): # Don't show plugins requiring auth if not authenticated
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs) if not x.requiresAuth]
else:
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs)]
return jsonify({"plugins": plugins})
# /plugin/_get_cve_actions
def _get_cve_actions(self):
cve = request.args.get("cve", type=str)
if (not current_user.is_authenticated
): # Don't show actions requiring auth if not authenticated
actions = [
x for x in self.plugManager.getCVEActions(
cve, **self.pluginArgs) if not x["auth"]
]
else:
actions = self.plugManager.getCVEActions(cve, **self.pluginArgs)
return jsonify({"actions": actions})
# /plugin/<plugin>
def openPlugin(self, plugin):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated:
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openPage(plugin, **self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={"except": "plugin-page-corrupt"})
except jinja2.exceptions.TemplateNotFound:
return render_template(
"error.html",
status={
"except": "plugin-page-not-found",
"page": page
},
)
else:
abort(404)
# /plugin/<plugin>/subpage/<page>
def openPluginSubpage(self, plugin, page):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated:
return render_template("requiresAuth.html")
else:
page, args = self.plugManager.openSubpage(plugin, page,
**self.pluginArgs)
if page:
try:
return render_template(page, **args)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html", status={"except": "plugin-page-corrupt"})
except jinja2.exceptions.TemplateNotFound:
return render_template(
"error.html",
status={
"except": "plugin-page-not-found",
"page": page
},
)
else:
abort(404)
# /plugin/<plugin>/_cve_action/<action>
def _jsonCVEAction(self, plugin, action):
cve = request.args.get("cve", type=str)
response = self.plugManager.onCVEAction(cve,
plugin,
action,
fields=dict(request.args),
**self.pluginArgs)
if type(response) is bool and response is True:
return jsonify({"status": "plugin_action_complete"})
elif type(response) is bool and response is False or response is None:
return jsonify({"status": "plugin_action_failed"})
elif type(response) is dict:
return jsonify(response)
# /admin
# /admin/
def admin(self):
if Configuration.loginRequired():
if not current_user.is_authenticated:
return render_template("login.html")
else:
person = User.get("_dummy_", self.auth_handler)
login_user(person)
output = None
if os.path.isfile(Configuration.getUpdateLogFile()):
with open(Configuration.getUpdateLogFile()) as updateFile:
separator = "==========================\n"
output = updateFile.read().split(separator)[-2:]
output = separator + separator.join(output)
return render_template("admin.html",
status="default",
**self.adminInfo(output))
# /admin/change_pass
@login_required
def change_pass(self):
current_pass = request.args.get("current_pass")
new_pass = request.args.get("new_pass")
if current_user.authenticate(current_pass):
if new_pass:
changePassword(current_user.id, new_pass)
return jsonify({"status": "password_changed"})
return jsonify({"status": "no_password"})
else:
return jsonify({"status": "wrong_user_pass"})
# /admin/request_token
@login_required
def request_token(self):
return jsonify({"token": generateToken(current_user.id)})
# /admin/updatedb
@login_required
def updatedb(self):
process = subprocess.Popen(
[
sys.executable,
os.path.join(_runPath, "../../../sbin/db_updater.py"),
"-civ",
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out, err = process.communicate()
output = ("%s\n\nErrors:\n%s" % (str(out, "utf-8"), str(err, "utf-8"))
if err else str(out, "utf-8"))
return jsonify({"updateOutput": output, "status": "db_updated"})
# /admin/whitelist
# /admin/blacklist
@login_required
def listView(self):
if request.url_rule.rule.split("/")[2].lower() == "whitelist":
return render_template("list.html",
rules=getWhitelist(),
listType="Whitelist")
else:
return render_template("list.html",
rules=getBlacklist(),
listType="Blacklist")
# /admin/whitelist/import
# /admin/blacklist/import
@login_required
def listImport(self, force=None, path=None):
_list = request.url_rule.split("/")[2]
file = request.files["file"]
force = request.form.get("force")
count = countWhitelist(
) if _list.lower == "whitelist" else countBlacklist()
if (count == 0) | (not count) | (force == "f"):
if _list.lower == "whitelist":
dropWhitelist()
importWhitelist(TextIOWrapper(file.stream))
else:
dropBlacklist()
importBlacklist(TextIOWrapper(file.stream))
status = _list[0] + "l_imported"
else:
status = _list[0] + "l_already_filled"
return render_template("admin.html", status=status, **self.adminInfo())
# /admin/whitelist/export
# /admin/blacklist/export
@login_required
def listExport(self, force=None, path=None):
_list = request.url_rule.rule.split("/")[2]
bytIO = BytesIO()
data = exportWhitelist(
) if _list.lower == "whitelist" else exportBlacklist()
bytIO.write(bytes(data, "utf-8"))
bytIO.seek(0)
return send_file(bytIO,
as_attachment=True,
attachment_filename=_list + ".txt")
# /admin/whitelist/drop
# /admin/blacklist/drop
@login_required
def listDrop(self):
_list = request.url_rule.split("/")[2].lower()
if _list == "whitelist":
dropWhitelist()
else:
dropBlacklist()
return jsonify({"status": _list[0] + "l_dropped"})
# /admin/addToList
@login_required
def listAdd(self):
cpe = request.args.get("cpe")
cpeType = request.args.get("type")
lst = request.args.get("list")
if cpe and cpeType and lst:
status = ("added_to_list" if self.addCPEToList(cpe, lst, cpeType)
else "already_exists_in_list")
returnList = getWhitelist(
) if lst == "whitelist" else getBlacklist()
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
else:
return jsonify({"status": "could_not_add_to_list"})
# /admin/removeFromList
@login_required
def listRemove(self):
cpe = request.args.get("cpe", type=str)
cpe = urllib.parse.quote_plus(cpe).lower()
cpe = cpe.replace("%3a", ":")
cpe = cpe.replace("%2f", "/")
lst = request.args.get("list", type=str)
if cpe and lst:
result = (removeWhitelist(cpe)
if lst.lower() == "whitelist" else removeBlacklist(cpe))
status = ("removed_from_list" if
(result > 0) else "already_removed_from_list")
else:
status = "invalid_cpe"
returnList = getWhitelist() if lst == "whitelist" else getBlacklist()
return jsonify({
"status": status,
"rules": returnList,
"listType": lst.title()
})
# /admin/editInList
@login_required
def listEdit(self):
old = request.args.get("oldCPE")
new = request.args.get("cpe")
lst = request.args.get("list")
CPEType = request.args.get("type")
if old and new:
result = (updateWhitelist(old, new, CPEType) if lst == "whitelist"
else updateBlacklist(old, new, CPEType))
status = "cpelist_updated" if (result) else "cpelist_update_failed"
else:
status = "invalid_cpe"
returnList = (list(getWhitelist())
if lst == "whitelist" else list(getBlacklist()))
return jsonify({
"rules": returnList,
"status": status,
"listType": lst
})
# /admin/listmanagement/<vendor>/<product>
# /admin/listmanagement/<vendor>
# /admin/listmanagement
@login_required
def listManagement(self, vendor=None, product=None):
try:
if product is None:
# no product selected yet, so same function as /browse can be used
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
browseList = getBrowseList(vendor)
vendor = browseList["vendor"]
product = browseList["product"]
version = None
else:
# product selected, product versions required
version = getVersionsOfProduct(
urllib.parse.quote_plus(product).lower())
return render_template("listmanagement.html",
vendor=vendor,
product=product,
version=version)
except redisExceptions.ConnectionError:
return render_template(
"error.html",
status={
"except": "redis-connection",
"info": {
"host": Configuration.getRedisHost(),
"port": Configuration.getRedisPort(),
},
},
)
# /admin/listmanagement/add
@login_required
def listManagementAdd(self):
# retrieve the separate item parts
item = request.args.get("item", type=str)
listType = request.args.get("list", type=str)
pattern = re.compile("^[a-z:/0-9.~_%-]+$")
if pattern.match(item):
item = item.split(":")
added = False
if len(item) == 1:
# only vendor, so a check on cpe type is needed
if self.redisdb.sismember("t:/o", item[0]):
if self.addCPEToList("cpe:/o:" + item[0], listType):
added = True
if self.redisdb.sismember("t:/a", item[0]):
if self.addCPEToList("cpe:/a:" + item[0], listType):
added = True
if self.redisdb.sismember("t:/h", item[0]):
if self.addCPEToList("cpe:/h:" + item[0], listType):
added = True
elif 4 > len(item) > 1:
# cpe type can be found with a mongo regex query
result = getCVEs(query={"cpe_2_2": {
"$regex": item[1]
}})["results"]
if result.count() != 0:
prefix = ((result[0])["cpe_2_2"])[:7]
if len(item) == 2:
if self.addCPEToList(prefix + item[0] + ":" + item[1],
listType):
added = True
if len(item) == 3:
if self.addCPEToList(
prefix + item[0] + ":" + item[1] + ":" +
item[2], listType):
added = True
status = "added_to_list" if added else "could_not_add_to_list"
else:
status = "invalid_cpe"
j = {"status": status, "listType": listType}
return jsonify(j)
# /login
def login_check(self):
# validate username and password
username = request.form.get("username")
password = request.form.get("password")
person = User.get(username, self.auth_handler)
try:
if person and person.authenticate(password):
login_user(person)
return render_template("admin.html",
status="logged_in",
**self.adminInfo())
else:
return render_template("login.html", status="wrong_user_pass")
except Exception as e:
print(e)
return render_template("login.html", status="outdated_database")
# /logout
@login_required
def logout(self):
logout_user()
return redirect("/")
class Index(Minimal, Advanced_API):
#############
# Variables #
#############
def __init__(self):
Advanced_API.__init__(self)
Minimal.__init__(self)
self.minimal = False
self.auth_handler = AuthenticationHandler()
self.plugManager = PluginManager()
self.login_manager = LoginManager()
self.plugManager.loadPlugins()
self.login_manager.init_app(self.app)
self.login_manager.user_loader(self.load_user)
self.redisdb = Configuration.getRedisVendorConnection()
self.args.update({'minimal': False})
self.pluginArgs = {
"current_user": current_user,
"plugin_manager": self.plugManager
}
#############
# Functions #
#############
def generate_full_query(self, f):
query = self.generate_minimal_query(f)
query.extend(self.plugManager.doFilter(f, **self.pluginArgs))
return query
def filter_logic(self, filters, skip, limit=None):
query = self.generate_full_query(filters)
limit = limit if limit else self.args['pageLength']
cve = self.db.CVE.query(limit=limit, skip=skip, query=query)
cve = [CVEPresentation(c) for c in cve]
# marking relevant records
self.plugManager.mark(cve, **self.pluginArgs)
cve = list(cve)
return cve
def filterUpdateField(self, data):
if not data: return data
returnvalue = []
for line in data.split("\n"):
if (not line.startswith("[+]Success to create index")
and not line == "Not modified"
and not line.startswith("Starting")):
returnvalue.append(line)
return "\n".join(returnvalue)
def adminInfo(self, output=None):
return {
'stats': self.db.db_info(True),
'plugins': self.plugManager.getPlugins(),
'updateOutput': self.filterUpdateField(output),
'token': self.db.Users.getToken(current_user.id)
}
# user management
def load_user(self, id):
return User.get(id, self.auth_handler)
##########
# ROUTES #
##########
def index(self):
cve = self.filter_logic(self.defaultFilters, 0)
filters = self.plugManager.getFilters(**self.pluginArgs)
return render_template('index.html',
cve=cve,
r=0,
filters=filters,
**self.args)
# /
def index_post(self):
args = dict(self.getFilterSettingsFromPost(0), **self.args)
filters = self.plugManager.getFilters(**self.pluginArgs)
return render_template('index.html', r=0, filters=filters, **args)
# /r/<r>
def index_filter_get(self, r):
if not r or r < 0: r = 0
cve = self.filter_logic(self.defaultFilters, r)
filters = self.plugManager.getFilters(**self.pluginArgs)
return render_template('index.html',
cve=cve,
r=r,
filters=filters,
**self.args)
# /r/<r>
def index_filter_post(self, r):
if not r or r < 0: r = 0
args = dict(self.getFilterSettingsFromPost(r), **self.args)
filters = self.plugManager.getFilters(**self.pluginArgs)
return render_template('index.html', r=r, filters=filters, **args)
# /cve/<cveid>
def cve(self, cveid):
cve = self.api_cve(cveid)
if cve is None:
return render_template('error.html',
status={
'except': 'cve-not-found',
'info': {
'cve': cveid
}
})
self.plugManager.markCPE(cve)
self.plugManager.onCVEOpen(cveid, **self.pluginArgs)
pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs)
return render_template('cve.html', cve=cve, plugins=pluginData)
# /_get_plugins
def _get_plugins(self):
if not current_user.is_authenticated(
): # Don't show plugins requiring auth if not authenticated
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs) if not x.requiresAuth]
else:
plugins = [{
"name": x.getName(),
"link": x.getUID()
} for x in self.plugManager.getWebPluginsWithPage(
**self.pluginArgs)]
return jsonify({"plugins": plugins})
# /plugin/_get_cve_actions
def _get_cve_actions(self):
cve = request.args.get('cve', type=str)
if not current_user.is_authenticated(
): # Don't show actions requiring auth if not authenticated
actions = [
x for x in self.plugManager.getCVEActions(
cve, **self.pluginArgs) if not x['auth']
]
else:
actions = self.plugManager.getCVEActions(cve, **self.pluginArgs)
return jsonify({"actions": actions})
# /plugin/<plugin>
def openPlugin(self, plugin):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
data = self.plugManager.openPage(plugin, **self.pluginArgs)
if data:
page, data, mimetype = data
if page:
try:
return render_template(page, **data)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html",
status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
elif data:
return Response(data, mimetype=mimetype)
abort(404)
# /plugin/<plugin>/subpage/<page>
def openPluginSubpage(self, plugin, page):
if self.plugManager.requiresAuth(
plugin) and not current_user.is_authenticated():
return render_template("requiresAuth.html")
else:
data = self.plugManager.openSubpage(plugin, page,
**self.pluginArgs)
if data:
page, data, mimetype = data
if page:
try:
return render_template(page, **data)
except jinja2.exceptions.TemplateSyntaxError:
return render_template(
"error.html",
status={'except': 'plugin-page-corrupt'})
except jinja2.exceptions.TemplateNotFound:
return render_template("error.html",
status={
'except':
'plugin-page-not-found',
'page': page
})
elif data:
return Response(data, mimetype=mimetype)
abort(404)
# /plugin/<plugin>/_cve_action/<action>
def _jsonCVEAction(self, plugin, action):
cve = request.args.get('cve', type=str)
response = self.plugManager.onCVEAction(cve,
plugin,
action,
fields=dict(request.args),
**self.pluginArgs)
if type(response) is bool and response is True:
return jsonify({'status': 'plugin_action_complete'})
elif type(response) is bool and response is False or response is None:
return jsonify({'status': 'plugin_action_failed'})
elif type(response) is dict:
return jsonify(response)
else:
return jsonify({'status': 'no_plugin_output'})
# /admin
# /admin/
def admin(self):
if Configuration.loginRequired():
if not current_user.is_authenticated():
return render_template('login.html')
else:
person = User.get("_dummy_", self.auth_handler)
login_user(person)
output = None
if os.path.isfile(Configuration.getUpdateLogFile()):
with open(Configuration.getUpdateLogFile()) as updateFile:
separator = "==========================\n"
output = updateFile.read().split(separator)[-2:]
output = separator + separator.join(output)
return render_template('admin.html',
status="default",
**self.adminInfo(output))
# /admin/change_pass
@login_required
def change_pass(self):
current_pass = request.args.get('current_pass')
new_pass = request.args.get('new_pass')
if current_user.authenticate(current_pass):
if new_pass:
self.db.Users.changePassword(current_user.id, new_pass)
return jsonify({"status": "password_changed"})
return jsonify({"status": "no_password"})
else:
return jsonify({"status": "wrong_user_pass"})
# /admin/request_token
@login_required
def request_token(self):
return jsonify({"token": self.db.Users.generateToken(current_user.id)})
# /admin/updatedb
@login_required
def updatedb(self):
process = subprocess.Popen([
sys.executable,
os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = process.communicate()
output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str(
err, 'utf-8')) if err else str(out, 'utf-8')
return jsonify({"updateOutput": output, "status": "db_updated"})
# /login
def login_check(self):
# validate username and password
username = request.form.get('username')
password = request.form.get('password')
person = User.get(username, self.auth_handler)
try:
if person and person.authenticate(password):
login_user(person)
return render_template('admin.html',
status="logged_in",
**self.adminInfo())
else:
return render_template('login.html', status="wrong_user_pass")
except Exception as e:
print(e)
return render_template('login.html', status="outdated_database")
# /logout
@login_required
def logout(self):
logout_user()
return redirect("/")
