elif args.c is True and source["name"] == "redis-cache-cpe": logger.info("Starting " + source["name"]) up = source["updater"]() up.update() logger.info(source["name"] + " updated") for post in posts: logger.info("Starting " + post["name"]) indexer = post["updater"]() indexer.create_indexes() if args.i and int(newelement) > 0: subprocess.Popen((shlex.split("{} {} {}".format( sys.executable, os.path.join(runPath, "db_fulltext.py"), "-v -l " + newelement, )))).wait() if args.l is not False: logger.info("Sleeping...") time.sleep(3600) if not args.p: plugManager = PluginManager() plugManager.loadPlugins() plugins = plugManager.getPlugins() if len(plugins) != 0: for plug in plugins: logger.info("Starting " + plug.getName() + " plugin") message = plug.onDatabaseUpdate() if message: logger.info(message)
subprocess.Popen((shlex.split(source['updater']))).wait() after = nbelement(collection=source['name']) message = source['name'] + " has " + str(after) + " elements (" + str(after - before) + " update)" newelement = str(after - before) log(message) elif (args.c is True and source['name'] is "redis-cache-cpe"): log('Starting ' + source['name']) subprocess.Popen((shlex.split(source['updater']))).wait() log(source['name'] + " updated") for post in posts: log('Starting ' + post['name']) subprocess.Popen((shlex.split(post['updater']))).wait() if args.i and int(newelement) > 0: subprocess.Popen((shlex.split("{} {} {}".format(sys.executable, os.path.join(runPath, "db_fulltext.py"), "-v -l " + newelement)))).wait() if args.l is not False: log("Sleeping...") time.sleep(3600) log() if not args.p: plugManager = PluginManager() plugManager.loadPlugins() plugins = plugManager.getPlugins() if len(plugins) != 0: for plug in plugins: log("Starting " + plug.getName() + " plugin") message = plug.onDatabaseUpdate() if message: log(message)
class Index(Minimal, Advanced_API): ############# # Variables # ############# def __init__(self): # TODO: make auth handler and plugin manager singletons Advanced_API.__init__(self) Minimal.__init__(self) self.minimal = False self.auth_handler = AuthenticationHandler() self.plugManager = PluginManager() self.login_manager = LoginManager() self.plugManager.loadPlugins() self.login_manager.init_app(self.app) self.login_manager.user_loader(self.load_user) self.redisdb = Configuration.getRedisVendorConnection() self.defaultFilters.update({'blacklistSelect': 'on', 'whitelistSelect': 'on', 'unlistedSelect': 'show',}) self.args.update({'minimal': False}) self.pluginArgs = {"current_user": current_user, "plugin_manager": self.plugManager} routes = [{'r': '/cve/<cveid>', 'm': ['GET'], 'f': self.cve}, {'r': '/_get_plugins', 'm': ['GET'], 'f': self._get_plugins}, {'r': '/plugin/_get_cve_actions', 'm': ['GET'], 'f': self._get_cve_actions}, {'r': '/plugin/<plugin>', 'm': ['GET'], 'f': self.openPlugin}, {'r': '/plugin/<plugin>/subpage/<page>', 'm': ['GET'], 'f': self.openPluginSubpage}, {'r': '/plugin/<plugin>/_cve_action/<action>', 'm': ['GET'], 'f': self._jsonCVEAction}, {'r': '/login', 'm': ['POST'], 'f': self.login_check}, {'r': '/logout', 'm': ['POST'], 'f': self.logout}, {'r': '/admin', 'm': ['GET'], 'f': self.admin}, {'r': '/admin/', 'm': ['GET'], 'f': self.admin}, {'r': '/admin/change_pass', 'm': ['GET'], 'f': self.change_pass}, {'r': '/admin/request_token', 'm': ['GET'], 'f': self.request_token}, {'r': '/admin/updatedb', 'm': ['GET'], 'f': self.updatedb}, {'r': '/admin/whitelist/import', 'm': ['POST'], 'f': self.listImport}, {'r': '/admin/blacklist/import', 'm': ['POST'], 'f': self.listImport}, {'r': '/admin/whitelist/export', 'm': ['GET'], 'f': self.listExport}, {'r': '/admin/blacklist/export', 'm': ['GET'], 'f': self.listExport}, {'r': '/admin/whitelist/drop', 'm': ['POST'], 'f': self.listDrop}, {'r': '/admin/blacklist/drop', 'm': ['POST'], 'f': self.listDrop}, {'r': '/admin/whitelist', 'm': ['GET'], 'f': self.listView}, {'r': '/admin/blacklist', 'm': ['GET'], 'f': self.listView}, {'r': '/admin/addToList', 'm': ['GET'], 'f': self.listAdd}, {'r': '/admin/removeFromList', 'm': ['GET'], 'f': self.listRemove}, {'r': '/admin/editInList', 'm': ['GET'], 'f': self.listEdit}, {'r': '/admin/listmanagement', 'm': ['GET'], 'f': self.listManagement}, {'r': '/admin/listmanagement/<vendor>', 'm': ['GET'], 'f': self.listManagement}, {'r': '/admin/listmanagement/<vendor>/<product>', 'm': ['GET'], 'f': self.listManagement}, {'r': '/admin/listmanagement/add', 'm': ['GET'], 'f': self.listManagementAdd}, {'r': '/login', 'm': ['POST'], 'f': self.login_check}] for route in routes: self.addRoute(route) ############# # Functions # ############# def generate_full_query(self, f): query = self.generate_minimal_query(f) if current_user.is_authenticated(): if f['blacklistSelect'] == "on": regexes = db.getRules('blacklist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({'$or': [{'vulnerable_configuration': re.compile(exp)}, {'vulnerable_configuration': {'$exists': False}}, {'vulnerable_configuration': []} ]}) if f['whitelistSelect'] == "hide": regexes = db.getRules('whitelist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({'$or': [{'vulnerable_configuration': re.compile(exp)}, {'vulnerable_configuration': {'$exists': False}}, {'vulnerable_configuration': []} ]}) if f['unlistedSelect'] == "hide": wlregexes = tk.compile(db.getRules('whitelist')) blregexes = tk.compile(db.getRules('blacklist')) query.append({'$or': [{'vulnerable_configuration': {'$in': wlregexes}}, {'vulnerable_configuration': {'$in': blregexes}}]}) return query def markCPEs(self, cve): blacklist = tk.compile(db.getRules('blacklist')) whitelist = tk.compile(db.getRules('whitelist')) for conf in cve['vulnerable_configuration']: conf['list'] = 'none' conf['match'] = 'none' for w in whitelist: if w.match(conf['id']): conf['list'] = 'white' conf['match'] = w for b in blacklist: if b.match(conf['id']): conf['list'] = 'black' conf['match'] = b return cve def filter_logic(self, filters, skip, limit=None): query = self.generate_full_query(filters) limit = limit if limit else self.args['pageLength'] cve = db.getCVEs(limit=limit, skip=skip, query=query) # marking relevant records if current_user.is_authenticated(): if filters['whitelistSelect'] == "on": cve = self.list_mark('white', cve) if filters['blacklistSelect'] == "mark": cve = self.list_mark('black', cve) self.plugManager.mark(cve, **self.pluginArgs) cve = list(cve) return cve def addCPEToList(self, cpe, listType, cpeType=None): def addCPE(cpe, cpeType, funct): return True if funct(cpe, cpeType) else False if not cpeType: cpeType='cpe' if listType.lower() in ("blacklist", "black", "b", "bl"): return addCPE(cpe, cpeType, bl.insertBlacklist) if listType.lower() in ("whitelist", "white", "w", "wl"): return addCPE(cpe, cpeType, wl.insertWhitelist) def list_mark(self, listed, cveList): if listed not in ['white', 'black']: return list(cves) items = tk.compile(db.getRules(listed+'list')) # check the cpes (full or partially) in the black/whitelist for i, cve in enumerate(list(cveList)): # the list() is to ensure we don't have a pymongo cursor object for c in cve['vulnerable_configuration']: if any(regex.match(c) for regex in items): cveList[i][listed+'listed'] = 'yes' return cveList def filterUpdateField(self, data): if not data: return data returnvalue = [] for line in data.split("\n"): if (not line.startswith("[+]Success to create index") and not line == "Not modified" and not line.startswith("Starting")): returnvalue.append(line) return "\n".join(returnvalue) def adminInfo(self, output=None): return {'stats': db.getDBStats(True), 'plugins': self.plugManager.getPlugins(), 'updateOutput': self.filterUpdateField(output), 'token': db.getToken(current_user.id)} # user management def load_user(self, id): return User.get(id, self.auth_handler) ########## # ROUTES # ########## # /cve/<cveid> def cve(self, cveid): cveid = cveid.upper() cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True,subscorelookup=True) cve = cvesp.getcve(cveid=cveid) if cve is None: return render_template('error.html',status={'except':'cve-not-found','info':{'cve':cveid}}) cve = self.markCPEs(cve) self.plugManager.onCVEOpen(cveid, **self.pluginArgs) pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs) return render_template('cve.html', cve=cve, plugins=pluginData) # /_get_plugins def _get_plugins(self): if not current_user.is_authenticated(): # Don't show plugins requiring auth if not authenticated plugins = [{"name": x.getName(), "link": x.getUID()} for x in self.plugManager.getWebPluginsWithPage(**self.pluginArgs) if not x.requiresAuth] else: plugins = [{"name": x.getName(), "link": x.getUID()} for x in self.plugManager.getWebPluginsWithPage(**self.pluginArgs)] return jsonify({"plugins": plugins}) # /plugin/_get_cve_actions def _get_cve_actions(self): cve = request.args.get('cve', type=str) if not current_user.is_authenticated(): # Don't show actions requiring auth if not authenticated actions = [x for x in self.plugManager.getCVEActions(cve, **self.pluginArgs) if not x['auth']] else: actions = self.plugManager.getCVEActions(cve, **self.pluginArgs) return jsonify({"actions": actions}) # /plugin/<plugin> def openPlugin(self, plugin): if self.plugManager.requiresAuth(plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openPage(plugin, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template("error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={'except': 'plugin-page-not-found', 'page': page}) else: abort(404) # /plugin/<plugin>/subpage/<page> def openPluginSubpage(self, plugin, page): if self.plugManager.requiresAuth(plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openSubpage(plugin, page, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template("error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={'except': 'plugin-page-not-found', 'page': page}) else: abort(404) # /plugin/<plugin>/_cve_action/<action> def _jsonCVEAction(self, plugin, action): cve = request.args.get('cve', type=str) response = self.plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **self.pluginArgs) if type(response) is bool and response is True: return jsonify({'status': 'plugin_action_complete'}) elif type(response) is bool and response is False or response is None: return jsonify({'status': 'plugin_action_failed'}) elif type(response) is dict: return jsonify(response) # /admin # /admin/ def admin(self): if Configuration.loginRequired(): if not current_user.is_authenticated(): return render_template('login.html') else: person = User.get("_dummy_", self.auth_handler) login_user(person) output = None if os.path.isfile(Configuration.getUpdateLogFile()): with open(Configuration.getUpdateLogFile()) as updateFile: separator="==========================\n" output=updateFile.read().split(separator)[-2:] output=separator+separator.join(output) return render_template('admin.html', status="default", **self.adminInfo(output)) # /admin/change_pass @login_required def change_pass(self): current_pass = request.args.get('current_pass') new_pass = request.args.get('new_pass') if current_user.authenticate(current_pass): if new_pass: db.changePassword(current_user.id , new_pass) return jsonify({"status": "password_changed"}) return jsonify({"status": "no_password"}) else: return jsonify({"status": "wrong_user_pass"}) # /admin/request_token @login_required def request_token(self): return jsonify({"token": db.generateToken(current_user.id)}) # /admin/updatedb @login_required def updatedb(self): process = subprocess.Popen([sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() output="%s\n\nErrors:\n%s"%(str(out,'utf-8'),str(err,'utf-8')) if err else str(out,'utf-8') return jsonify({"updateOutput": output, "status": "db_updated"}) # /admin/whitelist # /admin/blacklist @login_required def listView(self): if request.url_rule.rule.split('/')[2].lower() == 'whitelist': return render_template('list.html', rules=db.getWhitelist(), listType="Whitelist") else: return render_template('list.html', rules=db.getBlacklist(), listType="Blacklist") # /admin/whitelist/import # /admin/blacklist/import @login_required def listImport(self, force=None, path=None): _list = request.url_rule.split('/')[2] file = request.files['file'] force = request.form.get('force') count = wl.countWhitelist() if _list.lower == 'whitelist' else bl.countBlacklist() if (count == 0) | (not count) | (force == "f"): if _list.lower == 'whitelist': wl.dropWhitelist() wl.importWhitelist(TextIOWrapper(file.stream)) else: bl.dropBlacklist() bl.importBlacklist(TextIOWrapper(file.stream)) status = _list[0]+"l_imported" else: status = _list[0]+"l_already_filled" return render_template('admin.html', status=status, **self.adminInfo()) # /admin/whitelist/export # /admin/blacklist/export @login_required def listExport(self, force=None, path=None): _list = request.url_rule.rule.split('/')[2] bytIO = BytesIO() data = wl.exportWhitelist() if _list.lower == 'whitelist' else bl.exportBlacklist() bytIO.write(bytes(data, "utf-8")) bytIO.seek(0) return send_file(bytIO, as_attachment=True, attachment_filename=_list+".txt") # /admin/whitelist/drop # /admin/blacklist/drop @login_required def listDrop(self): _list = request.url_rule.split('/')[2].lower() if _list == 'whitelist': wl.dropWhitelist() else: bl.dropBlacklist() return jsonify({"status": _list[0]+"l_dropped"}) # /admin/addToList @login_required def listAdd(self): cpe = request.args.get('cpe') cpeType = request.args.get('type') lst = request.args.get('list') if cpe and cpeType and lst: status = "added_to_list" if self.addCPEToList(cpe, lst, cpeType) else "already_exists_in_list" returnList = db.getWhitelist() if lst=="whitelist" else db.getBlacklist() return jsonify({"status":status, "rules":returnList, "listType":lst.title()}) else: return jsonify({"status": "could_not_add_to_list"}) # /admin/removeFromList @login_required def listRemove(self): cpe = request.args.get('cpe', type=str) cpe = urllib.parse.quote_plus(cpe).lower() cpe = cpe.replace("%3a", ":") cpe = cpe.replace("%2f", "/") lst = request.args.get('list', type=str) if cpe and lst: result=wl.removeWhitelist(cpe) if lst.lower()=="whitelist" else bl.removeBlacklist(cpe) status = "removed_from_list" if (result > 0) else "already_removed_from_list" else: status = "invalid_cpe" returnList = db.getWhitelist() if lst=="whitelist" else db.getBlacklist() return jsonify({"status":status, "rules":returnList, "listType":lst.title()}) # /admin/editInList @login_required def listEdit(self): old = request.args.get('oldCPE') new = request.args.get('cpe') lst = request.args.get('list') CPEType = request.args.get('type') if old and new: result = wl.updateWhitelist(old, new, CPEType) if lst=="whitelist" else bl.updateBlacklist(old, new, CPEType) status = "cpelist_updated" if (result) else "cpelist_update_failed" else: status = "invalid_cpe" returnList = list(db.getWhitelist()) if lst=="whitelist" else list(db.getBlacklist()) return jsonify({"rules":returnList, "status":status, "listType":lst}) # /admin/listmanagement/<vendor>/<product> # /admin/listmanagement/<vendor> # /admin/listmanagement @login_required def listManagement(self, vendor=None, product=None): try: if product is None: # no product selected yet, so same function as /browse can be used if vendor: vendor = urllib.parse.quote_plus(vendor).lower() browseList = query.getBrowseList(vendor) vendor = browseList["vendor"] product = browseList["product"] version = None else: # product selected, product versions required version = query.getVersionsOfProduct(urllib.parse.quote_plus(product).lower()) return render_template('listmanagement.html', vendor=vendor, product=product, version=version) except redisExceptions.ConnectionError: return render_template('error.html', status={'except':'redis-connection', 'info':{'host':Configuration.getRedisHost(),'port':Configuration.getRedisPort()}}) # /admin/listmanagement/add @login_required def listManagementAdd(self): # retrieve the separate item parts item = request.args.get('item', type=str) listType = request.args.get('list', type=str) pattern = re.compile('^[a-z:/0-9.~_%-]+$') if pattern.match(item): item = item.split(":") added = False if len(item) == 1: # only vendor, so a check on cpe type is needed if self.redisdb.sismember("t:/o", item[0]): if self.addCPEToList("cpe:/o:" + item[0], listType): added = True if self.redisdb.sismember("t:/a", item[0]): if self.addCPEToList("cpe:/a:" + item[0], listType): added = True if self.redisdb.sismember("t:/h", item[0]): if self.addCPEToList("cpe:/h:" + item[0], listType): added = True elif 4 > len(item) > 1: # cpe type can be found with a mongo regex query result = db.getCVEs(query={'cpe_2_2': {'$regex': item[1]}}) if result.count() != 0: prefix = ((result[0])['cpe_2_2'])[:7] if len(item) == 2: if self.addCPEToList(prefix + item[0] + ":" + item[1], listType): added = True if len(item) == 3: if self.addCPEToList(prefix + item[0] + ":" + item[1] + ":" + item[2], listType): added = True status = "added_to_list" if added else "could_not_add_to_list" else: status = "invalid_cpe" j={"status":status, "listType":listType} return jsonify(j) # /login def login_check(self): # validate username and password username = request.form.get('username') password = request.form.get('password') person = User.get(username, self.auth_handler) try: if person and person.authenticate(password): login_user(person) return render_template('admin.html', status="logged_in", **self.adminInfo()) else: return render_template('login.html', status="wrong_user_pass") except Exception as e: print(e) return render_template('login.html', status="outdated_database") # /logout @login_required def logout(self): logout_user() return redirect("/")
class Index(Minimal, Advanced_API): ############# # Variables # ############# def __init__(self): # TODO: make auth handler and plugin manager singletons Advanced_API.__init__(self) Minimal.__init__(self) self.minimal = False self.auth_handler = AuthenticationHandler() self.plugManager = PluginManager() self.login_manager = LoginManager() self.plugManager.loadPlugins() self.login_manager.init_app(self.app) self.login_manager.user_loader(self.load_user) self.redisdb = Configuration.getRedisVendorConnection() self.defaultFilters.update({ "blacklistSelect": "on", "whitelistSelect": "on", "unlistedSelect": "show", }) self.args.update({"minimal": False}) self.pluginArgs = { "current_user": current_user, "plugin_manager": self.plugManager, } routes = [ { "r": "/cve/<cveid>", "m": ["GET"], "f": self.cve }, { "r": "/_get_plugins", "m": ["GET"], "f": self._get_plugins }, { "r": "/plugin/_get_cve_actions", "m": ["GET"], "f": self._get_cve_actions }, { "r": "/plugin/<plugin>", "m": ["GET"], "f": self.openPlugin }, { "r": "/plugin/<plugin>/subpage/<page>", "m": ["GET"], "f": self.openPluginSubpage, }, { "r": "/plugin/<plugin>/_cve_action/<action>", "m": ["GET"], "f": self._jsonCVEAction, }, { "r": "/login", "m": ["POST"], "f": self.login_check }, { "r": "/logout", "m": ["GET"], "f": self.logout }, { "r": "/admin", "m": ["GET"], "f": self.admin }, { "r": "/admin/", "m": ["GET"], "f": self.admin }, { "r": "/admin/change_pass", "m": ["GET"], "f": self.change_pass }, { "r": "/admin/request_token", "m": ["GET"], "f": self.request_token }, { "r": "/admin/updatedb", "m": ["GET"], "f": self.updatedb }, { "r": "/admin/whitelist/import", "m": ["POST"], "f": self.listImport }, { "r": "/admin/blacklist/import", "m": ["POST"], "f": self.listImport }, { "r": "/admin/whitelist/export", "m": ["GET"], "f": self.listExport }, { "r": "/admin/blacklist/export", "m": ["GET"], "f": self.listExport }, { "r": "/admin/whitelist/drop", "m": ["POST"], "f": self.listDrop }, { "r": "/admin/blacklist/drop", "m": ["POST"], "f": self.listDrop }, { "r": "/admin/whitelist", "m": ["GET"], "f": self.listView }, { "r": "/admin/blacklist", "m": ["GET"], "f": self.listView }, { "r": "/admin/addToList", "m": ["GET"], "f": self.listAdd }, { "r": "/admin/removeFromList", "m": ["GET"], "f": self.listRemove }, { "r": "/admin/editInList", "m": ["GET"], "f": self.listEdit }, { "r": "/admin/listmanagement", "m": ["GET"], "f": self.listManagement }, { "r": "/admin/listmanagement/<vendor>", "m": ["GET"], "f": self.listManagement, }, { "r": "/admin/listmanagement/<vendor>/<product>", "m": ["GET"], "f": self.listManagement, }, { "r": "/admin/listmanagement/add", "m": ["GET"], "f": self.listManagementAdd, }, { "r": "/login", "m": ["POST"], "f": self.login_check }, ] for route in routes: self.addRoute(route) ############# # Functions # ############# def generate_full_query(self, f): query = self.generate_minimal_query(f) if current_user.is_authenticated: if f["blacklistSelect"] == "on": regexes = getRules("blacklist") if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ "$or": [ { "vulnerable_configuration": re.compile(exp) }, { "vulnerable_configuration": { "$exists": False } }, { "vulnerable_configuration": [] }, ] }) if f["whitelistSelect"] == "hide": regexes = getRules("whitelist") if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ "$or": [ { "vulnerable_configuration": re.compile(exp) }, { "vulnerable_configuration": { "$exists": False } }, { "vulnerable_configuration": [] }, ] }) if f["unlistedSelect"] == "hide": wlregexes = tk_compile(getRules("whitelist")) blregexes = tk_compile(getRules("blacklist")) query.append({ "$or": [ { "vulnerable_configuration": { "$in": wlregexes } }, { "vulnerable_configuration": { "$in": blregexes } }, ] }) return query def markCPEs(self, cve): blacklist = tk_compile(getRules("blacklist")) whitelist = tk_compile(getRules("whitelist")) for conf in cve["vulnerable_configuration"]: conf["list"] = "none" conf["match"] = "none" for w in whitelist: if w.match(conf["id"]): conf["list"] = "white" conf["match"] = w for b in blacklist: if b.match(conf["id"]): conf["list"] = "black" conf["match"] = b return cve def filter_logic(self, filters, skip, limit=None): query = self.generate_full_query(filters) limit = limit if limit else self.args["pageLength"] cve = getCVEs(limit=limit, skip=skip, query=query) # marking relevant records if current_user.is_authenticated: if filters["whitelistSelect"] == "on": cve["results"] = self.list_mark("white", cve["results"]) if filters["blacklistSelect"] == "mark": cve["results"] = self.list_mark("black", cve["results"]) self.plugManager.mark(cve, **self.pluginArgs) return cve def addCPEToList(self, cpe, listType, cpeType=None): def addCPE(cpe, cpeType, funct): return True if funct(cpe, cpeType) else False if not cpeType: cpeType = "cpe" if listType.lower() in ("blacklist", "black", "b", "bl"): return addCPE(cpe, cpeType, insertBlacklist) if listType.lower() in ("whitelist", "white", "w", "wl"): return addCPE(cpe, cpeType, insertWhitelist) def list_mark(self, listed, cveList): if listed not in ["white", "black"]: return list(cveList) items = tk_compile(getRules(listed + "list")) # check the cpes (full or partially) in the black/whitelist for i, cve in enumerate( list(cveList) ): # the list() is to ensure we don't have a pymongo cursor object for c in cve["vulnerable_configuration"]: if any(regex.match(c) for regex in items): cveList[i][listed + "listed"] = "yes" return cveList def filterUpdateField(self, data): if not data: return data returnvalue = [] for line in data.split("\n"): if (not line.startswith("[+]Success to create index") and not line == "Not modified" and not line.startswith("Starting")): returnvalue.append(line) return "\n".join(returnvalue) def adminInfo(self, output=None): return { "stats": getDBStats(True), "plugins": self.plugManager.getPlugins(), "updateOutput": self.filterUpdateField(output), "token": getToken(current_user.id), } # user management def load_user(self, id): return User.get(id, self.auth_handler) ########## # ROUTES # ########## # /cve/<cveid> def cve(self, cveid): cveid = cveid.upper() cvesp = CveHandler( rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True, subscorelookup=True, ) cve = cvesp.getcve(cveid=cveid) if cve is None: return render_template("error.html", status={ "except": "cve-not-found", "info": { "cve": cveid } }) cve = self.markCPEs(cve) self.plugManager.onCVEOpen(cveid, **self.pluginArgs) pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs) return render_template("cve.html", cve=cve, plugins=pluginData) # /_get_plugins def _get_plugins(self): if (not current_user.is_authenticated ): # Don't show plugins requiring auth if not authenticated plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs) if not x.requiresAuth] else: plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs)] return jsonify({"plugins": plugins}) # /plugin/_get_cve_actions def _get_cve_actions(self): cve = request.args.get("cve", type=str) if (not current_user.is_authenticated ): # Don't show actions requiring auth if not authenticated actions = [ x for x in self.plugManager.getCVEActions( cve, **self.pluginArgs) if not x["auth"] ] else: actions = self.plugManager.getCVEActions(cve, **self.pluginArgs) return jsonify({"actions": actions}) # /plugin/<plugin> def openPlugin(self, plugin): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated: return render_template("requiresAuth.html") else: page, args = self.plugManager.openPage(plugin, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={"except": "plugin-page-corrupt"}) except jinja2.exceptions.TemplateNotFound: return render_template( "error.html", status={ "except": "plugin-page-not-found", "page": page }, ) else: abort(404) # /plugin/<plugin>/subpage/<page> def openPluginSubpage(self, plugin, page): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated: return render_template("requiresAuth.html") else: page, args = self.plugManager.openSubpage(plugin, page, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={"except": "plugin-page-corrupt"}) except jinja2.exceptions.TemplateNotFound: return render_template( "error.html", status={ "except": "plugin-page-not-found", "page": page }, ) else: abort(404) # /plugin/<plugin>/_cve_action/<action> def _jsonCVEAction(self, plugin, action): cve = request.args.get("cve", type=str) response = self.plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **self.pluginArgs) if type(response) is bool and response is True: return jsonify({"status": "plugin_action_complete"}) elif type(response) is bool and response is False or response is None: return jsonify({"status": "plugin_action_failed"}) elif type(response) is dict: return jsonify(response) # /admin # /admin/ def admin(self): if Configuration.loginRequired(): if not current_user.is_authenticated: return render_template("login.html") else: person = User.get("_dummy_", self.auth_handler) login_user(person) output = None if os.path.isfile(Configuration.getUpdateLogFile()): with open(Configuration.getUpdateLogFile()) as updateFile: separator = "==========================\n" output = updateFile.read().split(separator)[-2:] output = separator + separator.join(output) return render_template("admin.html", status="default", **self.adminInfo(output)) # /admin/change_pass @login_required def change_pass(self): current_pass = request.args.get("current_pass") new_pass = request.args.get("new_pass") if current_user.authenticate(current_pass): if new_pass: changePassword(current_user.id, new_pass) return jsonify({"status": "password_changed"}) return jsonify({"status": "no_password"}) else: return jsonify({"status": "wrong_user_pass"}) # /admin/request_token @login_required def request_token(self): return jsonify({"token": generateToken(current_user.id)}) # /admin/updatedb @login_required def updatedb(self): process = subprocess.Popen( [ sys.executable, os.path.join(_runPath, "../../../sbin/db_updater.py"), "-civ", ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out, err = process.communicate() output = ("%s\n\nErrors:\n%s" % (str(out, "utf-8"), str(err, "utf-8")) if err else str(out, "utf-8")) return jsonify({"updateOutput": output, "status": "db_updated"}) # /admin/whitelist # /admin/blacklist @login_required def listView(self): if request.url_rule.rule.split("/")[2].lower() == "whitelist": return render_template("list.html", rules=getWhitelist(), listType="Whitelist") else: return render_template("list.html", rules=getBlacklist(), listType="Blacklist") # /admin/whitelist/import # /admin/blacklist/import @login_required def listImport(self, force=None, path=None): _list = request.url_rule.split("/")[2] file = request.files["file"] force = request.form.get("force") count = countWhitelist( ) if _list.lower == "whitelist" else countBlacklist() if (count == 0) | (not count) | (force == "f"): if _list.lower == "whitelist": dropWhitelist() importWhitelist(TextIOWrapper(file.stream)) else: dropBlacklist() importBlacklist(TextIOWrapper(file.stream)) status = _list[0] + "l_imported" else: status = _list[0] + "l_already_filled" return render_template("admin.html", status=status, **self.adminInfo()) # /admin/whitelist/export # /admin/blacklist/export @login_required def listExport(self, force=None, path=None): _list = request.url_rule.rule.split("/")[2] bytIO = BytesIO() data = exportWhitelist( ) if _list.lower == "whitelist" else exportBlacklist() bytIO.write(bytes(data, "utf-8")) bytIO.seek(0) return send_file(bytIO, as_attachment=True, attachment_filename=_list + ".txt") # /admin/whitelist/drop # /admin/blacklist/drop @login_required def listDrop(self): _list = request.url_rule.split("/")[2].lower() if _list == "whitelist": dropWhitelist() else: dropBlacklist() return jsonify({"status": _list[0] + "l_dropped"}) # /admin/addToList @login_required def listAdd(self): cpe = request.args.get("cpe") cpeType = request.args.get("type") lst = request.args.get("list") if cpe and cpeType and lst: status = ("added_to_list" if self.addCPEToList(cpe, lst, cpeType) else "already_exists_in_list") returnList = getWhitelist( ) if lst == "whitelist" else getBlacklist() return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) else: return jsonify({"status": "could_not_add_to_list"}) # /admin/removeFromList @login_required def listRemove(self): cpe = request.args.get("cpe", type=str) cpe = urllib.parse.quote_plus(cpe).lower() cpe = cpe.replace("%3a", ":") cpe = cpe.replace("%2f", "/") lst = request.args.get("list", type=str) if cpe and lst: result = (removeWhitelist(cpe) if lst.lower() == "whitelist" else removeBlacklist(cpe)) status = ("removed_from_list" if (result > 0) else "already_removed_from_list") else: status = "invalid_cpe" returnList = getWhitelist() if lst == "whitelist" else getBlacklist() return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) # /admin/editInList @login_required def listEdit(self): old = request.args.get("oldCPE") new = request.args.get("cpe") lst = request.args.get("list") CPEType = request.args.get("type") if old and new: result = (updateWhitelist(old, new, CPEType) if lst == "whitelist" else updateBlacklist(old, new, CPEType)) status = "cpelist_updated" if (result) else "cpelist_update_failed" else: status = "invalid_cpe" returnList = (list(getWhitelist()) if lst == "whitelist" else list(getBlacklist())) return jsonify({ "rules": returnList, "status": status, "listType": lst }) # /admin/listmanagement/<vendor>/<product> # /admin/listmanagement/<vendor> # /admin/listmanagement @login_required def listManagement(self, vendor=None, product=None): try: if product is None: # no product selected yet, so same function as /browse can be used if vendor: vendor = urllib.parse.quote_plus(vendor).lower() browseList = getBrowseList(vendor) vendor = browseList["vendor"] product = browseList["product"] version = None else: # product selected, product versions required version = getVersionsOfProduct( urllib.parse.quote_plus(product).lower()) return render_template("listmanagement.html", vendor=vendor, product=product, version=version) except redisExceptions.ConnectionError: return render_template( "error.html", status={ "except": "redis-connection", "info": { "host": Configuration.getRedisHost(), "port": Configuration.getRedisPort(), }, }, ) # /admin/listmanagement/add @login_required def listManagementAdd(self): # retrieve the separate item parts item = request.args.get("item", type=str) listType = request.args.get("list", type=str) pattern = re.compile("^[a-z:/0-9.~_%-]+$") if pattern.match(item): item = item.split(":") added = False if len(item) == 1: # only vendor, so a check on cpe type is needed if self.redisdb.sismember("t:/o", item[0]): if self.addCPEToList("cpe:/o:" + item[0], listType): added = True if self.redisdb.sismember("t:/a", item[0]): if self.addCPEToList("cpe:/a:" + item[0], listType): added = True if self.redisdb.sismember("t:/h", item[0]): if self.addCPEToList("cpe:/h:" + item[0], listType): added = True elif 4 > len(item) > 1: # cpe type can be found with a mongo regex query result = getCVEs(query={"cpe_2_2": { "$regex": item[1] }})["results"] if result.count() != 0: prefix = ((result[0])["cpe_2_2"])[:7] if len(item) == 2: if self.addCPEToList(prefix + item[0] + ":" + item[1], listType): added = True if len(item) == 3: if self.addCPEToList( prefix + item[0] + ":" + item[1] + ":" + item[2], listType): added = True status = "added_to_list" if added else "could_not_add_to_list" else: status = "invalid_cpe" j = {"status": status, "listType": listType} return jsonify(j) # /login def login_check(self): # validate username and password username = request.form.get("username") password = request.form.get("password") person = User.get(username, self.auth_handler) try: if person and person.authenticate(password): login_user(person) return render_template("admin.html", status="logged_in", **self.adminInfo()) else: return render_template("login.html", status="wrong_user_pass") except Exception as e: print(e) return render_template("login.html", status="outdated_database") # /logout @login_required def logout(self): logout_user() return redirect("/")
class Index(Minimal, Advanced_API): ############# # Variables # ############# def __init__(self): # TODO: make auth handler and plugin manager singletons Advanced_API.__init__(self) Minimal.__init__(self) self.minimal = False self.auth_handler = AuthenticationHandler() self.plugManager = PluginManager() self.login_manager = LoginManager() self.plugManager.loadPlugins() self.login_manager.init_app(self.app) self.login_manager.user_loader(self.load_user) self.redisdb = Configuration.getRedisVendorConnection() self.defaultFilters.update({ 'blacklistSelect': 'off', 'whitelistSelect': 'on', 'unlistedSelect': 'off', }) self.args.update({'minimal': False}) self.pluginArgs = { "current_user": current_user, "plugin_manager": self.plugManager } routes = [{ 'r': '/cve/<cveid>', 'm': ['GET'], 'f': self.cve }, { 'r': '/_get_plugins', 'm': ['GET'], 'f': self._get_plugins }, { 'r': '/plugin/_get_cve_actions', 'm': ['GET'], 'f': self._get_cve_actions }, { 'r': '/plugin/<plugin>', 'm': ['GET'], 'f': self.openPlugin }, { 'r': '/plugin/<plugin>/subpage/<page>', 'm': ['GET'], 'f': self.openPluginSubpage }, { 'r': '/plugin/<plugin>/_cve_action/<action>', 'm': ['GET'], 'f': self._jsonCVEAction }, { 'r': '/login', 'm': ['POST'], 'f': self.login_check }, { 'r': '/logout', 'm': ['GET'], 'f': self.logout }, { 'r': '/register', 'm': ['GET', 'POST'], 'f': self.register }, { 'r': '/admin', 'm': ['GET'], 'f': self.admin }, { 'r': '/admin/', 'm': ['GET'], 'f': self.admin }, { 'r': '/admin/change_pass', 'm': ['GET'], 'f': self.change_pass }, { 'r': '/admin/request_token', 'm': ['GET'], 'f': self.request_token }, { 'r': '/admin/updatedb', 'm': ['GET'], 'f': self.updatedb }, { 'r': '/admin/enableupdatedb', 'm': ['GET'], 'f': self.enableupdatedb }, { 'r': '/admin/enableupdatedb/status', 'm': ['GET'], 'f': self.enableupdatedbstatus }, { 'r': '/admin/whitelist/import', 'm': ['POST'], 'f': self.listImport }, { 'r': '/admin/blacklist/import', 'm': ['POST'], 'f': self.listImport }, { 'r': '/admin/whitelist/export', 'm': ['GET'], 'f': self.listExport }, { 'r': '/admin/blacklist/export', 'm': ['GET'], 'f': self.listExport }, { 'r': '/admin/whitelist/drop', 'm': ['GET'], 'f': self.listDrop }, { 'r': '/admin/blacklist/drop', 'm': ['GET'], 'f': self.listDrop }, { 'r': '/admin/whitelist', 'm': ['GET'], 'f': self.listView }, { 'r': '/admin/blacklist', 'm': ['GET'], 'f': self.listView }, { 'r': '/admin/addToList', 'm': ['GET'], 'f': self.listAdd }, { 'r': '/admin/removeFromList', 'm': ['GET'], 'f': self.listRemove }, { 'r': '/admin/editInList', 'm': ['GET'], 'f': self.listEdit }, { 'r': '/admin/listmanagement', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/<vendor>', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/<vendor>/<product>', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/add', 'm': ['GET'], 'f': self.listManagementAdd }, { 'r': '/login', 'm': ['POST'], 'f': self.login_check }] for route in routes: self.addRoute(route) ############# # Functions # ############# def generate_full_query(self, f): query = self.generate_minimal_query(f) if current_user.is_authenticated(): if f['blacklistSelect'] == "on": regexes = db.getRules('blacklist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ '$or': [{ 'vulnerable_configuration': re.compile(exp) }, { 'vulnerable_configuration': { '$exists': False } }, { 'vulnerable_configuration': [] }] }) if f['whitelistSelect'] == "hide": regexes = db.getRules('whitelist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ '$or': [{ 'vulnerable_configuration': re.compile(exp) }, { 'vulnerable_configuration': { '$exists': False } }, { 'vulnerable_configuration': [] }] }) if f['unlistedSelect'] == "hide": wlregexes = tk.compile(db.getRules('whitelist')) blregexes = tk.compile(db.getRules('blacklist')) query.append({ '$or': [{ 'vulnerable_configuration': { '$in': wlregexes } }, { 'vulnerable_configuration': { '$in': blregexes } }] }) return query def markCPEs(self, cve): blacklist = tk.compile(db.getRules('blacklist')) whitelist = tk.compile(db.getRules('whitelist')) for conf in cve['vulnerable_configuration']: conf['list'] = 'none' conf['match'] = 'none' for w in whitelist: if w.match(conf['id']): conf['list'] = 'white' conf['match'] = w for b in blacklist: if b.match(conf['id']): conf['list'] = 'black' conf['match'] = b return cve def filter_logic(self, filters, skip, limit=None): query = self.generate_full_query(filters) limit = limit if limit else self.args['pageLength'] cve = db.getCVEs(limit=limit, skip=skip, query=query) # marking relevant records if current_user.is_authenticated(): user = current_user.get_id() if filters['whitelistSelect'] == "on": cve = self.list_mark('white', cve, user=user) if filters['blacklistSelect'] == "mark": cve = self.list_mark('black', cve, user=user) self.plugManager.mark(cve, **self.pluginArgs) cve = list(cve) return cve def addCPEToList(self, cpe, listType, cpeType=None, isglobal=True, user=None): def addCPE(cpe, cpeType, funct, isglobal, user): return True if funct(cpe, cpeType, isglobal, user) else False if not cpeType: cpeType = 'cpe' if listType.lower() in ("blacklist", "black", "b", "bl"): return addCPE(cpe, cpeType, bl.insertBlacklist, isglobal, user) if listType.lower() in ("whitelist", "white", "w", "wl"): return addCPE(cpe, cpeType, wl.insertWhitelist, isglobal, user) #TODO: we need only to filer list that we own or it is marker as a global def list_mark(self, listed, cveList, user=None): if listed not in ['white', 'black']: return list(cves) items = tk.compile(db.getRules(listed + 'list')) # check the cpes (full or partially) in the black/whitelist for i, cve in enumerate( list(cveList) ): # the list() is to ensure we don't have a pymongo cursor object for c in cve['vulnerable_configuration']: if any(regex.match(c) for regex in items): cveList[i][listed + 'listed'] = 'yes' return cveList def filterUpdateField(self, data): if not data: return data returnvalue = [] for line in data.split("\n"): if (not line.startswith("[+]Success to create index") and not line == "Not modified" and not line.startswith("Starting")): returnvalue.append(line) return "\n".join(returnvalue) def adminInfo(self, output=None): return { 'stats': db.getDBStats(True), 'plugins': self.plugManager.getPlugins(), 'updateOutput': self.filterUpdateField(output), 'token': str(db.getToken(current_user.id)) } # user management def load_user(self, id): return User.get(id, self.auth_handler) ########## # ROUTES # ########## # /cve/<cveid> def cve(self, cveid): cveid = cveid.upper() cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True, subscorelookup=True) cve = cvesp.getcve(cveid=cveid) if cve is None: return render_template('error.html', status={ 'except': 'cve-not-found', 'info': { 'cve': cveid } }) cve = self.markCPEs(cve) self.plugManager.onCVEOpen(cveid, **self.pluginArgs) pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs) return render_template('cve.html', cve=cve, plugins=pluginData) # /_get_plugins def _get_plugins(self): if not current_user.is_authenticated( ): # Don't show plugins requiring auth if not authenticated plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs) if not x.requiresAuth] else: plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs)] return jsonify({"plugins": plugins}) # /plugin/_get_cve_actions def _get_cve_actions(self): cve = request.args.get('cve', type=str) if not current_user.is_authenticated( ): # Don't show actions requiring auth if not authenticated actions = [ x for x in self.plugManager.getCVEActions( cve, **self.pluginArgs) if not x['auth'] ] else: actions = self.plugManager.getCVEActions(cve, **self.pluginArgs) return jsonify({"actions": actions}) # /plugin/<plugin> def openPlugin(self, plugin): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openPage(plugin, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) else: abort(404) # /plugin/<plugin>/subpage/<page> def openPluginSubpage(self, plugin, page): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openSubpage(plugin, page, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) else: abort(404) # /plugin/<plugin>/_cve_action/<action> def _jsonCVEAction(self, plugin, action): cve = request.args.get('cve', type=str) response = self.plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **self.pluginArgs) if type(response) is bool and response is True: return jsonify({'status': 'plugin_action_complete'}) elif type(response) is bool and response is False or response is None: return jsonify({'status': 'plugin_action_failed'}) elif type(response) is dict: return jsonify(response) # /admin # /admin/ def admin(self): if Configuration.loginRequired(): if not current_user.is_authenticated(): return render_template('login.html') else: person = User.get("_dummy_", self.auth_handler) login_user(person) output = None master = db.isMasterAccount(current_user.get_id()) checked = ct.checkCronJobExists('cve_search') if os.path.isfile(Configuration.getUpdateLogFile()): with open(Configuration.getUpdateLogFile()) as updateFile: separator = "==========================\n" output = updateFile.read().split(separator)[-2:] output = separator + separator.join(output) return render_template('admin.html', status="default", master=master, checked=checked, **self.adminInfo(output)) # /admin/change_pass @login_required def change_pass(self): current_pass = request.args.get('current_pass') new_pass = request.args.get('new_pass') if current_user.authenticate(current_pass): if new_pass: db.changePassword(current_user.id, new_pass) return jsonify({"status": "password_changed"}) return jsonify({"status": "no_password"}) else: return jsonify({"status": "wrong_user_pass"}) # /admin/request_token @login_required def request_token(self): return jsonify({"token": db.generateBearer(current_user.id)}) # /admin/updatedb @login_required def updatedb(self): process = subprocess.Popen([ sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str( err, 'utf-8')) if err else str(out, 'utf-8') return jsonify({"updateOutput": output, "status": "db_updated"}) # /admin/enableupdatedb @login_required def enableupdatedb(self): status = request.args['checked'] if status == 'true': retval = ct.createCrontab() return jsonify({ "status": "crontab_created", "crontabupdate": "cve_search" }) else: retval = ct.removeCronTab() return jsonify({ "status": "crontab_removed", "crontabupdate": "cve_search" }) # /admin/enableupdatedb/status @login_required def enableupdatedbstatus(self): logging.info('Retriving information about cron') # /admin/whitelist # /admin/blacklist @login_required def listView(self): if request.url_rule.rule.split('/')[2].lower() == 'whitelist': return render_template('list.html', rules=db.getWhitelist( current_user.get_id()), user=current_user.get_id(), listType="Whitelist") else: return render_template('list.html', rules=db.getBlacklist( current_user.get_id()), user=current_user.get_id(), listType="Blacklist") # /admin/whitelist/import # /admin/blacklist/import @login_required def listImport(self, force=None, path=None): _list = request.url_rule.split('/')[2] file = request.files['file'] force = request.form.get('force') count = wl.countWhitelist( ) if _list.lower == 'whitelist' else bl.countBlacklist() if (count == 0) | (not count) | (force == "f"): if _list.lower == 'whitelist': wl.dropWhitelist() wl.importWhitelist(TextIOWrapper(file.stream)) else: bl.dropBlacklist() bl.importBlacklist(TextIOWrapper(file.stream)) status = _list[0] + "l_imported" else: status = _list[0] + "l_already_filled" return render_template('admin.html', status=status, **self.adminInfo()) # /admin/whitelist/export # /admin/blacklist/export @login_required def listExport(self, force=None, path=None): _list = request.url_rule.rule.split('/')[2] bytIO = BytesIO() data = wl.exportWhitelist( ) if _list.lower == 'whitelist' else bl.exportBlacklist() bytIO.write(bytes(data, "utf-8")) bytIO.seek(0) return send_file(bytIO, as_attachment=True, attachment_filename=_list + ".txt") #TODO: we want that only user can drop list that he owns # /admin/whitelist/drop # /admin/blacklist/drop @login_required def listDrop(self): _list = str(request.url_rule).split('/')[2].lower() logging.info('User {0} is droping {1}'.format(current_user.get_id(), _list)) if _list == 'whitelist': wl.dropWhitelist(current_user.get_id()) else: bl.dropBlacklist(current_user.get_id()) return jsonify({"status": _list[0] + "l_dropped"}) # /admin/addToList @login_required def listAdd(self): cpe = request.args.get('cpe') cpeType = request.args.get('type') lst = request.args.get('list') isglobal = False if db.isMasterAccount(current_user.get_id()): isglobal = True logging.info( "CPE:{0} cpeType:{1} lst:{2} isglobal:{3} user:{4}".format( cpe, cpeType, lst, isglobal, current_user.get_id())) if cpe and cpeType and lst: status = "added_to_list" if self.addCPEToList( cpe, lst, cpeType, isglobal, current_user.get_id()) else "already_exists_in_list" print(status) returnList = db.getWhitelist(user=current_user.get_id( )) if lst == "Whitelist" else db.getBlacklist( user=current_user.get_id()) pprint(returnList) return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) else: return jsonify({"status": "could_not_add_to_list"}) # /admin/removeFromList @login_required def listRemove(self): cpe = request.args.get('cpe', type=str) cpe = urllib.parse.quote_plus(cpe).lower() user = current_user.get_id() cpe = cpe.replace("%3a", ":") cpe = cpe.replace("%2f", "/") lst = request.args.get('list', type=str) if cpe and lst: result = wl.removeWhitelist( cpe, user) if lst.lower() == "whitelist" else bl.removeBlacklist( cpe, user) status = "removed_from_list" if ( result > 0) else "already_removed_from_list" else: status = "invalid_cpe" returnList = db.getWhitelist(user=current_user.get_id( )) if lst == "Whitelist" else db.getBlacklist( user=current_user.get_id()) pprint("Returned list {0}".format(returnList)) return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) # /admin/editInList @login_required def listEdit(self): old = request.args.get('oldCPE') new = request.args.get('cpe') lst = request.args.get('list') CPEType = request.args.get('type') if old and new: result = wl.updateWhitelist( old, new, CPEType) if lst == "whitelist" else bl.updateBlacklist( old, new, CPEType) status = "cpelist_updated" if (result) else "cpelist_update_failed" else: status = "invalid_cpe" returnList = list(db.getWhitelist()) if lst == "whitelist" else list( db.getBlacklist()) return jsonify({ "rules": returnList, "status": status, "listType": lst }) # /admin/listmanagement/<vendor>/<product> # /admin/listmanagement/<vendor> # /admin/listmanagement @login_required def listManagement(self, vendor=None, product=None): try: if product is None: # no product selected yet, so same function as /browse can be used if vendor: vendor = urllib.parse.quote_plus(vendor).lower() browseList = query.getBrowseList(vendor) vendor = browseList["vendor"] product = browseList["product"] version = None else: # product selected, product versions required version = query.getVersionsOfProduct( urllib.parse.quote_plus(product).lower()) return render_template('listmanagement.html', vendor=vendor, product=product, version=version) except redisExceptions.ConnectionError: return render_template('error.html', status={ 'except': 'redis-connection', 'info': { 'host': Configuration.getRedisHost(), 'port': Configuration.getRedisPort() } }) # /admin/listmanagement/add @login_required def listManagementAdd(self): # retrieve the separate item parts item = request.args.get('item', type=str) pprint("item0 {0}".format(item)) listType = request.args.get('list', type=str) isadmin = db.isMasterAccount(current_user.get_id()) pattern = re.compile('^[a-z:0-9.~_%-]+$') if pattern.match(item): item = item.split(":") added = False if len(item) == 1: # only vendor, so a check on cpe type is needed logging.info( "listManagementAdd: Adding from level 1:{0}".format( item[0])) if self.redisdb.sismember("t:/o", item[0]): if self.addCPEToList("cpe:/o:" + item[0], listType, isglobal=isadmin, user=current_user.get_id()): added = True if self.redisdb.sismember("t:/a", item[0]): if self.addCPEToList("cpe:/a:" + item[0], listType, isglobal=isadmin, user=current_user.get_id()): added = True if self.redisdb.sismember("t:/h", item[0]): if self.addCPEToList("cpe:/h:" + item[0], listType, isglobal=isadmin, user=current_user.get_id()): added = True elif 4 > len(item) > 1: logging.info( "size is bigger than, look for item[1]: {0}".format( item[1])) # cpe type can be found with a mongo regex query result = db.getCVEs(query={'cpe_2_2': { '$regex': item[1] }}, collection="cpe") if len(result) != 0: prefix = ((result[0])['cpe_2_2'])[:7] logging.info( "listManagementAdd: Adding from level 2:{0}{1}{2}". format(prefix, item[0], item[1])) if len(item) == 2: if self.addCPEToList(prefix + item[0] + ":" + item[1], listType, isglobal=isadmin, user=current_user.get_id()): added = True if len(item) == 3: if self.addCPEToList(prefix + item[0] + ":" + item[1] + ":" + item[2], listType, isglobal=isadmin, user=current_user.get_id()): added = True status = "added_to_list" if added else "could_not_add_to_list" else: status = "invalid_cpe" j = {"status": status, "listType": listType} return jsonify(j) # /login def login_check(self): # validate username and password username = request.form.get('username') password = request.form.get('password') person = User.get(username, self.auth_handler) try: if person and person.authenticate(password): login_user(person) return redirect('admin') else: return render_template('login.html', status="wrong_user_pass") except Exception as e: print(e) return render_template('login.html', status="outdated_database") def register(self): if request.method == 'POST': if request.form['username'] == '': return render_template('register.html', status='username_missing') if request.form['password'] != request.form['password1']: return render_template('register.html', status='password_equal') if db.userExists(request.form['username']): return render_template('register.html', status='user_exists') db.addUser(request.form['username'], request.form['password'], localOnly=True, hashed=False) if request.form['email'] != '': db.setEmailAddress(request.form['username'], request.form['email']) return render_template('login.html', status="please_login") if request.method == 'GET': return render_template('register.html', status='') # /logout @login_required def logout(self): logout_user() return redirect("/")
import json from lib.PluginManager import PluginManager pm = PluginManager() pm.loadPlugins() cves = {} for _id in pm.getAllCVEIDs(): cves[_id] = pm.getCVERefs(_id) for _id in pm.getAllCVEIDs(): cves[_id] = pm.updateRefs(_id, cves[_id]) open("VIA4CVE-feed.json", "w").write(json.dumps(cves))
class Index(Minimal, Advanced_API): """ Redis Functions """ def vendors(self, r): return r.sunion('t:/a', 't:/h', 't:/o') def vendor_products(self, r, vendor): allproduct = [] products = r.smembers('v:' + vendor) for product in products: allproduct.append(product) return allproduct def get_vendor(self, r, product): result = [] allvendors = r.sunion('t:/a', 't:/h', 't:/o') for vendor in allvendors: if product in self.vendor_products(r, vendor): result.append(vendor) return result def product_versions(self, r, product): allversion = [] versions = r.smembers('p:' + product) for version in versions: allversion.append(version) return allversion def search_vendor(self, r, search): result = [] for vendor in self.vendors(r): if search in vendor: result.append(vendor) return result def search_product(self, r, search): result = [] for vendor in self.vendors(r): for product in vendor_products(r, vendor): if search in product: result.append(product) return result def search_vendor_product(self, r, search, vendor): result = [] for product in self.vendor_products(r, vendor): if search in product: result.append(product) return result ############# # Variables # ############# def __init__(self): # TODO: make auth handler and plugin manager singletons Advanced_API.__init__(self) Minimal.__init__(self) self.minimal = False self.auth_handler = AuthenticationHandler() self.plugManager = PluginManager() self.login_manager = LoginManager() self.plugManager.loadPlugins() self.login_manager.init_app(self.app) self.login_manager.user_loader(self.load_user) self.redisdb = Configuration.getRedisVendorConnection() self.defaultFilters.update({ 'blacklistSelect': 'on', 'whitelistSelect': 'on', 'unlistedSelect': 'show', }) self.args.update({'minimal': False}) self.pluginArgs = { "current_user": current_user, "plugin_manager": self.plugManager } routes = [{ 'r': '/cve/<cveid>', 'm': ['GET'], 'f': self.cve }, { 'r': '/_get_plugins', 'm': ['GET'], 'f': self._get_plugins }, { 'r': '/plugin/_get_cve_actions', 'm': ['GET'], 'f': self._get_cve_actions }, { 'r': '/plugin/<plugin>', 'm': ['GET'], 'f': self.openPlugin }, { 'r': '/plugin/<plugin>/subpage/<page>', 'm': ['GET'], 'f': self.openPluginSubpage }, { 'r': '/plugin/<plugin>/_cve_action/<action>', 'm': ['GET'], 'f': self._jsonCVEAction }, { 'r': '/login', 'm': ['POST'], 'f': self.login_check }, { 'r': '/logout', 'm': ['GET'], 'f': self.logout }, { 'r': '/admin', 'm': ['GET'], 'f': self.admin }, { 'r': '/admin/', 'm': ['GET'], 'f': self.admin }, { 'r': '/admin/change_pass', 'm': ['GET'], 'f': self.change_pass }, { 'r': '/admin/request_token', 'm': ['GET'], 'f': self.request_token }, { 'r': '/admin/updatedb', 'm': ['GET'], 'f': self.updatedb }, { 'r': '/admin/whitelist/import', 'm': ['POST'], 'f': self.listImport }, { 'r': '/admin/blacklist/import', 'm': ['POST'], 'f': self.listImport }, { 'r': '/admin/whitelist/export', 'm': ['GET'], 'f': self.listExport }, { 'r': '/admin/blacklist/export', 'm': ['GET'], 'f': self.listExport }, { 'r': '/admin/whitelist/drop', 'm': ['POST'], 'f': self.listDrop }, { 'r': '/admin/blacklist/drop', 'm': ['POST'], 'f': self.listDrop }, { 'r': '/admin/whitelist', 'm': ['GET'], 'f': self.listView }, { 'r': '/admin/blacklist', 'm': ['GET'], 'f': self.listView }, { 'r': '/admin/addToList', 'm': ['GET'], 'f': self.listAdd }, { 'r': '/admin/removeFromList', 'm': ['GET'], 'f': self.listRemove }, { 'r': '/admin/editInList', 'm': ['GET'], 'f': self.listEdit }, { 'r': '/admin/listmanagement', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/<vendor>', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/<vendor>/<product>', 'm': ['GET'], 'f': self.listManagement }, { 'r': '/admin/listmanagement/add', 'm': ['GET'], 'f': self.listManagementAdd }, { 'r': '/login', 'm': ['POST'], 'f': self.login_check }, { 'r': '/notifications', 'm': ['GET'], 'f': self.notifications }, { 'r': '/notifications', 'm': ['POST'], 'f': self.notifications_add }, { 'r': '/notifications', 'm': ['DELETE'], 'f': self.notifications_delete }, { 'r': '/notiftab', 'm': ['GET'], 'f': self.notiftab }, { 'r': '/notifjson', 'm': ['POST'], 'f': self.notifjson }] for route in routes: self.addRoute(route) ############# # Functions # ############# def generate_full_query(self, f): query = self.generate_minimal_query(f) if current_user.is_authenticated(): if f['blacklistSelect'] == "on": regexes = db.getRules('blacklist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ '$or': [{ 'vulnerable_configuration': re.compile(exp) }, { 'vulnerable_configuration': { '$exists': False } }, { 'vulnerable_configuration': [] }] }) if f['whitelistSelect'] == "hide": regexes = db.getRules('whitelist') if len(regexes) != 0: exp = "^(?!" + "|".join(regexes) + ")" query.append({ '$or': [{ 'vulnerable_configuration': re.compile(exp) }, { 'vulnerable_configuration': { '$exists': False } }, { 'vulnerable_configuration': [] }] }) if f['unlistedSelect'] == "hide": wlregexes = tk.compile(db.getRules('whitelist')) blregexes = tk.compile(db.getRules('blacklist')) query.append({ '$or': [{ 'vulnerable_configuration': { '$in': wlregexes } }, { 'vulnerable_configuration': { '$in': blregexes } }] }) return query def markCPEs(self, cve): blacklist = tk.compile(db.getRules('blacklist')) whitelist = tk.compile(db.getRules('whitelist')) for conf in cve['vulnerable_configuration']: conf['list'] = 'none' conf['match'] = 'none' for w in whitelist: if w.match(conf['id']): conf['list'] = 'white' conf['match'] = w for b in blacklist: if b.match(conf['id']): conf['list'] = 'black' conf['match'] = b return cve def filter_logic(self, filters, skip, limit=None): query = self.generate_full_query(filters) limit = limit if limit else self.args['pageLength'] cve = db.getCVEs(limit=limit, skip=skip, query=query) # marking relevant records if current_user.is_authenticated(): if filters['whitelistSelect'] == "on": cve['results'] = self.list_mark('white', cve['results']) if filters['blacklistSelect'] == "mark": cve['results'] = self.list_mark('black', cve['results']) self.plugManager.mark(cve, **self.pluginArgs) cve = list(cve) return cve def addCPEToList(self, cpe, listType, cpeType=None): def addCPE(cpe, cpeType, funct): return True if funct(cpe, cpeType) else False if not cpeType: cpeType = 'cpe' if listType.lower() in ("blacklist", "black", "b", "bl"): return addCPE(cpe, cpeType, bl.insertBlacklist) if listType.lower() in ("whitelist", "white", "w", "wl"): return addCPE(cpe, cpeType, wl.insertWhitelist) def list_mark(self, listed, cveList): if listed not in ['white', 'black']: return list(cves) items = tk.compile(db.getRules(listed + 'list')) # check the cpes (full or partially) in the black/whitelist for i, cve in enumerate( list(cveList) ): # the list() is to ensure we don't have a pymongo cursor object for c in cve['vulnerable_configuration']: if any(regex.match(c) for regex in items): cveList[i][listed + 'listed'] = 'yes' return cveList def filterUpdateField(self, data): if not data: return data returnvalue = [] for line in data.split("\n"): if (not line.startswith("[+]Success to create index") and not line == "Not modified" and not line.startswith("Starting")): returnvalue.append(line) return "\n".join(returnvalue) def adminInfo(self, output=None): return { 'stats': db.getDBStats(True), 'plugins': self.plugManager.getPlugins(), 'updateOutput': self.filterUpdateField(output), 'token': db.getToken(current_user.id) } # user management def load_user(self, id): return User.get(id, self.auth_handler) ########## # ROUTES # ########## # /cve/<cveid> def cve(self, cveid): cveid = cveid.upper() cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True, subscorelookup=True) cve = cvesp.getcve(cveid=cveid) if cve is None: return render_template('error.html', status={ 'except': 'cve-not-found', 'info': { 'cve': cveid } }) cve = self.markCPEs(cve) self.plugManager.onCVEOpen(cveid, **self.pluginArgs) pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs) return render_template('cve.html', cve=cve, plugins=pluginData) # /_get_plugins def _get_plugins(self): if not current_user.is_authenticated( ): # Don't show plugins requiring auth if not authenticated plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs) if not x.requiresAuth] else: plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs)] return jsonify({"plugins": plugins}) # /plugin/_get_cve_actions def _get_cve_actions(self): cve = request.args.get('cve', type=str) if not current_user.is_authenticated( ): # Don't show actions requiring auth if not authenticated actions = [ x for x in self.plugManager.getCVEActions( cve, **self.pluginArgs) if not x['auth'] ] else: actions = self.plugManager.getCVEActions(cve, **self.pluginArgs) return jsonify({"actions": actions}) # /plugin/<plugin> def openPlugin(self, plugin): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openPage(plugin, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) else: abort(404) # /plugin/<plugin>/subpage/<page> def openPluginSubpage(self, plugin, page): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: page, args = self.plugManager.openSubpage(plugin, page, **self.pluginArgs) if page: try: return render_template(page, **args) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) else: abort(404) # /plugin/<plugin>/_cve_action/<action> def _jsonCVEAction(self, plugin, action): cve = request.args.get('cve', type=str) response = self.plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **self.pluginArgs) if type(response) is bool and response is True: return jsonify({'status': 'plugin_action_complete'}) elif type(response) is bool and response is False or response is None: return jsonify({'status': 'plugin_action_failed'}) elif type(response) is dict: return jsonify(response) # /admin # /admin/ def admin(self): if Configuration.loginRequired(): if not current_user.is_authenticated(): return render_template('login.html') else: person = User.get("_dummy_", self.auth_handler) login_user(person) output = None if os.path.isfile(Configuration.getUpdateLogFile()): with open(Configuration.getUpdateLogFile()) as updateFile: separator = "==========================\n" output = updateFile.read().split(separator)[-2:] output = separator + separator.join(output) return render_template('admin.html', status="default", **self.adminInfo(output)) # /admin/change_pass @login_required def change_pass(self): current_pass = request.args.get('current_pass') new_pass = request.args.get('new_pass') if current_user.authenticate(current_pass): if new_pass: db.changePassword(current_user.id, new_pass) return jsonify({"status": "password_changed"}) return jsonify({"status": "no_password"}) else: return jsonify({"status": "wrong_user_pass"}) # /admin/request_token @login_required def request_token(self): return jsonify({"token": db.generateToken(current_user.id)}) # /admin/updatedb @login_required def updatedb(self): process = subprocess.Popen([ sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str( err, 'utf-8')) if err else str(out, 'utf-8') return jsonify({"updateOutput": output, "status": "db_updated"}) # /admin/whitelist # /admin/blacklist @login_required def listView(self): if request.url_rule.rule.split('/')[2].lower() == 'whitelist': return render_template('list.html', rules=db.getWhitelist(), listType="Whitelist") else: return render_template('list.html', rules=db.getBlacklist(), listType="Blacklist") # /admin/whitelist/import # /admin/blacklist/import @login_required def listImport(self, force=None, path=None): _list = request.url_rule.split('/')[2] file = request.files['file'] force = request.form.get('force') count = wl.countWhitelist( ) if _list.lower == 'whitelist' else bl.countBlacklist() if (count == 0) | (not count) | (force == "f"): if _list.lower == 'whitelist': wl.dropWhitelist() wl.importWhitelist(TextIOWrapper(file.stream)) else: bl.dropBlacklist() bl.importBlacklist(TextIOWrapper(file.stream)) status = _list[0] + "l_imported" else: status = _list[0] + "l_already_filled" return render_template('admin.html', status=status, **self.adminInfo()) # /admin/whitelist/export # /admin/blacklist/export @login_required def listExport(self, force=None, path=None): _list = request.url_rule.rule.split('/')[2] bytIO = BytesIO() data = wl.exportWhitelist( ) if _list.lower == 'whitelist' else bl.exportBlacklist() bytIO.write(bytes(data, "utf-8")) bytIO.seek(0) return send_file(bytIO, as_attachment=True, attachment_filename=_list + ".txt") # /admin/whitelist/drop # /admin/blacklist/drop @login_required def listDrop(self): _list = request.url_rule.split('/')[2].lower() if _list == 'whitelist': wl.dropWhitelist() else: bl.dropBlacklist() return jsonify({"status": _list[0] + "l_dropped"}) # /admin/addToList @login_required def listAdd(self): cpe = request.args.get('cpe') cpeType = request.args.get('type') lst = request.args.get('list') if cpe and cpeType and lst: status = "added_to_list" if self.addCPEToList( cpe, lst, cpeType) else "already_exists_in_list" returnList = db.getWhitelist( ) if lst == "whitelist" else db.getBlacklist() return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) else: return jsonify({"status": "could_not_add_to_list"}) # /admin/removeFromList @login_required def listRemove(self): cpe = request.args.get('cpe', type=str) cpe = urllib.parse.quote_plus(cpe).lower() cpe = cpe.replace("%3a", ":") cpe = cpe.replace("%2f", "/") lst = request.args.get('list', type=str) if cpe and lst: result = wl.removeWhitelist( cpe) if lst.lower() == "whitelist" else bl.removeBlacklist(cpe) status = "removed_from_list" if ( result > 0) else "already_removed_from_list" else: status = "invalid_cpe" returnList = db.getWhitelist( ) if lst == "whitelist" else db.getBlacklist() return jsonify({ "status": status, "rules": returnList, "listType": lst.title() }) # /admin/editInList @login_required def listEdit(self): old = request.args.get('oldCPE') new = request.args.get('cpe') lst = request.args.get('list') CPEType = request.args.get('type') if old and new: result = wl.updateWhitelist( old, new, CPEType) if lst == "whitelist" else bl.updateBlacklist( old, new, CPEType) status = "cpelist_updated" if (result) else "cpelist_update_failed" else: status = "invalid_cpe" returnList = list(db.getWhitelist()) if lst == "whitelist" else list( db.getBlacklist()) return jsonify({ "rules": returnList, "status": status, "listType": lst }) # /admin/listmanagement/<vendor>/<product> # /admin/listmanagement/<vendor> # /admin/listmanagement @login_required def listManagement(self, vendor=None, product=None): try: if product is None: # no product selected yet, so same function as /browse can be used if vendor: vendor = urllib.parse.quote_plus(vendor).lower() browseList = query.getBrowseList(vendor) vendor = browseList["vendor"] product = browseList["product"] version = None else: # product selected, product versions required version = query.getVersionsOfProduct( urllib.parse.quote_plus(product).lower()) return render_template('listmanagement.html', vendor=vendor, product=product, version=version) except redisExceptions.ConnectionError: return render_template('error.html', status={ 'except': 'redis-connection', 'info': { 'host': Configuration.getRedisHost(), 'port': Configuration.getRedisPort() } }) # /admin/listmanagement/add @login_required def listManagementAdd(self): # retrieve the separate item parts item = request.args.get('item', type=str) listType = request.args.get('list', type=str) pattern = re.compile('^[a-z:/0-9.~_%-]+$') if pattern.match(item): item = item.split(":") added = False if len(item) == 1: # only vendor, so a check on cpe type is needed if self.redisdb.sismember("t:/o", item[0]): if self.addCPEToList("cpe:/o:" + item[0], listType): added = True if self.redisdb.sismember("t:/a", item[0]): if self.addCPEToList("cpe:/a:" + item[0], listType): added = True if self.redisdb.sismember("t:/h", item[0]): if self.addCPEToList("cpe:/h:" + item[0], listType): added = True elif 4 > len(item) > 1: # cpe type can be found with a mongo regex query result = db.getCVEs(query={'cpe_2_2': { '$regex': item[1] }})['results'] if result.count() != 0: prefix = ((result[0])['cpe_2_2'])[:7] if len(item) == 2: if self.addCPEToList(prefix + item[0] + ":" + item[1], listType): added = True if len(item) == 3: if self.addCPEToList( prefix + item[0] + ":" + item[1] + ":" + item[2], listType): added = True status = "added_to_list" if added else "could_not_add_to_list" else: status = "invalid_cpe" j = {"status": status, "listType": listType} return jsonify(j) # /login def login_check(self): # validate username and password username = request.form.get('username') password = request.form.get('password') person = User.get(username, self.auth_handler) try: if person and person.authenticate(password): login_user(person) return render_template('admin.html', status="logged_in", **self.adminInfo()) else: return render_template('login.html', status="wrong_user_pass") except Exception as e: print(e) return render_template('login.html', status="outdated_database") # /logout @login_required def logout(self): logout_user() return redirect("/") # /notifications @login_required def notifications(self): return render_template('notifications.html') # /notifications POST @login_required def notifications_add(self): self.app.logger.info(request.json['notifemail']) if request.json["notifemail"]: email = request.json["notifemail"] if request.json['allversion'] is True and request.json[ 'allproduct'] is False: notification = self.Notification( user=current_user.id, email=email, vendor=escape(request.json['queryvendor'].lower()), product=escape(request.json['queryproduct'].lower()), version='') elif request.json['allproduct'] is True: notification = self.Notification( user=current_user.id, email=email, vendor=escape(request.json['queryvendor'].lower()), product='', version='') else: notification = self.Notification( user=current_user.id, email=email, vendor=escape(request.json['queryvendor'].lower()), product=escape(request.json['queryproduct'].lower()), version=escape(request.json['queryversion'].lower())) # Checking Integrity Before Insert # if self.Notification.query.filter_by( user=notification.user, email=email, vendor=notification.vendor, product=notification.product, version=notification.version).first() is None: self.session.add(notification) self.session.commit() flash('Notification Successfully Created.', 'success') return "{}", 200 else: flash('Notification Already existing.', 'warning') return "{}", 200 else: return "{}", 400 # /notifications DELETE @login_required def notifications_delete(self): row = self.Notification.query.filter_by(id=request.json).first() self.session.delete(row) self.session.commit() flash('Notification removed ', 'info') return "{}", 200 # /notiftab @login_required def notiftab(self): jnotif = [] dic = {} limit = request.args.get('limit') offset = request.args.get('offset') sort = request.args.get('sort') order = request.args.get('order') if order == 'desc': notif_list = self.Notification.query.filter_by( user=current_user.id).order_by( desc(sort)).limit(limit).offset(offset).all() else: notif_list = self.Notification.query.filter_by( user=current_user.id).order_by(sort).limit(limit).offset( offset).all() num = self.Notification.query.filter_by(user=current_user.id).count() for notif in notif_list: dnotif = { 'id': notif.id, 'fulltxt': notif.fulltxt, 'vendor': notif.vendor, 'product': notif.product, 'version': notif.version, 'email': notif.email } jnotif.append(dnotif) dic['total'] = num dic['rows'] = jnotif return jsonify(dic) @login_required def notifjson(self): vendors = [] products = [] versions = [] if request.json is not None: if request.json['queryvendor'] != "": # SEARCH BY VENDORS vendors = self.search_vendor( self.redisdb, request.json['queryvendor'].lower()) products = self.search_vendor_product( self.redisdb, request.json['queryproduct'].lower(), request.json['queryvendor'].lower()) versions = self.product_versions( self.redisdb, request.json['queryproduct'].lower()) elif request.json['queryproduct'] != "": # SEARCH BY PRODUCTS products = self.search_product( self.redisdb, request.json['queryproduct'].lower()) vendors = self.get_vendor(self.redisdb, request.json['queryproduct'].lower()) versions = self.product_versions( self.redisdb, request.json['queryproduct'].lower()) return jsonify({ "vendors": vendors, "products": products, "versions": versions })
class Index(Minimal, Advanced_API): ############# # Variables # ############# def __init__(self): Advanced_API.__init__(self) Minimal.__init__(self) self.minimal = False self.auth_handler = AuthenticationHandler() self.plugManager = PluginManager() self.login_manager = LoginManager() self.plugManager.loadPlugins() self.login_manager.init_app(self.app) self.login_manager.user_loader(self.load_user) self.redisdb = Configuration.getRedisVendorConnection() self.args.update({'minimal': False}) self.pluginArgs = { "current_user": current_user, "plugin_manager": self.plugManager } ############# # Functions # ############# def generate_full_query(self, f): query = self.generate_minimal_query(f) query.extend(self.plugManager.doFilter(f, **self.pluginArgs)) return query def filter_logic(self, filters, skip, limit=None): query = self.generate_full_query(filters) limit = limit if limit else self.args['pageLength'] cve = self.db.CVE.query(limit=limit, skip=skip, query=query) cve = [CVEPresentation(c) for c in cve] # marking relevant records self.plugManager.mark(cve, **self.pluginArgs) cve = list(cve) return cve def filterUpdateField(self, data): if not data: return data returnvalue = [] for line in data.split("\n"): if (not line.startswith("[+]Success to create index") and not line == "Not modified" and not line.startswith("Starting")): returnvalue.append(line) return "\n".join(returnvalue) def adminInfo(self, output=None): return { 'stats': self.db.db_info(True), 'plugins': self.plugManager.getPlugins(), 'updateOutput': self.filterUpdateField(output), 'token': self.db.Users.getToken(current_user.id) } # user management def load_user(self, id): return User.get(id, self.auth_handler) ########## # ROUTES # ########## def index(self): cve = self.filter_logic(self.defaultFilters, 0) filters = self.plugManager.getFilters(**self.pluginArgs) return render_template('index.html', cve=cve, r=0, filters=filters, **self.args) # / def index_post(self): args = dict(self.getFilterSettingsFromPost(0), **self.args) filters = self.plugManager.getFilters(**self.pluginArgs) return render_template('index.html', r=0, filters=filters, **args) # /r/<r> def index_filter_get(self, r): if not r or r < 0: r = 0 cve = self.filter_logic(self.defaultFilters, r) filters = self.plugManager.getFilters(**self.pluginArgs) return render_template('index.html', cve=cve, r=r, filters=filters, **self.args) # /r/<r> def index_filter_post(self, r): if not r or r < 0: r = 0 args = dict(self.getFilterSettingsFromPost(r), **self.args) filters = self.plugManager.getFilters(**self.pluginArgs) return render_template('index.html', r=r, filters=filters, **args) # /cve/<cveid> def cve(self, cveid): cve = self.api_cve(cveid) if cve is None: return render_template('error.html', status={ 'except': 'cve-not-found', 'info': { 'cve': cveid } }) self.plugManager.markCPE(cve) self.plugManager.onCVEOpen(cveid, **self.pluginArgs) pluginData = self.plugManager.cvePluginInfo(cveid, **self.pluginArgs) return render_template('cve.html', cve=cve, plugins=pluginData) # /_get_plugins def _get_plugins(self): if not current_user.is_authenticated( ): # Don't show plugins requiring auth if not authenticated plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs) if not x.requiresAuth] else: plugins = [{ "name": x.getName(), "link": x.getUID() } for x in self.plugManager.getWebPluginsWithPage( **self.pluginArgs)] return jsonify({"plugins": plugins}) # /plugin/_get_cve_actions def _get_cve_actions(self): cve = request.args.get('cve', type=str) if not current_user.is_authenticated( ): # Don't show actions requiring auth if not authenticated actions = [ x for x in self.plugManager.getCVEActions( cve, **self.pluginArgs) if not x['auth'] ] else: actions = self.plugManager.getCVEActions(cve, **self.pluginArgs) return jsonify({"actions": actions}) # /plugin/<plugin> def openPlugin(self, plugin): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: data = self.plugManager.openPage(plugin, **self.pluginArgs) if data: page, data, mimetype = data if page: try: return render_template(page, **data) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) elif data: return Response(data, mimetype=mimetype) abort(404) # /plugin/<plugin>/subpage/<page> def openPluginSubpage(self, plugin, page): if self.plugManager.requiresAuth( plugin) and not current_user.is_authenticated(): return render_template("requiresAuth.html") else: data = self.plugManager.openSubpage(plugin, page, **self.pluginArgs) if data: page, data, mimetype = data if page: try: return render_template(page, **data) except jinja2.exceptions.TemplateSyntaxError: return render_template( "error.html", status={'except': 'plugin-page-corrupt'}) except jinja2.exceptions.TemplateNotFound: return render_template("error.html", status={ 'except': 'plugin-page-not-found', 'page': page }) elif data: return Response(data, mimetype=mimetype) abort(404) # /plugin/<plugin>/_cve_action/<action> def _jsonCVEAction(self, plugin, action): cve = request.args.get('cve', type=str) response = self.plugManager.onCVEAction(cve, plugin, action, fields=dict(request.args), **self.pluginArgs) if type(response) is bool and response is True: return jsonify({'status': 'plugin_action_complete'}) elif type(response) is bool and response is False or response is None: return jsonify({'status': 'plugin_action_failed'}) elif type(response) is dict: return jsonify(response) else: return jsonify({'status': 'no_plugin_output'}) # /admin # /admin/ def admin(self): if Configuration.loginRequired(): if not current_user.is_authenticated(): return render_template('login.html') else: person = User.get("_dummy_", self.auth_handler) login_user(person) output = None if os.path.isfile(Configuration.getUpdateLogFile()): with open(Configuration.getUpdateLogFile()) as updateFile: separator = "==========================\n" output = updateFile.read().split(separator)[-2:] output = separator + separator.join(output) return render_template('admin.html', status="default", **self.adminInfo(output)) # /admin/change_pass @login_required def change_pass(self): current_pass = request.args.get('current_pass') new_pass = request.args.get('new_pass') if current_user.authenticate(current_pass): if new_pass: self.db.Users.changePassword(current_user.id, new_pass) return jsonify({"status": "password_changed"}) return jsonify({"status": "no_password"}) else: return jsonify({"status": "wrong_user_pass"}) # /admin/request_token @login_required def request_token(self): return jsonify({"token": self.db.Users.generateToken(current_user.id)}) # /admin/updatedb @login_required def updatedb(self): process = subprocess.Popen([ sys.executable, os.path.join(_runPath, "../sbin/db_updater.py"), "-civ" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = process.communicate() output = "%s\n\nErrors:\n%s" % (str(out, 'utf-8'), str( err, 'utf-8')) if err else str(out, 'utf-8') return jsonify({"updateOutput": output, "status": "db_updated"}) # /login def login_check(self): # validate username and password username = request.form.get('username') password = request.form.get('password') person = User.get(username, self.auth_handler) try: if person and person.authenticate(password): login_user(person) return render_template('admin.html', status="logged_in", **self.adminInfo()) else: return render_template('login.html', status="wrong_user_pass") except Exception as e: print(e) return render_template('login.html', status="outdated_database") # /logout @login_required def logout(self): logout_user() return redirect("/")
parser.add_argument('--no-update', action='store_true', help="DEBUG: don't run the update part of the sources") parser.add_argument( '--no-cleanup', action='store_true', help="DEBUG: don't run the cleanup part of the sources") parser.add_argument( '--verify', action='store_true', help="Verify that the created file passes the unit test") args = parser.parse_args() pm = PluginManager() # Create plug-in manager pm.loadPlugins() # Load all sources & parse data cves = {} # Create empty dictionary to fill up path = args.file if args.file else "VIA4CVE-feed.json" # Generate path based on user preferences for _id in pm.getAllCVEIDs(): cves[_id] = pm.getCVERefs(_id) # Get data per CVE if not args.no_update: for _id in cves.keys(): pm.updateRefs(_id, cves[_id]) # Update data based on previous data if not args.no_cleanup: for _id in cves.keys(): pm.cleanUp(_id, cves[_id]) # Clean data data = { 'cves': cves,