def adminStats(): cveU = db.info.find_one({"db": "cve"}) cpeU = db.info.find_one({"db": "cpe"}) cpeOtherU = db.info.find_one({"db": "cpeother"}) capecU = db.info.find_one({"db": "capec"}) d2secU = db.info.find_one({"db": "d2sec"}) vendorU = db.info.find_one({"db": "vendor"}) vfeedU = db.info.find_one({"db": "vfeed"}) stats = { "cveA": db.cves.count(), "cveU": cveU["last-modified"] if cveU is not None else None, "cpeA": db.cpe.count(), "cpeU": cpeU["last-modified"] if cpeU is not None else None, "cpeOtherA": db.cpeother.count(), "cpeOtherU": cpeOtherU["last-modified"] if cpeOtherU is not None else None, "capecA": db.capec.count(), "capecU": capecU["last-modified"] if capecU is not None else None, "d2secA": db.d2sec.count(), "d2secU": d2secU["last-modified"] if d2secU is not None else None, "vendorA": db.vendor.count(), "vendorU": vendorU["last-modified"] if vendorU is not None else None, "vfeedA": db.vfeed.count(), "vfeedU": vfeedU["last-modified"] if vfeedU is not None else None, "blA": db.mgmt_blacklist.count(), "wlA": db.mgmt_whitelist.count(), "dbName": Configuration.getMongoDB(), "dbSize": db.command("dbstats")["dataSize"], "dbOnDisk": db.command("dbstats")["storageSize"], } return stats
def getVersionsOfProduct(product): p = redisdb.smembers("p:" + product) return sorted(list(p)) cveU = db.info.find_one({'db': 'cve'}) cpeU = db.info.find_one({'db': 'cpe'}) cpeOtherU = db.info.find_one({'db': 'cpeother'}) capecU = db.info.find_one({'db': 'capec'}) d2secU = db.info.find_one({'db': 'd2sec'}) vendorU = db.info.find_one({'db': 'vendor'}) vfeedU = db.info.find_one({'db': 'vfeed'}) stats = { 'cveA': db.cves.count(), 'cveU': cveU['last-modified'] if cveU is not None else None, 'cpeA': db.cpe.count(), 'cpeU': cpeU['last-modified'] if cpeU is not None else None, 'cpeOtherA': db.cpeother.count(), 'cpeOtherU': cpeOtherU['last-modified'] if cpeOtherU is not None else None, 'capecA': db.capec.count(), 'capecU': capecU['last-modified'] if capecU is not None else None, 'd2secA': db.d2sec.count(), 'd2secU': d2secU['last-modified'] if d2secU is not None else None, 'vendorA': db.vendor.count(), 'vendorU': vendorU['last-modified'] if vendorU is not None else None, 'vfeedA': db.vfeed.count(), 'vfeedU': vfeedU['last-modified'] if vfeedU is not None else None, 'blA': db.mgmt_blacklist.count(), 'wlA': db.mgmt_whitelist.count(), 'dbName': Configuration.getMongoDB(), 'dbSize': db.command("dbstats")['dataSize'], 'dbOnDisk': db.command("dbstats")['storageSize'] } return stats
def getDBStats(include_admin=False): data = { 'cves': {}, 'cpe': {}, 'cpeOther': {}, 'capec': {}, 'cwe': {}, 'via4': {} } for key in data.keys(): data[key] = { 'size': getSize(key.lower()), 'last_update': getLastModified(key.lower()) } if include_admin: data['whitelist'] = {'size': colWHITELIST.count()} data['blacklist'] = {'size': colBLACKLIST.count()} data = { 'stats': { 'size_on_disk': db.command("dbstats")['storageSize'], 'db_size': db.command('dbstats')['dataSize'], 'name': conf.getMongoDB() }, 'data': data } return data
def getDBStats(include_admin=False): data = { "cves": {}, "cpe": {}, "cpeOther": {}, "capec": {}, "cwe": {}, "via4": {} } for key in data.keys(): data[key] = { "size": getSize(key.lower()), "last_update": getLastModified(key.lower()), } if include_admin: data["whitelist"] = {"size": colWHITELIST.count()} data["blacklist"] = {"size": colBLACKLIST.count()} data = { "stats": { "size_on_disk": db.command("dbstats")["storageSize"], "db_size": db.command("dbstats")["dataSize"], "name": conf.getMongoDB(), }, "data": data, } return data
def getDBStats(): cols = ["cve", "cpe", "cpeOther", "capec", "d2sec", "vendor", "vfeed"] stats = {x + "A": getSize(x.lower()) for x in cols} stats["cveA"] = getSize("cves") stats.update({x + "U": getLastModified(x.lower()) for x in cols}) stats.update({"blA": colBLACKLIST.count(), "wlA": colWHITELIST.count()}) stats.update({"dbOnDisk": db.command("dbstats")["storageSize"], "dbSize": db.command("dbstats")["dataSize"]}) stats["dbName"] = conf.getMongoDB() return stats
def getDBStats(): cols=['cve', 'cpe', 'cpeOther', 'capec', 'd2sec', 'vendor'] stats={x+'A': getSize(x.lower()) for x in cols} stats['cveA']=getSize('cves') stats.update({x+'U': getLastModified(x.lower()) for x in cols}) stats.update({'blA': colBLACKLIST.count(), 'wlA':colWHITELIST.count()}) stats.update({'dbOnDisk': db.command("dbstats")['storageSize'], 'dbSize':db.command('dbstats')['dataSize']}) stats['dbName']=conf.getMongoDB() return stats
def getDBStats(include_admin=False): data={'cves': {}, 'cpe': {}, 'cpeOther': {}, 'capec': {}, 'cwe': {}, 'via4': {}} for key in data.keys(): data[key] = {'size': getSize(key.lower()), 'last_update': getLastModified(key.lower())} if include_admin: data['whitelist']={'size': colWHITELIST.count()} data['blacklist']={'size': colBLACKLIST.count()} data = {'stats': {'size_on_disk': db.command("dbstats")['storageSize'], 'db_size': db.command('dbstats')['dataSize'], 'name': conf.getMongoDB()}, 'data': data} return data
def adminStats(): cveU = db.info.find_one({'db': 'cve'}) cpeU = db.info.find_one({'db': 'cpe'}) cpeOtherU = db.info.find_one({'db': 'cpeother'}) capecU = db.info.find_one({'db': 'capec'}) d2secU = db.info.find_one({'db': 'd2sec'}) vendorU = db.info.find_one({'db': 'vendor'}) vfeedU = db.info.find_one({'db': 'vfeed'}) stats = {'cveA': db.cves.count(), 'cveU': cveU['last-modified'] if cveU is not None else None, 'cpeA': db.cpe.count(), 'cpeU': cpeU['last-modified'] if cpeU is not None else None, 'cpeOtherA': db.cpeother.count(), 'cpeOtherU': cpeOtherU['last-modified'] if cpeOtherU is not None else None, 'capecA': db.capec.count(), 'capecU': capecU['last-modified'] if capecU is not None else None, 'd2secA': db.d2sec.count(), 'd2secU': d2secU['last-modified'] if d2secU is not None else None, 'vendorA': db.vendor.count(), 'vendorU': vendorU['last-modified'] if vendorU is not None else None, 'vfeedA': db.vfeed.count(), 'vfeedU': vfeedU['last-modified'] if vfeedU is not None else None, 'blA': db.mgmt_blacklist.count(), 'wlA': db.mgmt_whitelist.count(), 'dbName': Configuration.getMongoDB(), 'dbSize': db.command("dbstats")['dataSize'], 'dbOnDisk': db.command("dbstats")['storageSize']} return stats
def getDBStats(include_admin=False): data = { 'cves': {}, 'cpe': {}, 'cpeOther': {}, 'capec': {}, 'cwe': {}, 'via4': {} } for key in data.keys(): data[key] = { 'size': getSize(key.lower()), 'last_update': getLastModified(key.lower()) } if include_admin: data['whitelist'] = { 'size': Configuration.getMongoConnection().get_collection( Collections.WhiteList.value).count() } data['blacklist'] = { 'size': Configuration.getMongoConnection().get_collection( Collections.BlackList.value).count() } data = { 'stats': { 'size_on_disk': Configuration.getMongoConnection().command( "dbstats")['storageSize'], 'db_size': Configuration.getMongoConnection().command('dbstats') ['dataSize'], 'name': Configuration.getMongoDB() }, 'data': data } return data
from lib.User import User from lib.Config import Configuration from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat import lib.CVEs as cves import lib.DatabaseLayer as dbLayer from sbin.db_whitelist import * from sbin.db_blacklist import * # parse command line arguments argparser = argparse.ArgumentParser(description='Start CVE-Search web component') argparser.add_argument('-v', action='store_true', help='verbose output') args = argparser.parse_args() # variables app = Flask(__name__, static_folder='static', static_url_path='/static') app.config['MONGO_DBNAME'] = Configuration.getMongoDB() app.config['SECRET_KEY'] = str(random.getrandbits(256)) pageLength = Configuration.getPageLength() # login manager login_manager = LoginManager() login_manager.init_app(app) # db connectors redisdb = Configuration.getRedisVendorConnection() # functions def getBrowseList(vendor): result = {} if (vendor is None) or type(vendor) == list: v1 = redisdb.smembers("t:/o") v2 = redisdb.smembers("t:/a")
from logging.handlers import RotatingFileHandler from lib.Config import Configuration from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat import lib.CVEs as cves import lib.DatabaseLayer as dbLayer # parse command line arguments argparser = argparse.ArgumentParser( description='Start CVE-Search web component') argparser.add_argument('-v', action='store_true', help='verbose output') args = argparser.parse_args() # variables app = Flask(__name__, static_folder='static', static_url_path='/static') app.config['MONGO_DBNAME'] = Configuration.getMongoDB() app.config['SECRET_KEY'] = str(random.getrandbits(256)) pageLength = Configuration.getPageLength() # db connectors redisdb = Configuration.getRedisVendorConnection() # functions def getBrowseList(vendor): result = {} if (vendor is None) or type(vendor) == list: v1 = redisdb.smembers("t:/o") v2 = redisdb.smembers("t:/a") v3 = redisdb.smembers("t:/h") vendor = sorted(list(set(list(v1) + list(v2) + list(v3))))
from collections import defaultdict import bson import pymongo from pymongo import DESCENDING, ASCENDING from pymongo.collection import Collection from werkzeug.security import generate_password_hash, check_password_hash from lib.DatabaseLayer import sanitize from lib.DatabasePluginBase import DatabasePluginBase from lib.Config import Configuration config = Configuration() HOST = config.readSetting("Database", "Host", config.default["mongoHost"]) PORT = config.readSetting("Database", "Port", config.default["mongoPort"]) DATABASE = config.getMongoDB() USERNAME = urllib.parse.quote( config.readSetting("Database", "Username", config.default["mongoUsername"])) PASSWORD = urllib.parse.quote( config.readSetting("Database", "Password", config.default["mongoPassword"])) exits = { "userInDb": "User already exists in database", "userNotInDb": "User does not exist in database", "userpasscombo": "Master user/password combination does not exist", "passwordMatch": "The passwords don't match!", "noMaster": "Not a master account!", "lastMaster": "This user is the last admin in the database and thus can not be removed",
from lib.User import User from lib.Config import Configuration from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat import lib.CVEs as cves import lib.DatabaseLayer as db from sbin.db_whitelist import * from sbin.db_blacklist import * # parse command line arguments argparser = argparse.ArgumentParser(description="Start CVE-Search web component") argparser.add_argument("-v", action="store_true", help="verbose output") args = argparser.parse_args() # variables app = Flask(__name__, static_folder="static", static_url_path="/static") app.config["MONGO_DBNAME"] = Configuration.getMongoDB() app.config["SECRET_KEY"] = str(random.getrandbits(256)) pageLength = Configuration.getPageLength() # login manager login_manager = LoginManager() login_manager.init_app(app) # db connectors redisdb = Configuration.getRedisVendorConnection() # functions def getBrowseList(vendor): result = {} if (vendor is None) or type(vendor) == list: v1 = redisdb.smembers("t:/o") v2 = redisdb.smembers("t:/a")
class API(): app = Flask(__name__, static_folder='static', static_url_path='/static') app.config['MONGO_DBNAME'] = Configuration.getMongoDB() app.config['SECRET_KEY'] = str(random.getrandbits(256)) def __init__(self): routes = [{ 'r': '/api/', 'm': ['GET'], 'f': self.api_documentation }, { 'r': '/api/cpe2.3/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe23 }, { 'r': '/api/cpe2.2/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe22 }, { 'r': '/api/cvefor/<path:cpe>', 'm': ['GET'], 'f': self.api_cvesFor }, { 'r': '/api/cve/<cveid>', 'm': ['GET'], 'f': self.api_cve }, { 'r': '/api/cwe', 'm': ['GET'], 'f': self.api_cwe }, { 'r': '/api/cwe/<int:cwe_id>', 'm': ['GET'], 'f': self.api_cwe }, { 'r': '/api/capec/<cweid>', 'm': ['GET'], 'f': self.api_capec }, { 'r': '/api/last', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/last/', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/last/<int:limit>', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/query', 'm': ['GET'], 'f': self.api_query }, { 'r': '/api/browse', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/browse/', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/browse/<path:vendor>', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/search/<vendor>/<path:product>', 'm': ['GET'], 'f': self.api_search }, { 'r': '/api/search/<path:search>', 'm': ['GET'], 'f': self.api_text_search }, { 'r': '/api/search/days/<days>/<path:search>', 'm': ['GET'], 'f': self.api_text_limitday_search }, { 'r': '/api/link/<key>/<value>', 'm': ['GET'], 'f': self.api_link }, { 'r': '/api/dbInfo', 'm': ['GET'], 'f': self.api_dbInfo }] for route in routes: self.addRoute(route) def addRoute(self, route): self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m']) ############# # Decorator # ############# def api(funct): @wraps(funct) def api_wrapper(*args, **kwargs): data = error = None # Get data (and possibly errors) try: data = funct(*args, **kwargs) except APIError as e: error = ({'status': 'error', 'reason': e.message}, e.status) except Exception as e: print(e) error = ({ 'status': 'error', 'reason': 'Internal server error' }, 500) # Check if data should be returned as html or data try: returnType = 'application/json' if (request.url_rule.rule.lower().startswith("/api/") or request.url_rule.rule.lower().endswith(".json")): # Support JSONP if request.args.get('callback', False): data = "%s(%s)" % (request.args.get('callback'), data) # Check API version for backwards compatibility. We'll call the old API v1.0 elif request.headers.get('Version') in ['1.1']: # Get the requested return type returnType = request.headers.get('Accept', '*/*') # Default to JSON if any(t in returnType for t in ['json', 'application/*', 'text/*', '*/*']): data = error if error else { 'status': 'success', 'data': data } elif 'plain' in returnType: pass # No need to do anything, but needs to be accepted else: data = ({ 'status': 'error', 'reason': 'Unknown Content-type requested' }, 415) returnType = 'application/json' if type(data) is not str: if type(data) is tuple: data = list(data) data[0] = json.dumps(convertDatetime(dct=data[0]), indent=4, sort_keys=True, default=json_util.default) else: data = (json.dumps(convertDatetime(dct=data), indent=4, sort_keys=True, default=json_util.default), 200) return Response(data[0], mimetype=returnType), data[1] except Exception as e: print(e) pass if error and error[1] == 500: raise (APIError(error[0]['reason'])) return data return api_wrapper ############# # FUNCTIONS # ############# def generate_minimal_query(self, f): query = [] # retrieving lists if f['rejectedSelect'] == "hide": exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)" query.append({'summary': re.compile(exp)}) # cvss logic if f['cvssSelect'] == "above": query.append({'cvss': {'$gt': float(f['cvss'])}}) elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])}) elif f['cvssSelect'] == "below": query.append({'cvss': {'$lt': float(f['cvss'])}}) # date logic if f['timeSelect'] != "all": if f['startDate']: startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True) if f['endDate']: endDate = parse_datetime(f['endDate'], ignoretz=True, dayfirst=True) if f['timeSelect'] == "from": query.append({f['timeTypeSelect']: {'$gt': startDate}}) elif f['timeSelect'] == "until": query.append({f['timeTypeSelect']: {'$lt': endDate}}) elif f['timeSelect'] == "between": query.append( {f['timeTypeSelect']: { '$gt': startDate, '$lt': endDate }}) elif f['timeSelect'] == "outside": query.append({ '$or': [{ f['timeTypeSelect']: { '$lt': startDate } }, { f['timeTypeSelect']: { '$gt': endDate } }] }) return query def filter_logic(self, filters, skip, limit=None): query = self.generate_minimal_query(filters) limit = limit if limit else self.args['pageLength'] return db.getCVEs(limit=limit, skip=skip, query=query) ########## # ROUTES # ########## # /api def api_documentation(self): return render_template('api.html') # /api/cpe2.3/<cpe> @api def api_cpe23(self, cpe): cpe = tk.toStringFormattedCPE(cpe) return cpe if cpe else "None" # /api/cpe2.2/<cpe> @api def api_cpe22(self, cpe): cpe = tk.toOldCPE(cpe) return cpe if cpe else "None" # /api/cvefor/<cpe> @api def api_cvesFor(self, cpe): cpe = urllib.parse.unquote_plus(cpe) return query.cvesForCPE(cpe) # /api/cve/<cveid> @api def api_cve(self, cveid): cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.getcve(cveid=cveid.upper()) if not cve: raise (APIError('cve not found', 404)) return cve # /api/cwe # /api/cwe/<cwe_id> @api def api_cwe(self, cwe_id=None): return db.getCAPECFor(str(cwe_id)) if cwe_id else db.getCWEs() # /api/capec/<cweid> @api def api_capec(self, cweid): return db.getCAPEC(cweid) # /api/last # /api/last/ # /api/last/<limit> @api def api_last(self, limit=None): limit = limit if limit else 30 cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.get(limit=limit) return cve # /query @api def api_query(self): f = { 'rejectedSelect': request.headers.get('rejected'), 'cvss': request.headers.get('cvss_score'), 'cvssSelect': request.headers.get('cvss_modifier'), 'startDate': request.headers.get('time_start'), 'endDate': request.headers.get('time_end'), 'timeSelect': request.headers.get('time_modifier'), 'timeTypeSelect': request.headers.get('time_type'), 'skip': request.headers.get('skip'), 'limit': request.headers.get('limit') } try: skip = int(f['skip']) if f['skip'] else 0 except: raise (APIError('skip must be an int', 400)) try: limit = int(f['limit']) if f['limit'] else 0 except: raise (APIError('limit must be an int', 400)) return self.filter_logic(f, skip, limit) # /api/browse # /api/browse/ # /api/browse/<vendor> @api def api_browse(self, vendor=None): if vendor: vendor = urllib.parse.quote_plus(vendor).lower() try: browseList = query.getBrowseList(vendor) except redisExceptions.ConnectionError: raise (APIError( "Server could not connect to the browsing repository", 503)) if isinstance(browseList, dict): return browseList else: return {} # /api/search/<vendor>/<path:product> @api def api_search(self, vendor=None, product=None): if not (vendor and product): return {} search = vendor + ":" + product # Not using query.cvesForCPE, because that one gives too much info #return json.dumps(db.cvesForCPE(search), default=json_util.default) return db.cvesForCPE(search) # /api/search/<path:search> @api def api_text_search(self, search=None): return db.getSearchResults(search) # /api/search/days/<days>/<path:search> @api def api_text_limitday_search(self, days=None, search=None): if not (search or days): return {} cve_d = [] date_n_days_ago = datetime.datetime.now() - timedelta(int(days)) result = db.getSearchResults(search) cve = result['data'] for item in cve: if item['Modified'] > date_n_days_ago: cve_d.append(item) return cve_d # /api/link/<key>/<value> @api def api_link(self, key=None, value=None): key = self.htmlDecode(key) value = self.htmlDecode(value) regex = re.compile(re.escape(value), re.I) data = {'cves': db.via4Linked(key, regex)} cvssList = [float(x['cvss']) for x in data['cves'] if x.get('cvss')] if cvssList: data['stats'] = { 'maxCVSS': max(cvssList), 'minCVSS': min(cvssList), 'count': len(data['cves']) } else: data['stats'] = { 'maxCVSS': 0, 'minCVSS': 0, 'count': len(data['cves']) } return data # /api/dbInfo @api def api_dbInfo(self): return db.getDBStats() ######################## # Web Server Functions # ######################## # signal handlers def sig_handler(self, sig, frame): print('Caught signal: %s' % sig) IOLoop.instance().add_callback(self.shutdown) def shutdown(self): MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3 print('Stopping http server') self.http_server.stop() print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN) io_loop = IOLoop.instance() deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN def stop_loop(): now = time.time() if now < deadline and (io_loop._callbacks or io_loop._timeouts): io_loop.add_timeout(now + 1, stop_loop) else: io_loop.stop() print('Shutdown') stop_loop() def start(self): # get properties flaskHost = Configuration.getFlaskHost() flaskPort = Configuration.getFlaskPort() flaskDebug = Configuration.getFlaskDebug() # logging if Configuration.getLogging(): logfile = Configuration.getLogfile() pathToLog = logfile.rsplit('/', 1)[0] if not os.path.exists(pathToLog): os.makedirs(pathToLog) maxLogSize = Configuration.getMaxLogSize() backlog = Configuration.getBacklog() file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog) file_handler.setLevel(logging.ERROR) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) self.app.logger.addHandler(file_handler) if flaskDebug: # start debug flask server self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug) else: # start asynchronous server using tornado wrapper for flask # ssl connection print("Server starting...") if Configuration.useSSL(): ssl_options = { "certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()), "keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey()) } else: ssl_options = None signal.signal(signal.SIGTERM, self.sig_handler) signal.signal(signal.SIGINT, self.sig_handler) self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options) self.http_server.bind(flaskPort, address=flaskHost) self.http_server.start(0) # Forks multiple sub-processes IOLoop.instance().start()
class API: app = Flask(__name__, static_folder="static", static_url_path="/static") app.config["MONGO_DBNAME"] = Configuration.getMongoDB() app.config["SECRET_KEY"] = str(random.getrandbits(256)) def __init__(self): routes = [ { "r": "/api/", "m": ["GET"], "f": self.api_documentation }, { "r": "/api/cpe2.3/<path:cpe>", "m": ["GET"], "f": self.api_cpe23 }, { "r": "/api/cpe2.2/<path:cpe>", "m": ["GET"], "f": self.api_cpe22 }, { "r": "/api/cvefor/<path:cpe>", "m": ["GET"], "f": self.api_cvesFor }, { "r": "/api/cvefor/<path:cpe>/<limit>", "m": ["GET"], "f": self.api_cvesFor }, { "r": "/api/cve/<cveid>", "m": ["GET"], "f": self.api_cve }, { "r": "/api/cwe", "m": ["GET"], "f": self.api_cwe }, { "r": "/api/cwe/<cwe_id>", "m": ["GET"], "f": self.api_cwe }, { "r": "/api/capec/<cweid>", "m": ["GET"], "f": self.api_capec }, { "r": "/api/capec/show/<capecid>", "m": ["GET"], "f": self.api_capec_by_id }, { "r": "/api/last", "m": ["GET"], "f": self.api_last }, { "r": "/api/last/", "m": ["GET"], "f": self.api_last }, { "r": "/api/last/<int:limit>", "m": ["GET"], "f": self.api_last }, { "r": "/api/query", "m": ["GET"], "f": self.api_query }, { "r": "/api/browse", "m": ["GET"], "f": self.api_browse }, { "r": "/api/browse/", "m": ["GET"], "f": self.api_browse }, { "r": "/api/browse/<path:vendor>", "m": ["GET"], "f": self.api_browse }, { "r": "/api/search/<vendor>/<path:product>", "m": ["GET"], "f": self.api_search, }, { "r": "/api/search/<path:search>", "m": ["GET"], "f": self.api_text_search }, { "r": "/api/link/<key>/<value>", "m": ["GET"], "f": self.api_link }, { "r": "/api/dbInfo", "m": ["GET"], "f": self.api_dbInfo }, ] for route in routes: self.addRoute(route) def addRoute(self, route): self.app.add_url_rule(route["r"], view_func=route["f"], methods=route["m"]) ############# # Decorator # ############# def api(funct): @wraps(funct) def api_wrapper(*args, **kwargs): data = error = None # Get data (and possibly errors) try: data = funct(*args, **kwargs) except APIError as e: error = ({"status": "error", "reason": e.message}, e.status) except Exception as e: print(e) error = ({ "status": "error", "reason": "Internal server error" }, 500) # Check if data should be returned as html or data try: returnType = "application/json" if request.url_rule.rule.lower().startswith( "/api/") or request.url_rule.rule.lower().endswith( ".json"): # Support JSONP if request.args.get("callback", False): data = "%s(%s)" % (request.args.get("callback"), data) # Check API version for backwards compatibility. We'll call the old API v1.0 elif request.headers.get("Version") in ["1.1"]: # Get the requested return type returnType = request.headers.get("Accept", "*/*") # Default to JSON if any(t in returnType for t in ["json", "application/*", "text/*", "*/*"]): data = (error if error else { "status": "success", "data": data }) elif "plain" in returnType: pass # No need to do anything, but needs to be accepted else: data = ( { "status": "error", "reason": "Unknown Content-type requested", }, 415, ) returnType = "application/json" if type(data) is not str: if type(data) is tuple: data = list(data) data[0] = json.dumps( convertDatetime(dct=data[0]), indent=4, sort_keys=True, default=json_util.default, ) else: data = ( json.dumps( convertDatetime(dct=data), indent=4, sort_keys=True, default=json_util.default, ), 200, ) return Response(data[0], mimetype=returnType), data[1] except Exception as e: print(e) pass if error and error[1] == 500: raise (APIError(error[0]["reason"])) return data return api_wrapper ############# # FUNCTIONS # ############# def generate_minimal_query(self, f): query = [] # retrieving lists if f["rejectedSelect"] == "hide": query.append({ "summary": re.compile( r"^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)" ) }) # cvss logic if f["cvssSelect"] == "above": query.append({"cvss": {"$gt": float(f["cvss"])}}) elif f["cvssSelect"] == "equals": query.append({"cvss": float(f["cvss"])}) elif f["cvssSelect"] == "below": query.append({"cvss": {"$lt": float(f["cvss"])}}) # date logic if f["timeSelect"] != "all": if f["startDate"]: startDate = parse_datetime(f["startDate"], ignoretz=True, dayfirst=True) if f["endDate"]: endDate = parse_datetime(f["endDate"], ignoretz=True, dayfirst=True) if f["timeSelect"] == "from": query.append({f["timeTypeSelect"]: {"$gt": startDate}}) elif f["timeSelect"] == "until": query.append({f["timeTypeSelect"]: {"$lt": endDate}}) elif f["timeSelect"] == "between": query.append( {f["timeTypeSelect"]: { "$gt": startDate, "$lt": endDate }}) elif f["timeSelect"] == "outside": query.append({ "$or": [ { f["timeTypeSelect"]: { "$lt": startDate } }, { f["timeTypeSelect"]: { "$gt": endDate } }, ] }) return query def filter_logic(self, filters, skip, limit=None): query = self.generate_minimal_query(filters) limit = limit if limit else self.args["pageLength"] return getCVEs(limit=limit, skip=skip, query=query) ########## # ROUTES # ########## # /api def api_documentation(self): return render_template("api.html") # /api/cpe2.3/<cpe> @api def api_cpe23(self, cpe): cpe = toStringFormattedCPE(cpe) return cpe, 200 if cpe else "None", 404 # /api/cpe2.2/<cpe> @api def api_cpe22(self, cpe): cpe = toOldCPE(cpe) return cpe, 200 if cpe else "None", 404 # /api/cvefor/<cpe> # /api/cvefor/<cpe>/<limit> @api def api_cvesFor(self, cpe, limit=0): cpe = urllib.parse.unquote_plus(cpe) return qcvesForCPE(cpe, int(limit)) # /api/cve/<cveid> @api def api_cve(self, cveid): cvesp = CveHandler(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.getcve(cveid=cveid.upper()) if not cve: raise (APIError("cve not found", 404)) return cve # /api/cwe # /api/cwe/<cwe_id> @api def api_cwe(self, cwe_id=None): return getCWEs(cwe_id) if cwe_id else getCWEs() # /api/capec/<cweid> @api def api_capec(self, cweid): return getCAPECFor(cweid) # /api/capec/show/<capecid> @api def api_capec_by_id(self, capecid): return getCAPEC(capecid) # /api/last # /api/last/ # /api/last/<limit> @api def api_last(self, limit=None): limit = limit if limit else 30 cvesp = CveHandler(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.get(limit=limit) return cve # /query @api def api_query(self): f = { "rejectedSelect": request.headers.get("rejected"), "cvss": request.headers.get("cvss_score"), "cvssSelect": request.headers.get("cvss_modifier"), "startDate": request.headers.get("time_start"), "endDate": request.headers.get("time_end"), "timeSelect": request.headers.get("time_modifier"), "timeTypeSelect": request.headers.get("time_type"), "skip": request.headers.get("skip"), "limit": request.headers.get("limit"), } try: skip = int(f["skip"]) if f["skip"] else 0 except: raise (APIError("skip must be an int", 400)) try: limit = int(f["limit"]) if f["limit"] else 0 except: raise (APIError("limit must be an int", 400)) return self.filter_logic(f, skip, limit) # /api/browse # /api/browse/ # /api/browse/<vendor> @api def api_browse(self, vendor=None): if vendor: vendor = urllib.parse.quote_plus(vendor).lower() try: browseList = getBrowseList(vendor) except redis_connection_error: raise (APIError( "Server could not connect to the browsing repository", 503)) if isinstance(browseList, dict): return browseList else: return {} # /api/search/<vendor>/<path:product> @api def api_search(self, vendor=None, product=None): if not (vendor and product): return {} search = vendor + ":" + product # Not using query.cvesForCPE, because that one gives too much info # return json.dumps(db.cvesForCPE(search), default=json_util.default) return cvesForCPE(search) # /api/search/<path:search> @api def api_text_search(self, search=None): return getSearchResults(search) # /api/link/<key>/<value> @api def api_link(self, key=None, value=None): key = self.htmlDecode(key) value = self.htmlDecode(value) regex = re.compile(re.escape(value), re.I) data = {"cves": via4Linked(key, regex)} cvssList = [float(x["cvss"]) for x in data["cves"] if x.get("cvss")] if cvssList: data["stats"] = { "maxCVSS": max(cvssList), "minCVSS": min(cvssList), "count": len(data["cves"]), } else: data["stats"] = { "maxCVSS": 0, "minCVSS": 0, "count": len(data["cves"]) } return data # /api/dbInfo @api def api_dbInfo(self): return getDBStats() ######################## # Web Server Functions # ######################## # signal handlers def sig_handler(self, sig, frame): print("Caught signal: %s" % sig) IOLoop.instance().add_callback(self.shutdown) def shutdown(self): MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3 print("Stopping http server") self.http_server.stop() print("Will shutdown in %s seconds ..." % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN) io_loop = IOLoop.instance() deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN def stop_loop(): now = time.time() if now < deadline and (io_loop._callbacks or io_loop._timeouts): io_loop.add_timeout(now + 1, stop_loop) else: io_loop.stop() print("Shutdown") stop_loop() def start(self): # get properties flaskHost = Configuration.getFlaskHost() flaskPort = Configuration.getFlaskPort() flaskDebug = Configuration.getFlaskDebug() # logging if Configuration.getLogging(): logfile = Configuration.getLogfile() pathToLog = logfile.rsplit("/", 1)[0] if not os.path.exists(pathToLog): os.makedirs(pathToLog) maxLogSize = Configuration.getMaxLogSize() backlog = Configuration.getBacklog() file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog) file_handler.setLevel(logging.ERROR) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) self.app.logger.addHandler(file_handler) if flaskDebug: # start debug flask server self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug) else: # start asynchronous server using tornado wrapper for flask # ssl connection print("Server starting...") if Configuration.useSSL(): ssl_options = { "certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()), "keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey()), } else: ssl_options = None signal.signal(signal.SIGTERM, self.sig_handler) signal.signal(signal.SIGINT, self.sig_handler) self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options) self.http_server.bind(flaskPort, address=flaskHost) self.http_server.start(0) # Forks multiple sub-processes IOLoop.instance().start()
def create_app(version, run_path): global app, token_blacklist, socketio app = Flask(__name__, static_url_path="/cvesearch/static", static_folder="static") cors = CORS(app) app.config['CORS_HEADERS'] = 'Content-Type' app.config["version"] = version app.config["run_path"] = run_path app.config["APPLICATION_ROOT"] = "/cvesearch" config = Configuration() if config.getWebInterface().lower() == "full": app.config["WebInterface"] = False else: app.config["WebInterface"] = True app.config["MONGO_DBNAME"] = config.getMongoDB() app.config["SECRET_KEY"] = str(random.getrandbits(256)) app.config["JWT_SECRET_KEY"] = str(random.getrandbits(256)) app.config["JWT_ACCESS_TOKEN_EXPIRES"] = ACCESS_EXPIRES app.config["JWT_REFRESH_TOKEN_EXPIRES"] = REFRESH_EXPIRES app.config["JWT_BLACKLIST_ENABLED"] = True app.config["JWT_BLACKLIST_TOKEN_CHECKS"] = ["access", "refresh"] token_blacklist = config.getRedisTokenConnection() app.config["RESTX_MASK_SWAGGER"] = False socketio = SocketIO(app) Breadcrumbs(app=app) Bootstrap(app) jwt = JWTManager(app) @jwt.additional_claims_loader def add_claims_to_access_token(identity): return {"user": identity} @jwt.token_in_blocklist_loader def check_if_token_is_revoked(decrypted_token): jti = decrypted_token["jti"] entry = token_blacklist.get(jti) if entry == "true": return True return False login_manager.init_app(app) login_manager.login_message = "You must be logged in to access this page!!!" login_manager.login_view = "auth.login" @login_manager.user_loader def load_user(id): return User.get(id, auth_handler) from .home import home as home_blueprint app.register_blueprint(home_blueprint) from .plugins import plugins as plugins_blueprint app.register_blueprint(plugins_blueprint, url_prefix="/cvesearch/plugin") if not app.config["WebInterface"]: from .auth import auth as auth_blueprint app.register_blueprint(auth_blueprint) from .admin import admin as admin_blueprint app.register_blueprint(admin_blueprint, url_prefix="/cvesearch/admin") from .restapi import blueprint as api app.register_blueprint(api) from .restapidocs import docs as docs_blueprint app.register_blueprint(docs_blueprint) @app.context_processor def version(): def get_version(): return app.config["version"] return dict(get_version=get_version) @app.context_processor def db_schema(): def db_schema(): sc = SchemaChecker() try: return sc.validate_schema() except DatabaseSchemaError as err: return err return dict(db_schema=db_schema) @app.context_processor def WebInterface(): def get_WebInterface(): return app.config["WebInterface"] return dict(get_WebInterface=get_WebInterface) @app.context_processor def JSON2HTMLTable(): # Doublequote, because we have to |safe the content for the tags def doublequote(data): return urllib.parse.quote_plus(urllib.parse.quote_plus(data)) def JSON2HTMLTableFilter(data, stack=None): _return = "" if type(stack) == str: stack = [stack] if type(data) == list: if len(data) == 1: _return += JSON2HTMLTableFilter(data[0], stack) else: _return += '<ul class="via4">' for item in data: _return += "<li>%s</li>" % JSON2HTMLTableFilter(item, stack) _return += "</ul>" elif type(data) == dict: _return += '<table class="invisiTable">' for key, val in sorted(data.items()): _return += "<tr><td><b>%s</b></td><td>%s</td></tr>" % ( key, JSON2HTMLTableFilter(val, stack + [key]), ) _return += "</table>" elif type(data) == str: if stack: _return += ( "<a href='/link/" + doublequote(".".join(stack)) + "/" + doublequote(data) + "'>" ) # link opening _return += "<i class='fas fa-link' aria-hidden='true'></i> </a>" _return += ( "<a target='_blank' href='%s'>%s</a>" % (data, data) if isURL(data) else data ) _return += "" return _return return dict(JSON2HTMLTable=JSON2HTMLTableFilter) @app.template_filter("htmlEncode") def htmlEncode(string): return urllib.parse.quote_plus(string).lower() @app.template_filter("htmlDecode") def htmlDecode(string): return urllib.parse.unquote_plus(string) @app.template_filter("sortIntLikeStr") def sortIntLikeStr(datalist): return sorted(datalist, key=lambda k: int(k)) @app.errorhandler(404) def page_not_found(error): return ( render_template("404.html",), 404, ) return app, socketio
class API(): app = Flask(__name__, static_folder='static', static_url_path='/static') app.config['MONGO_DBNAME'] = Configuration.getMongoDB() app.config['SECRET_KEY'] = str(random.getrandbits(256)) def __init__(self): routes = [{ 'r': '/api/cpe2.3/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe23 }, { 'r': '/api/cpe2.2/<path:cpe>', 'm': ['GET'], 'f': self.api_cpe22 }, { 'r': '/api/cvefor/<path:cpe>', 'm': ['GET'], 'f': self.api_cvesFor }, { 'r': '/api/cve/<cveid>', 'm': ['GET'], 'f': self.api_cve }, { 'r': '/api/last', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/last/', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/last/<int:limit>', 'm': ['GET'], 'f': self.api_last }, { 'r': '/api/browse', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/browse/', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/browse/<vendor>', 'm': ['GET'], 'f': self.api_browse }, { 'r': '/api/search/<vendor>/<path:product>', 'm': ['GET'], 'f': self.api_search }, { 'r': '/api/dbInfo', 'm': ['GET'], 'f': self.api_dbInfo }] for route in routes: self.addRoute(route) def addRoute(self, route): self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m']) ############# # Decorator # ############# def api(funct): @wraps(funct) def api_wrapper(*args, **kwargs): data = funct(*args, **kwargs) try: if request.url_rule and type(data) in [dict, list]: data = json.dumps(data, default=json_util.default) return data except: return data return api_wrapper ########## # ROUTES # ########## # /api/cpe2.3/<cpe> def api_cpe23(self, cpe): cpe = tk.toStringFormattedCPE(cpe) return cpe if cpe else "None" # /api/cpe2.2/<cpe> def api_cpe22(self, cpe): cpe = tk.toOldCPE(cpe) return cpe if cpe else "None" # /api/cvefor/<cpe> @api def api_cvesFor(self, cpe): cpe = urllib.parse.unquote_plus(cpe) cves = query.cvesForCPE(cpe) return cves # /api/cve/<cveid> @api def api_cve(self, cveid): cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.getcve(cveid=cveid.upper()) if cve is None: cve = {} return cve # /api/last # /api/last/ # /api/last/<limit> @api def api_last(self, limit=None): limit = limit if limit else 30 cvesp = cves.last(rankinglookup=True, namelookup=True, via4lookup=True, capeclookup=True) cve = cvesp.get(limit=limit) return cve # /api/browse # /api/browse/ # /api/browse/<vendor> @api def api_browse(self, vendor=None): if vendor: vendor = urllib.parse.quote_plus(vendor).lower() browseList = query.getBrowseList(vendor) if isinstance(browseList, dict): return browseList else: return {} # /api/search/<vendor>/<path:product> @api def api_search(self, vendor=None, product=None): if vendor is None or product is None: return jsonify({}) search = vendor + ":" + product # Not using query.cvesForCPE, because that one gives too much info #return json.dumps(db.cvesForCPE(search), default=json_util.default) return db.cvesForCPE(search) # /api/dbInfo @api def api_dbInfo(self): return db.getDBStats() ######################## # Web Server Functions # ######################## # signal handlers def sig_handler(self, sig, frame): print('Caught signal: %s' % sig) IOLoop.instance().add_callback(self.shutdown) def shutdown(self): MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3 print('Stopping http server') self.http_server.stop() print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN) io_loop = IOLoop.instance() deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN def stop_loop(): now = time.time() if now < deadline and (io_loop._callbacks or io_loop._timeouts): io_loop.add_timeout(now + 1, stop_loop) else: io_loop.stop() print('Shutdown') stop_loop() def start(self): # get properties flaskHost = Configuration.getFlaskHost() flaskPort = Configuration.getFlaskPort() flaskDebug = Configuration.getFlaskDebug() # logging if Configuration.getLogging(): logfile = Configuration.getLogfile() pathToLog = logfile.rsplit('/', 1)[0] if not os.path.exists(pathToLog): os.makedirs(pathToLog) maxLogSize = Configuration.getMaxLogSize() backlog = Configuration.getBacklog() file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog) file_handler.setLevel(logging.ERROR) formatter = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) self.app.logger.addHandler(file_handler) if flaskDebug: # start debug flask server self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug) else: # start asynchronous server using tornado wrapper for flask # ssl connection print("Server starting...") if Configuration.useSSL(): ssl_options = { "certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()), "keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey()) } else: ssl_options = None signal.signal(signal.SIGTERM, self.sig_handler) signal.signal(signal.SIGINT, self.sig_handler) self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options) self.http_server.bind(flaskPort, address=flaskHost) self.http_server.start(0) # Forks multiple sub-processes IOLoop.instance().start()