def adminStats():
cveU = db.info.find_one({"db": "cve"})
cpeU = db.info.find_one({"db": "cpe"})
cpeOtherU = db.info.find_one({"db": "cpeother"})
capecU = db.info.find_one({"db": "capec"})
d2secU = db.info.find_one({"db": "d2sec"})
vendorU = db.info.find_one({"db": "vendor"})
vfeedU = db.info.find_one({"db": "vfeed"})
stats = {
"cveA": db.cves.count(),
"cveU": cveU["last-modified"] if cveU is not None else None,
"cpeA": db.cpe.count(),
"cpeU": cpeU["last-modified"] if cpeU is not None else None,
"cpeOtherA": db.cpeother.count(),
"cpeOtherU": cpeOtherU["last-modified"] if cpeOtherU is not None else None,
"capecA": db.capec.count(),
"capecU": capecU["last-modified"] if capecU is not None else None,
"d2secA": db.d2sec.count(),
"d2secU": d2secU["last-modified"] if d2secU is not None else None,
"vendorA": db.vendor.count(),
"vendorU": vendorU["last-modified"] if vendorU is not None else None,
"vfeedA": db.vfeed.count(),
"vfeedU": vfeedU["last-modified"] if vfeedU is not None else None,
"blA": db.mgmt_blacklist.count(),
"wlA": db.mgmt_whitelist.count(),
"dbName": Configuration.getMongoDB(),
"dbSize": db.command("dbstats")["dataSize"],
"dbOnDisk": db.command("dbstats")["storageSize"],
}
return stats
def getVersionsOfProduct(product):
p = redisdb.smembers("p:" + product)
return sorted(list(p))
cveU = db.info.find_one({'db': 'cve'})
cpeU = db.info.find_one({'db': 'cpe'})
cpeOtherU = db.info.find_one({'db': 'cpeother'})
capecU = db.info.find_one({'db': 'capec'})
d2secU = db.info.find_one({'db': 'd2sec'})
vendorU = db.info.find_one({'db': 'vendor'})
vfeedU = db.info.find_one({'db': 'vfeed'})
stats = {
'cveA': db.cves.count(),
'cveU': cveU['last-modified'] if cveU is not None else None,
'cpeA': db.cpe.count(),
'cpeU': cpeU['last-modified'] if cpeU is not None else None,
'cpeOtherA': db.cpeother.count(),
'cpeOtherU':
cpeOtherU['last-modified'] if cpeOtherU is not None else None,
'capecA': db.capec.count(),
'capecU': capecU['last-modified'] if capecU is not None else None,
'd2secA': db.d2sec.count(),
'd2secU': d2secU['last-modified'] if d2secU is not None else None,
'vendorA': db.vendor.count(),
'vendorU': vendorU['last-modified'] if vendorU is not None else None,
'vfeedA': db.vfeed.count(),
'vfeedU': vfeedU['last-modified'] if vfeedU is not None else None,
'blA': db.mgmt_blacklist.count(),
'wlA': db.mgmt_whitelist.count(),
'dbName': Configuration.getMongoDB(),
'dbSize': db.command("dbstats")['dataSize'],
'dbOnDisk': db.command("dbstats")['storageSize']
}
return stats
def getDBStats(include_admin=False):
data = {
'cves': {},
'cpe': {},
'cpeOther': {},
'capec': {},
'cwe': {},
'via4': {}
}
for key in data.keys():
data[key] = {
'size': getSize(key.lower()),
'last_update': getLastModified(key.lower())
}
if include_admin:
data['whitelist'] = {'size': colWHITELIST.count()}
data['blacklist'] = {'size': colBLACKLIST.count()}
data = {
'stats': {
'size_on_disk': db.command("dbstats")['storageSize'],
'db_size': db.command('dbstats')['dataSize'],
'name': conf.getMongoDB()
},
'data': data
}
return data
def getDBStats(include_admin=False):
data = {
"cves": {},
"cpe": {},
"cpeOther": {},
"capec": {},
"cwe": {},
"via4": {}
}
for key in data.keys():
data[key] = {
"size": getSize(key.lower()),
"last_update": getLastModified(key.lower()),
}
if include_admin:
data["whitelist"] = {"size": colWHITELIST.count()}
data["blacklist"] = {"size": colBLACKLIST.count()}
data = {
"stats": {
"size_on_disk": db.command("dbstats")["storageSize"],
"db_size": db.command("dbstats")["dataSize"],
"name": conf.getMongoDB(),
},
"data": data,
}
return data
def getDBStats():
cols = ["cve", "cpe", "cpeOther", "capec", "d2sec", "vendor", "vfeed"]
stats = {x + "A": getSize(x.lower()) for x in cols}
stats["cveA"] = getSize("cves")
stats.update({x + "U": getLastModified(x.lower()) for x in cols})
stats.update({"blA": colBLACKLIST.count(), "wlA": colWHITELIST.count()})
stats.update({"dbOnDisk": db.command("dbstats")["storageSize"], "dbSize": db.command("dbstats")["dataSize"]})
stats["dbName"] = conf.getMongoDB()
return stats
def getDBStats():
cols=['cve', 'cpe', 'cpeOther', 'capec', 'd2sec', 'vendor']
stats={x+'A': getSize(x.lower()) for x in cols}
stats['cveA']=getSize('cves')
stats.update({x+'U': getLastModified(x.lower()) for x in cols})
stats.update({'blA': colBLACKLIST.count(), 'wlA':colWHITELIST.count()})
stats.update({'dbOnDisk': db.command("dbstats")['storageSize'], 'dbSize':db.command('dbstats')['dataSize']})
stats['dbName']=conf.getMongoDB()
return stats
def getDBStats(include_admin=False):
data={'cves': {}, 'cpe': {}, 'cpeOther': {}, 'capec': {}, 'cwe': {}, 'via4': {}}
for key in data.keys():
data[key] = {'size': getSize(key.lower()),
'last_update': getLastModified(key.lower())}
if include_admin:
data['whitelist']={'size': colWHITELIST.count()}
data['blacklist']={'size': colBLACKLIST.count()}
data = {'stats': {'size_on_disk': db.command("dbstats")['storageSize'],
'db_size': db.command('dbstats')['dataSize'],
'name': conf.getMongoDB()},
'data': data}
return data
def adminStats():
cveU = db.info.find_one({'db': 'cve'})
cpeU = db.info.find_one({'db': 'cpe'})
cpeOtherU = db.info.find_one({'db': 'cpeother'})
capecU = db.info.find_one({'db': 'capec'})
d2secU = db.info.find_one({'db': 'd2sec'})
vendorU = db.info.find_one({'db': 'vendor'})
vfeedU = db.info.find_one({'db': 'vfeed'})
stats = {'cveA': db.cves.count(), 'cveU': cveU['last-modified'] if cveU is not None else None,
'cpeA': db.cpe.count(), 'cpeU': cpeU['last-modified'] if cpeU is not None else None,
'cpeOtherA': db.cpeother.count(), 'cpeOtherU': cpeOtherU['last-modified'] if cpeOtherU is not None else None,
'capecA': db.capec.count(), 'capecU': capecU['last-modified'] if capecU is not None else None,
'd2secA': db.d2sec.count(), 'd2secU': d2secU['last-modified'] if d2secU is not None else None,
'vendorA': db.vendor.count(), 'vendorU': vendorU['last-modified'] if vendorU is not None else None,
'vfeedA': db.vfeed.count(), 'vfeedU': vfeedU['last-modified'] if vfeedU is not None else None,
'blA': db.mgmt_blacklist.count(), 'wlA': db.mgmt_whitelist.count(),
'dbName': Configuration.getMongoDB(), 'dbSize': db.command("dbstats")['dataSize'],
'dbOnDisk': db.command("dbstats")['storageSize']}
return stats
def getDBStats(include_admin=False):
data = {
'cves': {},
'cpe': {},
'cpeOther': {},
'capec': {},
'cwe': {},
'via4': {}
}
for key in data.keys():
data[key] = {
'size': getSize(key.lower()),
'last_update': getLastModified(key.lower())
}
if include_admin:
data['whitelist'] = {
'size':
Configuration.getMongoConnection().get_collection(
Collections.WhiteList.value).count()
}
data['blacklist'] = {
'size':
Configuration.getMongoConnection().get_collection(
Collections.BlackList.value).count()
}
data = {
'stats': {
'size_on_disk':
Configuration.getMongoConnection().command(
"dbstats")['storageSize'],
'db_size':
Configuration.getMongoConnection().command('dbstats')
['dataSize'],
'name':
Configuration.getMongoDB()
},
'data': data
}
return data
from lib.User import User
from lib.Config import Configuration
from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat
import lib.CVEs as cves
import lib.DatabaseLayer as dbLayer
from sbin.db_whitelist import *
from sbin.db_blacklist import *
# parse command line arguments
argparser = argparse.ArgumentParser(description='Start CVE-Search web component')
argparser.add_argument('-v', action='store_true', help='verbose output')
args = argparser.parse_args()
# variables
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
pageLength = Configuration.getPageLength()
# login manager
login_manager = LoginManager()
login_manager.init_app(app)
# db connectors
redisdb = Configuration.getRedisVendorConnection()
# functions
def getBrowseList(vendor):
result = {}
if (vendor is None) or type(vendor) == list:
v1 = redisdb.smembers("t:/o")
v2 = redisdb.smembers("t:/a")
from logging.handlers import RotatingFileHandler
from lib.Config import Configuration
from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat
import lib.CVEs as cves
import lib.DatabaseLayer as dbLayer
# parse command line arguments
argparser = argparse.ArgumentParser(
description='Start CVE-Search web component')
argparser.add_argument('-v', action='store_true', help='verbose output')
args = argparser.parse_args()
# variables
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
pageLength = Configuration.getPageLength()
# db connectors
redisdb = Configuration.getRedisVendorConnection()
# functions
def getBrowseList(vendor):
result = {}
if (vendor is None) or type(vendor) == list:
v1 = redisdb.smembers("t:/o")
v2 = redisdb.smembers("t:/a")
v3 = redisdb.smembers("t:/h")
vendor = sorted(list(set(list(v1) + list(v2) + list(v3))))
from collections import defaultdict
import bson
import pymongo
from pymongo import DESCENDING, ASCENDING
from pymongo.collection import Collection
from werkzeug.security import generate_password_hash, check_password_hash
from lib.DatabaseLayer import sanitize
from lib.DatabasePluginBase import DatabasePluginBase
from lib.Config import Configuration
config = Configuration()
HOST = config.readSetting("Database", "Host", config.default["mongoHost"])
PORT = config.readSetting("Database", "Port", config.default["mongoPort"])
DATABASE = config.getMongoDB()
USERNAME = urllib.parse.quote(
config.readSetting("Database", "Username",
config.default["mongoUsername"]))
PASSWORD = urllib.parse.quote(
config.readSetting("Database", "Password",
config.default["mongoPassword"]))
exits = {
"userInDb": "User already exists in database",
"userNotInDb": "User does not exist in database",
"userpasscombo": "Master user/password combination does not exist",
"passwordMatch": "The passwords don't match!",
"noMaster": "Not a master account!",
"lastMaster":
"This user is the last admin in the database and thus can not be removed",
from lib.User import User
from lib.Config import Configuration
from lib.Toolkit import toStringFormattedCPE, toOldCPE, currentTime, isURL, vFeedName, convertDateToDBFormat
import lib.CVEs as cves
import lib.DatabaseLayer as db
from sbin.db_whitelist import *
from sbin.db_blacklist import *
# parse command line arguments
argparser = argparse.ArgumentParser(description="Start CVE-Search web component")
argparser.add_argument("-v", action="store_true", help="verbose output")
args = argparser.parse_args()
# variables
app = Flask(__name__, static_folder="static", static_url_path="/static")
app.config["MONGO_DBNAME"] = Configuration.getMongoDB()
app.config["SECRET_KEY"] = str(random.getrandbits(256))
pageLength = Configuration.getPageLength()
# login manager
login_manager = LoginManager()
login_manager.init_app(app)
# db connectors
redisdb = Configuration.getRedisVendorConnection()
# functions
def getBrowseList(vendor):
result = {}
if (vendor is None) or type(vendor) == list:
v1 = redisdb.smembers("t:/o")
v2 = redisdb.smembers("t:/a")
class API():
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
def __init__(self):
routes = [{
'r': '/api/',
'm': ['GET'],
'f': self.api_documentation
}, {
'r': '/api/cpe2.3/<path:cpe>',
'm': ['GET'],
'f': self.api_cpe23
}, {
'r': '/api/cpe2.2/<path:cpe>',
'm': ['GET'],
'f': self.api_cpe22
}, {
'r': '/api/cvefor/<path:cpe>',
'm': ['GET'],
'f': self.api_cvesFor
}, {
'r': '/api/cve/<cveid>',
'm': ['GET'],
'f': self.api_cve
}, {
'r': '/api/cwe',
'm': ['GET'],
'f': self.api_cwe
}, {
'r': '/api/cwe/<int:cwe_id>',
'm': ['GET'],
'f': self.api_cwe
}, {
'r': '/api/capec/<cweid>',
'm': ['GET'],
'f': self.api_capec
}, {
'r': '/api/last',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/last/',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/last/<int:limit>',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/query',
'm': ['GET'],
'f': self.api_query
}, {
'r': '/api/browse',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/browse/',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/browse/<path:vendor>',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/search/<vendor>/<path:product>',
'm': ['GET'],
'f': self.api_search
}, {
'r': '/api/search/<path:search>',
'm': ['GET'],
'f': self.api_text_search
}, {
'r': '/api/search/days/<days>/<path:search>',
'm': ['GET'],
'f': self.api_text_limitday_search
}, {
'r': '/api/link/<key>/<value>',
'm': ['GET'],
'f': self.api_link
}, {
'r': '/api/dbInfo',
'm': ['GET'],
'f': self.api_dbInfo
}]
for route in routes:
self.addRoute(route)
def addRoute(self, route):
self.app.add_url_rule(route['r'],
view_func=route['f'],
methods=route['m'])
#############
# Decorator #
#############
def api(funct):
@wraps(funct)
def api_wrapper(*args, **kwargs):
data = error = None
# Get data (and possibly errors)
try:
data = funct(*args, **kwargs)
except APIError as e:
error = ({'status': 'error', 'reason': e.message}, e.status)
except Exception as e:
print(e)
error = ({
'status': 'error',
'reason': 'Internal server error'
}, 500)
# Check if data should be returned as html or data
try:
returnType = 'application/json'
if (request.url_rule.rule.lower().startswith("/api/")
or request.url_rule.rule.lower().endswith(".json")):
# Support JSONP
if request.args.get('callback', False):
data = "%s(%s)" % (request.args.get('callback'), data)
# Check API version for backwards compatibility. We'll call the old API v1.0
elif request.headers.get('Version') in ['1.1']:
# Get the requested return type
returnType = request.headers.get('Accept', '*/*')
# Default to JSON
if any(t in returnType for t in
['json', 'application/*', 'text/*', '*/*']):
data = error if error else {
'status': 'success',
'data': data
}
elif 'plain' in returnType:
pass # No need to do anything, but needs to be accepted
else:
data = ({
'status': 'error',
'reason': 'Unknown Content-type requested'
}, 415)
returnType = 'application/json'
if type(data) is not str:
if type(data) is tuple:
data = list(data)
data[0] = json.dumps(convertDatetime(dct=data[0]),
indent=4,
sort_keys=True,
default=json_util.default)
else:
data = (json.dumps(convertDatetime(dct=data),
indent=4,
sort_keys=True,
default=json_util.default), 200)
return Response(data[0], mimetype=returnType), data[1]
except Exception as e:
print(e)
pass
if error and error[1] == 500: raise (APIError(error[0]['reason']))
return data
return api_wrapper
#############
# FUNCTIONS #
#############
def generate_minimal_query(self, f):
query = []
# retrieving lists
if f['rejectedSelect'] == "hide":
exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
query.append({'summary': re.compile(exp)})
# cvss logic
if f['cvssSelect'] == "above":
query.append({'cvss': {'$gt': float(f['cvss'])}})
elif f['cvssSelect'] == "equals":
query.append({'cvss': float(f['cvss'])})
elif f['cvssSelect'] == "below":
query.append({'cvss': {'$lt': float(f['cvss'])}})
# date logic
if f['timeSelect'] != "all":
if f['startDate']:
startDate = parse_datetime(f['startDate'],
ignoretz=True,
dayfirst=True)
if f['endDate']:
endDate = parse_datetime(f['endDate'],
ignoretz=True,
dayfirst=True)
if f['timeSelect'] == "from":
query.append({f['timeTypeSelect']: {'$gt': startDate}})
elif f['timeSelect'] == "until":
query.append({f['timeTypeSelect']: {'$lt': endDate}})
elif f['timeSelect'] == "between":
query.append(
{f['timeTypeSelect']: {
'$gt': startDate,
'$lt': endDate
}})
elif f['timeSelect'] == "outside":
query.append({
'$or': [{
f['timeTypeSelect']: {
'$lt': startDate
}
}, {
f['timeTypeSelect']: {
'$gt': endDate
}
}]
})
return query
def filter_logic(self, filters, skip, limit=None):
query = self.generate_minimal_query(filters)
limit = limit if limit else self.args['pageLength']
return db.getCVEs(limit=limit, skip=skip, query=query)
##########
# ROUTES #
##########
# /api
def api_documentation(self):
return render_template('api.html')
# /api/cpe2.3/<cpe>
@api
def api_cpe23(self, cpe):
cpe = tk.toStringFormattedCPE(cpe)
return cpe if cpe else "None"
# /api/cpe2.2/<cpe>
@api
def api_cpe22(self, cpe):
cpe = tk.toOldCPE(cpe)
return cpe if cpe else "None"
# /api/cvefor/<cpe>
@api
def api_cvesFor(self, cpe):
cpe = urllib.parse.unquote_plus(cpe)
return query.cvesForCPE(cpe)
# /api/cve/<cveid>
@api
def api_cve(self, cveid):
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.getcve(cveid=cveid.upper())
if not cve: raise (APIError('cve not found', 404))
return cve
# /api/cwe
# /api/cwe/<cwe_id>
@api
def api_cwe(self, cwe_id=None):
return db.getCAPECFor(str(cwe_id)) if cwe_id else db.getCWEs()
# /api/capec/<cweid>
@api
def api_capec(self, cweid):
return db.getCAPEC(cweid)
# /api/last
# /api/last/
# /api/last/<limit>
@api
def api_last(self, limit=None):
limit = limit if limit else 30
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.get(limit=limit)
return cve
# /query
@api
def api_query(self):
f = {
'rejectedSelect': request.headers.get('rejected'),
'cvss': request.headers.get('cvss_score'),
'cvssSelect': request.headers.get('cvss_modifier'),
'startDate': request.headers.get('time_start'),
'endDate': request.headers.get('time_end'),
'timeSelect': request.headers.get('time_modifier'),
'timeTypeSelect': request.headers.get('time_type'),
'skip': request.headers.get('skip'),
'limit': request.headers.get('limit')
}
try:
skip = int(f['skip']) if f['skip'] else 0
except:
raise (APIError('skip must be an int', 400))
try:
limit = int(f['limit']) if f['limit'] else 0
except:
raise (APIError('limit must be an int', 400))
return self.filter_logic(f, skip, limit)
# /api/browse
# /api/browse/
# /api/browse/<vendor>
@api
def api_browse(self, vendor=None):
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
try:
browseList = query.getBrowseList(vendor)
except redisExceptions.ConnectionError:
raise (APIError(
"Server could not connect to the browsing repository", 503))
if isinstance(browseList, dict):
return browseList
else:
return {}
# /api/search/<vendor>/<path:product>
@api
def api_search(self, vendor=None, product=None):
if not (vendor and product): return {}
search = vendor + ":" + product
# Not using query.cvesForCPE, because that one gives too much info
#return json.dumps(db.cvesForCPE(search), default=json_util.default)
return db.cvesForCPE(search)
# /api/search/<path:search>
@api
def api_text_search(self, search=None):
return db.getSearchResults(search)
# /api/search/days/<days>/<path:search>
@api
def api_text_limitday_search(self, days=None, search=None):
if not (search or days): return {}
cve_d = []
date_n_days_ago = datetime.datetime.now() - timedelta(int(days))
result = db.getSearchResults(search)
cve = result['data']
for item in cve:
if item['Modified'] > date_n_days_ago:
cve_d.append(item)
return cve_d
# /api/link/<key>/<value>
@api
def api_link(self, key=None, value=None):
key = self.htmlDecode(key)
value = self.htmlDecode(value)
regex = re.compile(re.escape(value), re.I)
data = {'cves': db.via4Linked(key, regex)}
cvssList = [float(x['cvss']) for x in data['cves'] if x.get('cvss')]
if cvssList:
data['stats'] = {
'maxCVSS': max(cvssList),
'minCVSS': min(cvssList),
'count': len(data['cves'])
}
else:
data['stats'] = {
'maxCVSS': 0,
'minCVSS': 0,
'count': len(data['cves'])
}
return data
# /api/dbInfo
@api
def api_dbInfo(self):
return db.getDBStats()
########################
# Web Server Functions #
########################
# signal handlers
def sig_handler(self, sig, frame):
print('Caught signal: %s' % sig)
IOLoop.instance().add_callback(self.shutdown)
def shutdown(self):
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
print('Stopping http server')
self.http_server.stop()
print('Will shutdown in %s seconds ...' %
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = IOLoop.instance()
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
def start(self):
# get properties
flaskHost = Configuration.getFlaskHost()
flaskPort = Configuration.getFlaskPort()
flaskDebug = Configuration.getFlaskDebug()
# logging
if Configuration.getLogging():
logfile = Configuration.getLogfile()
pathToLog = logfile.rsplit('/', 1)[0]
if not os.path.exists(pathToLog):
os.makedirs(pathToLog)
maxLogSize = Configuration.getMaxLogSize()
backlog = Configuration.getBacklog()
file_handler = RotatingFileHandler(logfile,
maxBytes=maxLogSize,
backupCount=backlog)
file_handler.setLevel(logging.ERROR)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.app.logger.addHandler(file_handler)
if flaskDebug:
# start debug flask server
self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
else:
# start asynchronous server using tornado wrapper for flask
# ssl connection
print("Server starting...")
if Configuration.useSSL():
ssl_options = {
"certfile":
os.path.join(_runPath, "../", Configuration.getSSLCert()),
"keyfile":
os.path.join(_runPath, "../", Configuration.getSSLKey())
}
else:
ssl_options = None
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
self.http_server = HTTPServer(WSGIContainer(self.app),
ssl_options=ssl_options)
self.http_server.bind(flaskPort, address=flaskHost)
self.http_server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
class API:
app = Flask(__name__, static_folder="static", static_url_path="/static")
app.config["MONGO_DBNAME"] = Configuration.getMongoDB()
app.config["SECRET_KEY"] = str(random.getrandbits(256))
def __init__(self):
routes = [
{
"r": "/api/",
"m": ["GET"],
"f": self.api_documentation
},
{
"r": "/api/cpe2.3/<path:cpe>",
"m": ["GET"],
"f": self.api_cpe23
},
{
"r": "/api/cpe2.2/<path:cpe>",
"m": ["GET"],
"f": self.api_cpe22
},
{
"r": "/api/cvefor/<path:cpe>",
"m": ["GET"],
"f": self.api_cvesFor
},
{
"r": "/api/cvefor/<path:cpe>/<limit>",
"m": ["GET"],
"f": self.api_cvesFor
},
{
"r": "/api/cve/<cveid>",
"m": ["GET"],
"f": self.api_cve
},
{
"r": "/api/cwe",
"m": ["GET"],
"f": self.api_cwe
},
{
"r": "/api/cwe/<cwe_id>",
"m": ["GET"],
"f": self.api_cwe
},
{
"r": "/api/capec/<cweid>",
"m": ["GET"],
"f": self.api_capec
},
{
"r": "/api/capec/show/<capecid>",
"m": ["GET"],
"f": self.api_capec_by_id
},
{
"r": "/api/last",
"m": ["GET"],
"f": self.api_last
},
{
"r": "/api/last/",
"m": ["GET"],
"f": self.api_last
},
{
"r": "/api/last/<int:limit>",
"m": ["GET"],
"f": self.api_last
},
{
"r": "/api/query",
"m": ["GET"],
"f": self.api_query
},
{
"r": "/api/browse",
"m": ["GET"],
"f": self.api_browse
},
{
"r": "/api/browse/",
"m": ["GET"],
"f": self.api_browse
},
{
"r": "/api/browse/<path:vendor>",
"m": ["GET"],
"f": self.api_browse
},
{
"r": "/api/search/<vendor>/<path:product>",
"m": ["GET"],
"f": self.api_search,
},
{
"r": "/api/search/<path:search>",
"m": ["GET"],
"f": self.api_text_search
},
{
"r": "/api/link/<key>/<value>",
"m": ["GET"],
"f": self.api_link
},
{
"r": "/api/dbInfo",
"m": ["GET"],
"f": self.api_dbInfo
},
]
for route in routes:
self.addRoute(route)
def addRoute(self, route):
self.app.add_url_rule(route["r"],
view_func=route["f"],
methods=route["m"])
#############
# Decorator #
#############
def api(funct):
@wraps(funct)
def api_wrapper(*args, **kwargs):
data = error = None
# Get data (and possibly errors)
try:
data = funct(*args, **kwargs)
except APIError as e:
error = ({"status": "error", "reason": e.message}, e.status)
except Exception as e:
print(e)
error = ({
"status": "error",
"reason": "Internal server error"
}, 500)
# Check if data should be returned as html or data
try:
returnType = "application/json"
if request.url_rule.rule.lower().startswith(
"/api/") or request.url_rule.rule.lower().endswith(
".json"):
# Support JSONP
if request.args.get("callback", False):
data = "%s(%s)" % (request.args.get("callback"), data)
# Check API version for backwards compatibility. We'll call the old API v1.0
elif request.headers.get("Version") in ["1.1"]:
# Get the requested return type
returnType = request.headers.get("Accept", "*/*")
# Default to JSON
if any(t in returnType for t in
["json", "application/*", "text/*", "*/*"]):
data = (error if error else {
"status": "success",
"data": data
})
elif "plain" in returnType:
pass # No need to do anything, but needs to be accepted
else:
data = (
{
"status": "error",
"reason": "Unknown Content-type requested",
},
415,
)
returnType = "application/json"
if type(data) is not str:
if type(data) is tuple:
data = list(data)
data[0] = json.dumps(
convertDatetime(dct=data[0]),
indent=4,
sort_keys=True,
default=json_util.default,
)
else:
data = (
json.dumps(
convertDatetime(dct=data),
indent=4,
sort_keys=True,
default=json_util.default,
),
200,
)
return Response(data[0], mimetype=returnType), data[1]
except Exception as e:
print(e)
pass
if error and error[1] == 500:
raise (APIError(error[0]["reason"]))
return data
return api_wrapper
#############
# FUNCTIONS #
#############
def generate_minimal_query(self, f):
query = []
# retrieving lists
if f["rejectedSelect"] == "hide":
query.append({
"summary":
re.compile(
r"^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
)
})
# cvss logic
if f["cvssSelect"] == "above":
query.append({"cvss": {"$gt": float(f["cvss"])}})
elif f["cvssSelect"] == "equals":
query.append({"cvss": float(f["cvss"])})
elif f["cvssSelect"] == "below":
query.append({"cvss": {"$lt": float(f["cvss"])}})
# date logic
if f["timeSelect"] != "all":
if f["startDate"]:
startDate = parse_datetime(f["startDate"],
ignoretz=True,
dayfirst=True)
if f["endDate"]:
endDate = parse_datetime(f["endDate"],
ignoretz=True,
dayfirst=True)
if f["timeSelect"] == "from":
query.append({f["timeTypeSelect"]: {"$gt": startDate}})
elif f["timeSelect"] == "until":
query.append({f["timeTypeSelect"]: {"$lt": endDate}})
elif f["timeSelect"] == "between":
query.append(
{f["timeTypeSelect"]: {
"$gt": startDate,
"$lt": endDate
}})
elif f["timeSelect"] == "outside":
query.append({
"$or": [
{
f["timeTypeSelect"]: {
"$lt": startDate
}
},
{
f["timeTypeSelect"]: {
"$gt": endDate
}
},
]
})
return query
def filter_logic(self, filters, skip, limit=None):
query = self.generate_minimal_query(filters)
limit = limit if limit else self.args["pageLength"]
return getCVEs(limit=limit, skip=skip, query=query)
##########
# ROUTES #
##########
# /api
def api_documentation(self):
return render_template("api.html")
# /api/cpe2.3/<cpe>
@api
def api_cpe23(self, cpe):
cpe = toStringFormattedCPE(cpe)
return cpe, 200 if cpe else "None", 404
# /api/cpe2.2/<cpe>
@api
def api_cpe22(self, cpe):
cpe = toOldCPE(cpe)
return cpe, 200 if cpe else "None", 404
# /api/cvefor/<cpe>
# /api/cvefor/<cpe>/<limit>
@api
def api_cvesFor(self, cpe, limit=0):
cpe = urllib.parse.unquote_plus(cpe)
return qcvesForCPE(cpe, int(limit))
# /api/cve/<cveid>
@api
def api_cve(self, cveid):
cvesp = CveHandler(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.getcve(cveid=cveid.upper())
if not cve:
raise (APIError("cve not found", 404))
return cve
# /api/cwe
# /api/cwe/<cwe_id>
@api
def api_cwe(self, cwe_id=None):
return getCWEs(cwe_id) if cwe_id else getCWEs()
# /api/capec/<cweid>
@api
def api_capec(self, cweid):
return getCAPECFor(cweid)
# /api/capec/show/<capecid>
@api
def api_capec_by_id(self, capecid):
return getCAPEC(capecid)
# /api/last
# /api/last/
# /api/last/<limit>
@api
def api_last(self, limit=None):
limit = limit if limit else 30
cvesp = CveHandler(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.get(limit=limit)
return cve
# /query
@api
def api_query(self):
f = {
"rejectedSelect": request.headers.get("rejected"),
"cvss": request.headers.get("cvss_score"),
"cvssSelect": request.headers.get("cvss_modifier"),
"startDate": request.headers.get("time_start"),
"endDate": request.headers.get("time_end"),
"timeSelect": request.headers.get("time_modifier"),
"timeTypeSelect": request.headers.get("time_type"),
"skip": request.headers.get("skip"),
"limit": request.headers.get("limit"),
}
try:
skip = int(f["skip"]) if f["skip"] else 0
except:
raise (APIError("skip must be an int", 400))
try:
limit = int(f["limit"]) if f["limit"] else 0
except:
raise (APIError("limit must be an int", 400))
return self.filter_logic(f, skip, limit)
# /api/browse
# /api/browse/
# /api/browse/<vendor>
@api
def api_browse(self, vendor=None):
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
try:
browseList = getBrowseList(vendor)
except redis_connection_error:
raise (APIError(
"Server could not connect to the browsing repository", 503))
if isinstance(browseList, dict):
return browseList
else:
return {}
# /api/search/<vendor>/<path:product>
@api
def api_search(self, vendor=None, product=None):
if not (vendor and product):
return {}
search = vendor + ":" + product
# Not using query.cvesForCPE, because that one gives too much info
# return json.dumps(db.cvesForCPE(search), default=json_util.default)
return cvesForCPE(search)
# /api/search/<path:search>
@api
def api_text_search(self, search=None):
return getSearchResults(search)
# /api/link/<key>/<value>
@api
def api_link(self, key=None, value=None):
key = self.htmlDecode(key)
value = self.htmlDecode(value)
regex = re.compile(re.escape(value), re.I)
data = {"cves": via4Linked(key, regex)}
cvssList = [float(x["cvss"]) for x in data["cves"] if x.get("cvss")]
if cvssList:
data["stats"] = {
"maxCVSS": max(cvssList),
"minCVSS": min(cvssList),
"count": len(data["cves"]),
}
else:
data["stats"] = {
"maxCVSS": 0,
"minCVSS": 0,
"count": len(data["cves"])
}
return data
# /api/dbInfo
@api
def api_dbInfo(self):
return getDBStats()
########################
# Web Server Functions #
########################
# signal handlers
def sig_handler(self, sig, frame):
print("Caught signal: %s" % sig)
IOLoop.instance().add_callback(self.shutdown)
def shutdown(self):
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
print("Stopping http server")
self.http_server.stop()
print("Will shutdown in %s seconds ..." %
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = IOLoop.instance()
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print("Shutdown")
stop_loop()
def start(self):
# get properties
flaskHost = Configuration.getFlaskHost()
flaskPort = Configuration.getFlaskPort()
flaskDebug = Configuration.getFlaskDebug()
# logging
if Configuration.getLogging():
logfile = Configuration.getLogfile()
pathToLog = logfile.rsplit("/", 1)[0]
if not os.path.exists(pathToLog):
os.makedirs(pathToLog)
maxLogSize = Configuration.getMaxLogSize()
backlog = Configuration.getBacklog()
file_handler = RotatingFileHandler(logfile,
maxBytes=maxLogSize,
backupCount=backlog)
file_handler.setLevel(logging.ERROR)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.app.logger.addHandler(file_handler)
if flaskDebug:
# start debug flask server
self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
else:
# start asynchronous server using tornado wrapper for flask
# ssl connection
print("Server starting...")
if Configuration.useSSL():
ssl_options = {
"certfile":
os.path.join(_runPath, "../", Configuration.getSSLCert()),
"keyfile":
os.path.join(_runPath, "../", Configuration.getSSLKey()),
}
else:
ssl_options = None
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
self.http_server = HTTPServer(WSGIContainer(self.app),
ssl_options=ssl_options)
self.http_server.bind(flaskPort, address=flaskHost)
self.http_server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
def create_app(version, run_path):
global app, token_blacklist, socketio
app = Flask(__name__, static_url_path="/cvesearch/static", static_folder="static")
cors = CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'
app.config["version"] = version
app.config["run_path"] = run_path
app.config["APPLICATION_ROOT"] = "/cvesearch"
config = Configuration()
if config.getWebInterface().lower() == "full":
app.config["WebInterface"] = False
else:
app.config["WebInterface"] = True
app.config["MONGO_DBNAME"] = config.getMongoDB()
app.config["SECRET_KEY"] = str(random.getrandbits(256))
app.config["JWT_SECRET_KEY"] = str(random.getrandbits(256))
app.config["JWT_ACCESS_TOKEN_EXPIRES"] = ACCESS_EXPIRES
app.config["JWT_REFRESH_TOKEN_EXPIRES"] = REFRESH_EXPIRES
app.config["JWT_BLACKLIST_ENABLED"] = True
app.config["JWT_BLACKLIST_TOKEN_CHECKS"] = ["access", "refresh"]
token_blacklist = config.getRedisTokenConnection()
app.config["RESTX_MASK_SWAGGER"] = False
socketio = SocketIO(app)
Breadcrumbs(app=app)
Bootstrap(app)
jwt = JWTManager(app)
@jwt.additional_claims_loader
def add_claims_to_access_token(identity):
return {"user": identity}
@jwt.token_in_blocklist_loader
def check_if_token_is_revoked(decrypted_token):
jti = decrypted_token["jti"]
entry = token_blacklist.get(jti)
if entry == "true":
return True
return False
login_manager.init_app(app)
login_manager.login_message = "You must be logged in to access this page!!!"
login_manager.login_view = "auth.login"
@login_manager.user_loader
def load_user(id):
return User.get(id, auth_handler)
from .home import home as home_blueprint
app.register_blueprint(home_blueprint)
from .plugins import plugins as plugins_blueprint
app.register_blueprint(plugins_blueprint, url_prefix="/cvesearch/plugin")
if not app.config["WebInterface"]:
from .auth import auth as auth_blueprint
app.register_blueprint(auth_blueprint)
from .admin import admin as admin_blueprint
app.register_blueprint(admin_blueprint, url_prefix="/cvesearch/admin")
from .restapi import blueprint as api
app.register_blueprint(api)
from .restapidocs import docs as docs_blueprint
app.register_blueprint(docs_blueprint)
@app.context_processor
def version():
def get_version():
return app.config["version"]
return dict(get_version=get_version)
@app.context_processor
def db_schema():
def db_schema():
sc = SchemaChecker()
try:
return sc.validate_schema()
except DatabaseSchemaError as err:
return err
return dict(db_schema=db_schema)
@app.context_processor
def WebInterface():
def get_WebInterface():
return app.config["WebInterface"]
return dict(get_WebInterface=get_WebInterface)
@app.context_processor
def JSON2HTMLTable():
# Doublequote, because we have to |safe the content for the tags
def doublequote(data):
return urllib.parse.quote_plus(urllib.parse.quote_plus(data))
def JSON2HTMLTableFilter(data, stack=None):
_return = ""
if type(stack) == str:
stack = [stack]
if type(data) == list:
if len(data) == 1:
_return += JSON2HTMLTableFilter(data[0], stack)
else:
_return += '<ul class="via4">'
for item in data:
_return += "<li>%s</li>" % JSON2HTMLTableFilter(item, stack)
_return += "</ul>"
elif type(data) == dict:
_return += '<table class="invisiTable">'
for key, val in sorted(data.items()):
_return += "<tr><td><b>%s</b></td><td>%s</td></tr>" % (
key,
JSON2HTMLTableFilter(val, stack + [key]),
)
_return += "</table>"
elif type(data) == str:
if stack:
_return += (
"<a href='/link/"
+ doublequote(".".join(stack))
+ "/"
+ doublequote(data)
+ "'>"
) # link opening
_return += "<i class='fas fa-link' aria-hidden='true'></i> </a>"
_return += (
"<a target='_blank' href='%s'>%s</a>" % (data, data)
if isURL(data)
else data
)
_return += ""
return _return
return dict(JSON2HTMLTable=JSON2HTMLTableFilter)
@app.template_filter("htmlEncode")
def htmlEncode(string):
return urllib.parse.quote_plus(string).lower()
@app.template_filter("htmlDecode")
def htmlDecode(string):
return urllib.parse.unquote_plus(string)
@app.template_filter("sortIntLikeStr")
def sortIntLikeStr(datalist):
return sorted(datalist, key=lambda k: int(k))
@app.errorhandler(404)
def page_not_found(error):
return (
render_template("404.html",),
404,
)
return app, socketio
class API():
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getMongoDB()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
def __init__(self):
routes = [{
'r': '/api/cpe2.3/<path:cpe>',
'm': ['GET'],
'f': self.api_cpe23
}, {
'r': '/api/cpe2.2/<path:cpe>',
'm': ['GET'],
'f': self.api_cpe22
}, {
'r': '/api/cvefor/<path:cpe>',
'm': ['GET'],
'f': self.api_cvesFor
}, {
'r': '/api/cve/<cveid>',
'm': ['GET'],
'f': self.api_cve
}, {
'r': '/api/last',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/last/',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/last/<int:limit>',
'm': ['GET'],
'f': self.api_last
}, {
'r': '/api/browse',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/browse/',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/browse/<vendor>',
'm': ['GET'],
'f': self.api_browse
}, {
'r': '/api/search/<vendor>/<path:product>',
'm': ['GET'],
'f': self.api_search
}, {
'r': '/api/dbInfo',
'm': ['GET'],
'f': self.api_dbInfo
}]
for route in routes:
self.addRoute(route)
def addRoute(self, route):
self.app.add_url_rule(route['r'],
view_func=route['f'],
methods=route['m'])
#############
# Decorator #
#############
def api(funct):
@wraps(funct)
def api_wrapper(*args, **kwargs):
data = funct(*args, **kwargs)
try:
if request.url_rule and type(data) in [dict, list]:
data = json.dumps(data, default=json_util.default)
return data
except:
return data
return api_wrapper
##########
# ROUTES #
##########
# /api/cpe2.3/<cpe>
def api_cpe23(self, cpe):
cpe = tk.toStringFormattedCPE(cpe)
return cpe if cpe else "None"
# /api/cpe2.2/<cpe>
def api_cpe22(self, cpe):
cpe = tk.toOldCPE(cpe)
return cpe if cpe else "None"
# /api/cvefor/<cpe>
@api
def api_cvesFor(self, cpe):
cpe = urllib.parse.unquote_plus(cpe)
cves = query.cvesForCPE(cpe)
return cves
# /api/cve/<cveid>
@api
def api_cve(self, cveid):
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.getcve(cveid=cveid.upper())
if cve is None: cve = {}
return cve
# /api/last
# /api/last/
# /api/last/<limit>
@api
def api_last(self, limit=None):
limit = limit if limit else 30
cvesp = cves.last(rankinglookup=True,
namelookup=True,
via4lookup=True,
capeclookup=True)
cve = cvesp.get(limit=limit)
return cve
# /api/browse
# /api/browse/
# /api/browse/<vendor>
@api
def api_browse(self, vendor=None):
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
browseList = query.getBrowseList(vendor)
if isinstance(browseList, dict):
return browseList
else:
return {}
# /api/search/<vendor>/<path:product>
@api
def api_search(self, vendor=None, product=None):
if vendor is None or product is None: return jsonify({})
search = vendor + ":" + product
# Not using query.cvesForCPE, because that one gives too much info
#return json.dumps(db.cvesForCPE(search), default=json_util.default)
return db.cvesForCPE(search)
# /api/dbInfo
@api
def api_dbInfo(self):
return db.getDBStats()
########################
# Web Server Functions #
########################
# signal handlers
def sig_handler(self, sig, frame):
print('Caught signal: %s' % sig)
IOLoop.instance().add_callback(self.shutdown)
def shutdown(self):
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
print('Stopping http server')
self.http_server.stop()
print('Will shutdown in %s seconds ...' %
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = IOLoop.instance()
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
def start(self):
# get properties
flaskHost = Configuration.getFlaskHost()
flaskPort = Configuration.getFlaskPort()
flaskDebug = Configuration.getFlaskDebug()
# logging
if Configuration.getLogging():
logfile = Configuration.getLogfile()
pathToLog = logfile.rsplit('/', 1)[0]
if not os.path.exists(pathToLog):
os.makedirs(pathToLog)
maxLogSize = Configuration.getMaxLogSize()
backlog = Configuration.getBacklog()
file_handler = RotatingFileHandler(logfile,
maxBytes=maxLogSize,
backupCount=backlog)
file_handler.setLevel(logging.ERROR)
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.app.logger.addHandler(file_handler)
if flaskDebug:
# start debug flask server
self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
else:
# start asynchronous server using tornado wrapper for flask
# ssl connection
print("Server starting...")
if Configuration.useSSL():
ssl_options = {
"certfile":
os.path.join(_runPath, "../", Configuration.getSSLCert()),
"keyfile":
os.path.join(_runPath, "../", Configuration.getSSLKey())
}
else:
ssl_options = None
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
self.http_server = HTTPServer(WSGIContainer(self.app),
ssl_options=ssl_options)
self.http_server.bind(flaskPort, address=flaskHost)
self.http_server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
