def getConnection(self,
host=None,
port=None,
db=None,
user=None,
password=None):
# Default connection
_host, _port = conf.getDatabaseServer()
_user, _password = conf.getDatabaseAuth()
_db = conf.getDatabaseName()
# assign needed
host = host or _host
port = port or _port
user = user or _user
password = password or _password
db = db or _db
try:
if user and password:
mongoURI = "mongodb://{user}:{pwd}@{host}:{port}/{db}".format(
user=urllib.parse.quote(user),
password=urllib.parse.quote(password),
host=host,
port=port,
db=db)
connect = pymongo.MongoClient(mongoURI)
else:
connect = pymongo.MongoClient(host, port)[db]
except Exception as e:
print(e)
sys.exit("Unable to connect to Mongo. Is it running on %s:%s?" %
(host, port))
return connect
def db_getStats(self, include_admin=False):
data = {
'cves': {},
'cpe': {},
'cpeOther': {},
'capec': {},
'cwe': {},
'via4': {}
}
for key in data.keys():
data[key] = {
'size': self._size(self.db[key.lower()]),
'last_update': self._getUpdate(self.db[key.lower()])
}
if include_admin:
data = {
'stats': {
'size_on_disk': self.db.command("dbstats")['storageSize'],
'db_size': self.db.command('dbstats')['dataSize'],
'name': conf.getDatabaseName()
},
'data': data
}
return data
class API():
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getDatabaseName()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
def __init__(self):
self.db = DatabaseLayer()
def addRoute(self, route):
self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m'])
#############
# Decorator #
#############
def api(funct):
@wraps(funct)
def api_wrapper(*args, **kwargs):
data = error = None
# Get data (and possibly errors)
try:
data = (funct(*args, **kwargs), 200)
except APIError as e:
error = ({'status': 'error', 'reason': e.message}, e.status)
except Exception as e:
print(e)
error = ({'status': 'error', 'reason': 'Internal server error'}, 500)
# Check if data should be returned as html or data
try:
legacy = True
returnType = 'application/json'
if (request.url_rule.rule.lower().startswith("/api/")):
# Support JSONP
if request.args.get('callback', False):
data="%s(%s)"%(request.args.get('callback'), data)
# Check API version for backwards compatibility. We'll call the old API v1.0
elif request.headers.get('Version') in ['1.1', '2.0']:
# Get the requested return type
returnType = request.headers.get('Accept')
# Default to JSON
if any(t in returnType for t in ['json', 'application/*', 'text/*', '*/*']):
data = error if error else {'status': 'success', 'data': data[0]}
elif 'plain' in returnType:
pass # No need to do anything, but needs to be accepted
else:
data = ({'status': 'error', 'reason': 'Unknown Content-type requested'}, 415)
returnType = 'application/json'
if request.headers.get('Version') in ['2.0']:
legacy = False
if type(data) is tuple:
data = list(data)
data[0] = json.dumps(bsonConverter(data[0], legacy), indent=2, sort_keys=True, default=json_util.default)
else:
data = (json.dumps(bsonConverter(data, legacy), indent=2, sort_keys=True, default=json_util.default), 200)
return Response(data[0], mimetype=returnType), data[1]
else:
return data[0]
except Exception as e:
if str(e).startswith("Working outside of request context."):
return data[0]
pass
if error and error[1] == 500: raise(APIError(error[0]['reason']))
return api_wrapper
#############
# FUNCTIONS #
#############
def generate_minimal_query(self, f):
query = []
# retrieving lists
if f['rejectedSelect'] == "hide":
exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
query.append({'summary': re.compile(exp)})
# cvss logic
if f['cvssSelect'] == "above": query.append({'cvss': {'$gt': float(f['cvss'])}})
elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])})
elif f['cvssSelect'] == "below": query.append({'cvss': {'$lt': float(f['cvss'])}})
# date logic
if f['timeSelect'] != "all":
if f['startDate']:
startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True)
if f['endDate']:
endDate = parse_datetime(f['endDate'], ignoretz=True, dayfirst=True)
if f['timeSelect'] == "from":
query.append({f['timeTypeSelect']: {'$gt': startDate}})
elif f['timeSelect'] == "until":
query.append({f['timeTypeSelect']: {'$lt': endDate}})
elif f['timeSelect'] == "between":
query.append({f['timeTypeSelect']: {'$gt': startDate, '$lt': endDate}})
elif f['timeSelect'] == "outside":
query.append({'$or': [{f['timeTypeSelect']: {'$lt': startDate}}, {f['timeTypeSelect']: {'$gt': endDate}}]})
return query
def filter_logic(self, filters, skip, limit=None):
query = self.generate_minimal_query(filters)
limit = limit if limit else self.args['pageLength']
return self.db.CVE.query(limit=limit, skip=skip, query=query)
##########
# ROUTES #
##########
# /api
def api_documentation(self):
return render_template('api.html')
# /api/cpe2.3/<cpe>
@api
def api_cpe23(self, cpe):
cpe = tk.toStringFormattedCPE(cpe)
return cpe if cpe else "None"
# /api/cpe2.2/<cpe>
@api
def api_cpe22(self, cpe):
cpe = tk.toOldCPE(cpe)
return cpe if cpe else "None"
# /api/cvefor/<cpe>
@api
def api_cvesFor(self, cpe):
cpe = urllib.parse.unquote_plus(cpe)
return self.db.CVE.forCPE(cpe)
# /api/cve/<cveid>
@api
def api_cve(self, cveid):
cve = self.db.CVE.get(cveid, ranking=True, via4=True)
if not cve: raise(APIError('cve not found', 404))
return cve
# /api/cwe
# /api/cwe/<cwe_id>
@api
def api_cwe(self, cwe_id=None):
if cwe_id: return self.db.CAPEC.relatedTo(cwe_id)
else: return self.db.CWE.getAll()
# /api/capec/<cweid>
@api
def api_capec(self, cweid):
return self.db.CAPEC.get(cweid)
# /api/last
# /api/last/
# /api/last/<limit>
@api
def api_last(self, limit=None):
limit = limit or 30
return self.db.CVE.last(limit=limit, ranking=True, via4=True)
# /query
@api
def api_query(self):
f={'rejectedSelect': request.headers.get('rejected'),
'cvss': request.headers.get('cvss_score'),
'cvssSelect': request.headers.get('cvss_modifier'),
'startDate': request.headers.get('time_start'),
'endDate': request.headers.get('time_end'),
'timeSelect': request.headers.get('time_modifier'),
'timeTypeSelect': request.headers.get('time_type'),
'skip': request.headers.get('skip'),
'limit': request.headers.get('limit')}
try:
skip = int(f['skip']) if f['skip'] else 0
except:
raise(APIError('skip must be an int', 400))
try:
limit = int(f['limit']) if f['limit'] else 0
except:
raise(APIError('limit must be an int', 400))
return self.filter_logic(f, skip, limit)
# /api/browse
# /api/browse/
# /api/browse/<vendor>
@api
def api_browse(self, vendor=None):
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
try:
if vendor:
return {'vendor': vendor, 'product': self.db.Redis.products(vendor)}
return {'vendor': self.db.Redis.vendors(), 'product': None}
except redisExceptions.ConnectionError:
raise(APIError("Server could not connect to the browsing repository", 503))
# /api/search/<vendor>/<path:product>
@api
def api_search(self, vendor=None, product=None):
if not (vendor and product): return {}
search = vendor + ":" + product
return self.db.CVE.forCPE(search)
# /api/search/<path:search>
@api
def api_text_search(self, search):
result={'data':[]}
links = {'n': 'Link', 'd': self.db.CVE.via4links(search)}
try:
textsearch={'n': 'Text search', 'd': self.db.CVE.textSearch(search)}
except:
textsearch={'n': 'Text search', 'd': []}
result['errors']=['textsearch']
for collection in [links, textsearch]:
for item in collection['d']:
# Check if already in result data
if not any(item.id==entry.id for entry in result['data']):
item.reason=collection['n']
result['data'].append(item)
return result
# /api/link/<key>/<value>
@api
def api_link(self, key=None,value=None):
key=self.htmlDecode(key)
value=self.htmlDecode(value)
regex = re.compile(re.escape(value), re.I)
data = {'cves': self.db.VIA4.link(key, regex)}
cvssList=[float(x.cvss) for x in data['cves'] if x.cvss]
if cvssList:
data['stats']={'maxCVSS': max(cvssList), 'minCVSS': min(cvssList),'count':len(data['cves'])}
else:
data['stats']={'maxCVSS': 0, 'minCVSS': 0, 'count':len(data['cves'])}
return data
# /api/dbInfo
@api
def api_dbInfo(self):
return self.db.db_info()
########################
# Web Server Functions #
########################
# signal handlers
def sig_handler(self, sig, frame):
print('Caught signal: %s' % sig)
IOLoop.instance().add_callback(self.shutdown)
def shutdown(self):
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
print('Stopping http server')
self.http_server.stop()
print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = IOLoop.instance()
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
def start(self):
# get properties
flaskHost = Configuration.getFlaskHost()
flaskPort = Configuration.getFlaskPort()
flaskDebug = Configuration.getFlaskDebug()
# logging
if Configuration.getLogging():
logfile = Configuration.getLogfile()
pathToLog = logfile.rsplit('/', 1)[0]
if not os.path.exists(pathToLog):
os.makedirs(pathToLog)
maxLogSize = Configuration.getMaxLogSize()
backlog = Configuration.getBacklog()
file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog)
file_handler.setLevel(logging.ERROR)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.app.logger.addHandler(file_handler)
# Placing routes - Doing this later allows us to define multiple API instances, without flask problems
for route in web.Routes.get_routes(self): self.addRoute(route)
if flaskDebug:
# start debug flask server
self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
else:
# start asynchronous server using tornado wrapper for flask
# ssl connection
print("Server starting...")
if Configuration.useSSL():
ssl_options = {"certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()),
"keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey())}
else:
ssl_options = None
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options)
self.http_server.bind(flaskPort, address=flaskHost)
self.http_server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()