def setUp(self):
self.db = DatabaseLayer(_db="cvedb_test")
self.capec1 = CAPEC(id="10000",
name="test_capec",
summary="no summary",
prerequisites="No prerequisites",
solutions="There's no solution",
weaknesses=["10000"])
self.cwe1 = CWE(id="10000",
name="test_cwe",
description="test cwe",
status="testing",
weakness='Testing')
self.cpe1 = CPE(id="cpe:/a:test:test1",
title="Test CPE 1",
references=[])
self.cpe2 = CPE(id="cpe:2.3:a:test:test2",
title="Test CPE 2",
references=[])
self.cve1 = CVE(id="CVE-0001-0001",
cvss=0.1,
summary="Test Vulnerability",
vulnerable_configuration=[self.cpe1, self.cpe2],
published=datetime.datetime(2017, 1, 1),
impact=Impact("None", "None", "None"),
access=Access("Low", "None", "Local"),
cwe=self.cwe1)
self.db.CAPEC.upsert(self.capec1)
self.db.CWE.upsert(self.cwe1)
self.db.CPE.upsert([self.cpe1, self.cpe2])
self.db.CVE.upsert(self.cve1)
def listranking(format='json'):
ranks = []
for x in DatabaseLayer().Ranking.get():
if format == "json":
ranks.append(x)
else:
ranks.append(x['cpe'] + " " + str(x['rank']))
return ranks
def index(limit=5, cpe_lookup=False, verbose=False):
if limit == 0: limit = -1
data = DatabaseLayer().CVE.last(limit=limit)
for cve in progressbar(data, prefix="Processing"):
writer = get_schema_writer()
title = cve.summary[:70]
data = cve.summary
if cpe_lookup:
for cpe in cve.vulnerable_configuration:
data += " " + cpe.title
if verbose:
print('Indexing CVE-ID ' + str(cve.id) + ' ' + title)
writer.update_document(title=title, path=cve.id, content=data)
writer.commit()
def authErrors():
db = DatabaseLayer(
) # Required to circumvent the use of self, because of this being a wrapper (This is one reason to use a singleton ;) )
# Check auth
if not request.headers.get('Authorization'):
return ({
'status': 'error',
'reason': 'Authentication needed'
}, 401)
method, name, token = Advanced_API.getAuth()
data = None
if method.lower() not in ['basic', 'token', 'session']:
data = ({
'status': 'error',
'reason': 'Authorization method not allowed'
}, 400)
else:
try:
authenticated = False
if method.lower() == 'basic':
authenticator = AuthenticationHandler()
if authenticator.validateUser(name, token):
authenticated = True
elif method.lower() == 'token':
if db.Users.getToken(name) == token: authenticated = True
elif method.lower() == 'session':
authenticator = AuthenticationHandler()
if authenticator.api_sessions.get(name) == token:
authenticated = True
if not authenticated:
data = ({
'status': 'error',
'reason': 'Authentication failed'
}, 401)
except Exception as e:
print(e)
data = ({
'status': 'error',
'reason': 'Malformed Authentication String'
}, 400)
if data:
return data
else:
return None
def setUp(self):
self.db = DatabaseLayer(_db="cvedb_test")
self.capec1 = CAPEC(id="10000",
name="test_capec",
summary="no summary",
prerequisites="No prerequisites",
solutions="There's no solution",
weaknesses=[])
self.cwe1 = CWE(id="10000",
name="test_cwe",
description="test cwe",
status="testing",
weakness='Testing')
self.cpe1 = CPE(id="cpe:/a:test:test1",
title="Test CPE 1",
references=[])
self.cpe2 = CPE(id="cpe:2.3:a:test:test2",
title="Test CPE 2",
references=[])
_ap.add_argument('-a',
default=False,
action='store_true',
help='Lookup CAPEC for related CWE weaknesses')
_ap.add_argument('-v',
type=str,
help='vendor name to lookup in reference URLs')
_ap.add_argument(
'--api',
type=str,
help=
'When used, the script will query the specified API rather than the local instance'
)
args = _ap.parse_args()
db = DatabaseLayer()
query = Query(api=args.api)
items = []
kwargs = {
'namelookup': args.n,
'ranking': args.r,
'capec': args.a,
'product': args.p, # only used by html, otherwise ignored
'cveids': args.c
} # only used by html, otherwise ignored
# Fetch cves
if args.p:
items.extend(query.cveforcpe(args.p))
if args.f:
items.extend(query.search(args.f))
)
argParser.add_argument('-c',
default=False,
action='store_true',
help='Display CAPEC values')
args = argParser.parse_args()
if args.l:
last = args.l
else:
last = 10
ref = "http://adulau.github.com/cve-search/"
cves = DatabaseLayer().CVE.last(limit=last, ranking=args.r)
if not (args.f == "html"):
from feedformatter import Feed
feed = Feed()
feed.feed['title'] = "cve-search Last " + str(
last) + " CVE entries generated on " + str(datetime.datetime.now())
feed.feed['link'] = "http://adulau.github.com/cve-search/"
feed.feed['author'] = "Generated with cve-search available at " + ref
feed.feed['description'] = ""
else:
print("<html><head>")
print(
"<style>.cve table { border-collapse: collapse; text-align: left; width: 100%; } .cve {font: normal 12px/150% Geneva, Arial, Helvetica, sans-serif; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; -moz-border-radius: 3px; border-radius: 3px; }.cve table td, .cve table th { padding: 3px 10px; }.cve table tbody td { color: #00496B; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal; }.cve table tbody .alt td { background: #E1EEF4; color: #00496B; }.cve table tbody td:first-child { border-left: none; }.cve table tbody tr:last-child td { border-bottom: none; }.cve table tfoot td div { border-top: 1px solid #006699;background: #E1EEF4;} .cve table tfoot td { padding: 0; font-size: 12px } .cve table tfoot td div{ padding: 0px; }</style>"
)
def __init__(self):
self.plugins = {}
self.db = DatabaseLayer()
def __init__(self):
self.db = DatabaseLayer()
class API():
app = Flask(__name__, static_folder='static', static_url_path='/static')
app.config['MONGO_DBNAME'] = Configuration.getDatabaseName()
app.config['SECRET_KEY'] = str(random.getrandbits(256))
def __init__(self):
self.db = DatabaseLayer()
def addRoute(self, route):
self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m'])
#############
# Decorator #
#############
def api(funct):
@wraps(funct)
def api_wrapper(*args, **kwargs):
data = error = None
# Get data (and possibly errors)
try:
data = (funct(*args, **kwargs), 200)
except APIError as e:
error = ({'status': 'error', 'reason': e.message}, e.status)
except Exception as e:
print(e)
error = ({'status': 'error', 'reason': 'Internal server error'}, 500)
# Check if data should be returned as html or data
try:
legacy = True
returnType = 'application/json'
if (request.url_rule.rule.lower().startswith("/api/")):
# Support JSONP
if request.args.get('callback', False):
data="%s(%s)"%(request.args.get('callback'), data)
# Check API version for backwards compatibility. We'll call the old API v1.0
elif request.headers.get('Version') in ['1.1', '2.0']:
# Get the requested return type
returnType = request.headers.get('Accept')
# Default to JSON
if any(t in returnType for t in ['json', 'application/*', 'text/*', '*/*']):
data = error if error else {'status': 'success', 'data': data[0]}
elif 'plain' in returnType:
pass # No need to do anything, but needs to be accepted
else:
data = ({'status': 'error', 'reason': 'Unknown Content-type requested'}, 415)
returnType = 'application/json'
if request.headers.get('Version') in ['2.0']:
legacy = False
if type(data) is tuple:
data = list(data)
data[0] = json.dumps(bsonConverter(data[0], legacy), indent=2, sort_keys=True, default=json_util.default)
else:
data = (json.dumps(bsonConverter(data, legacy), indent=2, sort_keys=True, default=json_util.default), 200)
return Response(data[0], mimetype=returnType), data[1]
else:
return data[0]
except Exception as e:
if str(e).startswith("Working outside of request context."):
return data[0]
pass
if error and error[1] == 500: raise(APIError(error[0]['reason']))
return api_wrapper
#############
# FUNCTIONS #
#############
def generate_minimal_query(self, f):
query = []
# retrieving lists
if f['rejectedSelect'] == "hide":
exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
query.append({'summary': re.compile(exp)})
# cvss logic
if f['cvssSelect'] == "above": query.append({'cvss': {'$gt': float(f['cvss'])}})
elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])})
elif f['cvssSelect'] == "below": query.append({'cvss': {'$lt': float(f['cvss'])}})
# date logic
if f['timeSelect'] != "all":
if f['startDate']:
startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True)
if f['endDate']:
endDate = parse_datetime(f['endDate'], ignoretz=True, dayfirst=True)
if f['timeSelect'] == "from":
query.append({f['timeTypeSelect']: {'$gt': startDate}})
elif f['timeSelect'] == "until":
query.append({f['timeTypeSelect']: {'$lt': endDate}})
elif f['timeSelect'] == "between":
query.append({f['timeTypeSelect']: {'$gt': startDate, '$lt': endDate}})
elif f['timeSelect'] == "outside":
query.append({'$or': [{f['timeTypeSelect']: {'$lt': startDate}}, {f['timeTypeSelect']: {'$gt': endDate}}]})
return query
def filter_logic(self, filters, skip, limit=None):
query = self.generate_minimal_query(filters)
limit = limit if limit else self.args['pageLength']
return self.db.CVE.query(limit=limit, skip=skip, query=query)
##########
# ROUTES #
##########
# /api
def api_documentation(self):
return render_template('api.html')
# /api/cpe2.3/<cpe>
@api
def api_cpe23(self, cpe):
cpe = tk.toStringFormattedCPE(cpe)
return cpe if cpe else "None"
# /api/cpe2.2/<cpe>
@api
def api_cpe22(self, cpe):
cpe = tk.toOldCPE(cpe)
return cpe if cpe else "None"
# /api/cvefor/<cpe>
@api
def api_cvesFor(self, cpe):
cpe = urllib.parse.unquote_plus(cpe)
return self.db.CVE.forCPE(cpe)
# /api/cve/<cveid>
@api
def api_cve(self, cveid):
cve = self.db.CVE.get(cveid, ranking=True, via4=True)
if not cve: raise(APIError('cve not found', 404))
return cve
# /api/cwe
# /api/cwe/<cwe_id>
@api
def api_cwe(self, cwe_id=None):
if cwe_id: return self.db.CAPEC.relatedTo(cwe_id)
else: return self.db.CWE.getAll()
# /api/capec/<cweid>
@api
def api_capec(self, cweid):
return self.db.CAPEC.get(cweid)
# /api/last
# /api/last/
# /api/last/<limit>
@api
def api_last(self, limit=None):
limit = limit or 30
return self.db.CVE.last(limit=limit, ranking=True, via4=True)
# /query
@api
def api_query(self):
f={'rejectedSelect': request.headers.get('rejected'),
'cvss': request.headers.get('cvss_score'),
'cvssSelect': request.headers.get('cvss_modifier'),
'startDate': request.headers.get('time_start'),
'endDate': request.headers.get('time_end'),
'timeSelect': request.headers.get('time_modifier'),
'timeTypeSelect': request.headers.get('time_type'),
'skip': request.headers.get('skip'),
'limit': request.headers.get('limit')}
try:
skip = int(f['skip']) if f['skip'] else 0
except:
raise(APIError('skip must be an int', 400))
try:
limit = int(f['limit']) if f['limit'] else 0
except:
raise(APIError('limit must be an int', 400))
return self.filter_logic(f, skip, limit)
# /api/browse
# /api/browse/
# /api/browse/<vendor>
@api
def api_browse(self, vendor=None):
if vendor:
vendor = urllib.parse.quote_plus(vendor).lower()
try:
if vendor:
return {'vendor': vendor, 'product': self.db.Redis.products(vendor)}
return {'vendor': self.db.Redis.vendors(), 'product': None}
except redisExceptions.ConnectionError:
raise(APIError("Server could not connect to the browsing repository", 503))
# /api/search/<vendor>/<path:product>
@api
def api_search(self, vendor=None, product=None):
if not (vendor and product): return {}
search = vendor + ":" + product
return self.db.CVE.forCPE(search)
# /api/search/<path:search>
@api
def api_text_search(self, search):
result={'data':[]}
links = {'n': 'Link', 'd': self.db.CVE.via4links(search)}
try:
textsearch={'n': 'Text search', 'd': self.db.CVE.textSearch(search)}
except:
textsearch={'n': 'Text search', 'd': []}
result['errors']=['textsearch']
for collection in [links, textsearch]:
for item in collection['d']:
# Check if already in result data
if not any(item.id==entry.id for entry in result['data']):
item.reason=collection['n']
result['data'].append(item)
return result
# /api/link/<key>/<value>
@api
def api_link(self, key=None,value=None):
key=self.htmlDecode(key)
value=self.htmlDecode(value)
regex = re.compile(re.escape(value), re.I)
data = {'cves': self.db.VIA4.link(key, regex)}
cvssList=[float(x.cvss) for x in data['cves'] if x.cvss]
if cvssList:
data['stats']={'maxCVSS': max(cvssList), 'minCVSS': min(cvssList),'count':len(data['cves'])}
else:
data['stats']={'maxCVSS': 0, 'minCVSS': 0, 'count':len(data['cves'])}
return data
# /api/dbInfo
@api
def api_dbInfo(self):
return self.db.db_info()
########################
# Web Server Functions #
########################
# signal handlers
def sig_handler(self, sig, frame):
print('Caught signal: %s' % sig)
IOLoop.instance().add_callback(self.shutdown)
def shutdown(self):
MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
print('Stopping http server')
self.http_server.stop()
print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
io_loop = IOLoop.instance()
deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN
def stop_loop():
now = time.time()
if now < deadline and (io_loop._callbacks or io_loop._timeouts):
io_loop.add_timeout(now + 1, stop_loop)
else:
io_loop.stop()
print('Shutdown')
stop_loop()
def start(self):
# get properties
flaskHost = Configuration.getFlaskHost()
flaskPort = Configuration.getFlaskPort()
flaskDebug = Configuration.getFlaskDebug()
# logging
if Configuration.getLogging():
logfile = Configuration.getLogfile()
pathToLog = logfile.rsplit('/', 1)[0]
if not os.path.exists(pathToLog):
os.makedirs(pathToLog)
maxLogSize = Configuration.getMaxLogSize()
backlog = Configuration.getBacklog()
file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog)
file_handler.setLevel(logging.ERROR)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
file_handler.setFormatter(formatter)
self.app.logger.addHandler(file_handler)
# Placing routes - Doing this later allows us to define multiple API instances, without flask problems
for route in web.Routes.get_routes(self): self.addRoute(route)
if flaskDebug:
# start debug flask server
self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
else:
# start asynchronous server using tornado wrapper for flask
# ssl connection
print("Server starting...")
if Configuration.useSSL():
ssl_options = {"certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()),
"keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey())}
else:
ssl_options = None
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options)
self.http_server.bind(flaskPort, address=flaskHost)
self.http_server.start(0) # Forks multiple sub-processes
IOLoop.instance().start()
runPath = os.path.dirname(os.path.realpath(__file__))
sys.path.append(os.path.join(runPath, ".."))
from redis import exceptions as redisExceptions
from lib.Config import Configuration
from lib.DatabaseLayer import DatabaseLayer
from lib.Toolkit import pad
argParser = argparse.ArgumentParser(description='CPE entries importer in Redis cache')
argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging')
argParser.add_argument('-o', action='store_true', default=False, help='Import cpeother database in Redis cache')
args = argParser.parse_args()
if args.o:
cpe = DatabaseLayer().CPE.getAllAlternative()
else:
cpe = DatabaseLayer().CPE.getAll()
try:
r = Configuration.getRedisVendorConnection()
except:
sys.exit(1)
for e in cpe:
try:
if not args.o:
if e.id.count(':') > 4:
value = ":".join(e.id.split(':')[:6])
(prefix, cpeversion, cpetype, vendor, product, version) = pad(value.split(':'),6)
else:
print(json.dumps(output, sort_keys=True, indent=4))
elif output_format == "compact":
print("\n".join([i.id for i in data]))
elif output_format == "expanded":
print("\n".join(["%s %s" % (i.id, i.title) for i in data]))
if __name__ == "__main__":
argParser = argparse.ArgumentParser(
description='Search for CPE with a pattern')
argParser.add_argument('-s',
type=str,
required=True,
help='search in cpe list')
argParser.add_argument(
'-o',
type=str,
default='expanded',
help='O = output format [expanded, compact, json] (default: expanded)')
argParser.add_argument(
'-f',
action='store_true',
help=
'Enlarge the CPE search to all CPE indexed. Need the cpeother activated.',
default=False)
args = argParser.parse_args()
data = DatabaseLayer().CPE.get_regex(urllib.parse.quote(args.s), args.f)
output(data, args.o)
def dump(limit, ranking=False, via4=False, capec=False):
db = DatabaseLayer()
for cve in db.CVE.last(limit=limit, via4=via4, ranking=ranking):
item = cve.dict(capec, human_dates=True)
print(json.dumps(item, sort_keys=True))
def searchcve(cpe=None):
if cpe is None:
return False
cve = DatabaseLayer().CVE.forCPE(cpe)
return cve
argParser.add_argument('-c',
type=str,
help='CPE name to add (e.g. google:chrome)')
argParser.add_argument('-g',
type=str,
help='Name of the organization (e.g. mycompany)')
argParser.add_argument('-r',
type=int,
default=1,
help='Ranking value (integer) default value is 1')
argParser.add_argument('-f',
type=str,
help='Find ranking based on a CPE name regexp')
argParser.add_argument('-l', action='store_true', help='List all ranking')
argParser.add_argument('-d',
type=str,
default=None,
help='Remove ranking based on a CPE name regexp')
args = argParser.parse_args()
if args.c is not None and args.g is not None:
DatabaseLayer().Ranking.add(cpe=args.c, key=args.g, rank=args.r)
elif args.f is not None:
print(DatabaseLayer().CPE.ranking(cpe=args.f))
elif args.l:
print(listranking())
elif args.d:
DatabaseLayer().Ranking.remove(cpe=args.d)
else:
argParser.print_help()
def __init__(self, **kwargs):
self.methods = []
self._load_methods()
self.api_sessions = {}
self.db = DatabaseLayer()