def setUp(self): self.db = DatabaseLayer(_db="cvedb_test") self.capec1 = CAPEC(id="10000", name="test_capec", summary="no summary", prerequisites="No prerequisites", solutions="There's no solution", weaknesses=["10000"]) self.cwe1 = CWE(id="10000", name="test_cwe", description="test cwe", status="testing", weakness='Testing') self.cpe1 = CPE(id="cpe:/a:test:test1", title="Test CPE 1", references=[]) self.cpe2 = CPE(id="cpe:2.3:a:test:test2", title="Test CPE 2", references=[]) self.cve1 = CVE(id="CVE-0001-0001", cvss=0.1, summary="Test Vulnerability", vulnerable_configuration=[self.cpe1, self.cpe2], published=datetime.datetime(2017, 1, 1), impact=Impact("None", "None", "None"), access=Access("Low", "None", "Local"), cwe=self.cwe1) self.db.CAPEC.upsert(self.capec1) self.db.CWE.upsert(self.cwe1) self.db.CPE.upsert([self.cpe1, self.cpe2]) self.db.CVE.upsert(self.cve1)
def listranking(format='json'): ranks = [] for x in DatabaseLayer().Ranking.get(): if format == "json": ranks.append(x) else: ranks.append(x['cpe'] + " " + str(x['rank'])) return ranks
def index(limit=5, cpe_lookup=False, verbose=False): if limit == 0: limit = -1 data = DatabaseLayer().CVE.last(limit=limit) for cve in progressbar(data, prefix="Processing"): writer = get_schema_writer() title = cve.summary[:70] data = cve.summary if cpe_lookup: for cpe in cve.vulnerable_configuration: data += " " + cpe.title if verbose: print('Indexing CVE-ID ' + str(cve.id) + ' ' + title) writer.update_document(title=title, path=cve.id, content=data) writer.commit()
def authErrors(): db = DatabaseLayer( ) # Required to circumvent the use of self, because of this being a wrapper (This is one reason to use a singleton ;) ) # Check auth if not request.headers.get('Authorization'): return ({ 'status': 'error', 'reason': 'Authentication needed' }, 401) method, name, token = Advanced_API.getAuth() data = None if method.lower() not in ['basic', 'token', 'session']: data = ({ 'status': 'error', 'reason': 'Authorization method not allowed' }, 400) else: try: authenticated = False if method.lower() == 'basic': authenticator = AuthenticationHandler() if authenticator.validateUser(name, token): authenticated = True elif method.lower() == 'token': if db.Users.getToken(name) == token: authenticated = True elif method.lower() == 'session': authenticator = AuthenticationHandler() if authenticator.api_sessions.get(name) == token: authenticated = True if not authenticated: data = ({ 'status': 'error', 'reason': 'Authentication failed' }, 401) except Exception as e: print(e) data = ({ 'status': 'error', 'reason': 'Malformed Authentication String' }, 400) if data: return data else: return None
def setUp(self): self.db = DatabaseLayer(_db="cvedb_test") self.capec1 = CAPEC(id="10000", name="test_capec", summary="no summary", prerequisites="No prerequisites", solutions="There's no solution", weaknesses=[]) self.cwe1 = CWE(id="10000", name="test_cwe", description="test cwe", status="testing", weakness='Testing') self.cpe1 = CPE(id="cpe:/a:test:test1", title="Test CPE 1", references=[]) self.cpe2 = CPE(id="cpe:2.3:a:test:test2", title="Test CPE 2", references=[])
_ap.add_argument('-a', default=False, action='store_true', help='Lookup CAPEC for related CWE weaknesses') _ap.add_argument('-v', type=str, help='vendor name to lookup in reference URLs') _ap.add_argument( '--api', type=str, help= 'When used, the script will query the specified API rather than the local instance' ) args = _ap.parse_args() db = DatabaseLayer() query = Query(api=args.api) items = [] kwargs = { 'namelookup': args.n, 'ranking': args.r, 'capec': args.a, 'product': args.p, # only used by html, otherwise ignored 'cveids': args.c } # only used by html, otherwise ignored # Fetch cves if args.p: items.extend(query.cveforcpe(args.p)) if args.f: items.extend(query.search(args.f))
) argParser.add_argument('-c', default=False, action='store_true', help='Display CAPEC values') args = argParser.parse_args() if args.l: last = args.l else: last = 10 ref = "http://adulau.github.com/cve-search/" cves = DatabaseLayer().CVE.last(limit=last, ranking=args.r) if not (args.f == "html"): from feedformatter import Feed feed = Feed() feed.feed['title'] = "cve-search Last " + str( last) + " CVE entries generated on " + str(datetime.datetime.now()) feed.feed['link'] = "http://adulau.github.com/cve-search/" feed.feed['author'] = "Generated with cve-search available at " + ref feed.feed['description'] = "" else: print("<html><head>") print( "<style>.cve table { border-collapse: collapse; text-align: left; width: 100%; } .cve {font: normal 12px/150% Geneva, Arial, Helvetica, sans-serif; background: #fff; overflow: hidden; border: 1px solid #006699; -webkit-border-radius: 3px; -moz-border-radius: 3px; border-radius: 3px; }.cve table td, .cve table th { padding: 3px 10px; }.cve table tbody td { color: #00496B; border-left: 1px solid #E1EEF4;font-size: 12px;font-weight: normal; }.cve table tbody .alt td { background: #E1EEF4; color: #00496B; }.cve table tbody td:first-child { border-left: none; }.cve table tbody tr:last-child td { border-bottom: none; }.cve table tfoot td div { border-top: 1px solid #006699;background: #E1EEF4;} .cve table tfoot td { padding: 0; font-size: 12px } .cve table tfoot td div{ padding: 0px; }</style>" )
def __init__(self): self.plugins = {} self.db = DatabaseLayer()
def __init__(self): self.db = DatabaseLayer()
class API(): app = Flask(__name__, static_folder='static', static_url_path='/static') app.config['MONGO_DBNAME'] = Configuration.getDatabaseName() app.config['SECRET_KEY'] = str(random.getrandbits(256)) def __init__(self): self.db = DatabaseLayer() def addRoute(self, route): self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m']) ############# # Decorator # ############# def api(funct): @wraps(funct) def api_wrapper(*args, **kwargs): data = error = None # Get data (and possibly errors) try: data = (funct(*args, **kwargs), 200) except APIError as e: error = ({'status': 'error', 'reason': e.message}, e.status) except Exception as e: print(e) error = ({'status': 'error', 'reason': 'Internal server error'}, 500) # Check if data should be returned as html or data try: legacy = True returnType = 'application/json' if (request.url_rule.rule.lower().startswith("/api/")): # Support JSONP if request.args.get('callback', False): data="%s(%s)"%(request.args.get('callback'), data) # Check API version for backwards compatibility. We'll call the old API v1.0 elif request.headers.get('Version') in ['1.1', '2.0']: # Get the requested return type returnType = request.headers.get('Accept') # Default to JSON if any(t in returnType for t in ['json', 'application/*', 'text/*', '*/*']): data = error if error else {'status': 'success', 'data': data[0]} elif 'plain' in returnType: pass # No need to do anything, but needs to be accepted else: data = ({'status': 'error', 'reason': 'Unknown Content-type requested'}, 415) returnType = 'application/json' if request.headers.get('Version') in ['2.0']: legacy = False if type(data) is tuple: data = list(data) data[0] = json.dumps(bsonConverter(data[0], legacy), indent=2, sort_keys=True, default=json_util.default) else: data = (json.dumps(bsonConverter(data, legacy), indent=2, sort_keys=True, default=json_util.default), 200) return Response(data[0], mimetype=returnType), data[1] else: return data[0] except Exception as e: if str(e).startswith("Working outside of request context."): return data[0] pass if error and error[1] == 500: raise(APIError(error[0]['reason'])) return api_wrapper ############# # FUNCTIONS # ############# def generate_minimal_query(self, f): query = [] # retrieving lists if f['rejectedSelect'] == "hide": exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)" query.append({'summary': re.compile(exp)}) # cvss logic if f['cvssSelect'] == "above": query.append({'cvss': {'$gt': float(f['cvss'])}}) elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])}) elif f['cvssSelect'] == "below": query.append({'cvss': {'$lt': float(f['cvss'])}}) # date logic if f['timeSelect'] != "all": if f['startDate']: startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True) if f['endDate']: endDate = parse_datetime(f['endDate'], ignoretz=True, dayfirst=True) if f['timeSelect'] == "from": query.append({f['timeTypeSelect']: {'$gt': startDate}}) elif f['timeSelect'] == "until": query.append({f['timeTypeSelect']: {'$lt': endDate}}) elif f['timeSelect'] == "between": query.append({f['timeTypeSelect']: {'$gt': startDate, '$lt': endDate}}) elif f['timeSelect'] == "outside": query.append({'$or': [{f['timeTypeSelect']: {'$lt': startDate}}, {f['timeTypeSelect']: {'$gt': endDate}}]}) return query def filter_logic(self, filters, skip, limit=None): query = self.generate_minimal_query(filters) limit = limit if limit else self.args['pageLength'] return self.db.CVE.query(limit=limit, skip=skip, query=query) ########## # ROUTES # ########## # /api def api_documentation(self): return render_template('api.html') # /api/cpe2.3/<cpe> @api def api_cpe23(self, cpe): cpe = tk.toStringFormattedCPE(cpe) return cpe if cpe else "None" # /api/cpe2.2/<cpe> @api def api_cpe22(self, cpe): cpe = tk.toOldCPE(cpe) return cpe if cpe else "None" # /api/cvefor/<cpe> @api def api_cvesFor(self, cpe): cpe = urllib.parse.unquote_plus(cpe) return self.db.CVE.forCPE(cpe) # /api/cve/<cveid> @api def api_cve(self, cveid): cve = self.db.CVE.get(cveid, ranking=True, via4=True) if not cve: raise(APIError('cve not found', 404)) return cve # /api/cwe # /api/cwe/<cwe_id> @api def api_cwe(self, cwe_id=None): if cwe_id: return self.db.CAPEC.relatedTo(cwe_id) else: return self.db.CWE.getAll() # /api/capec/<cweid> @api def api_capec(self, cweid): return self.db.CAPEC.get(cweid) # /api/last # /api/last/ # /api/last/<limit> @api def api_last(self, limit=None): limit = limit or 30 return self.db.CVE.last(limit=limit, ranking=True, via4=True) # /query @api def api_query(self): f={'rejectedSelect': request.headers.get('rejected'), 'cvss': request.headers.get('cvss_score'), 'cvssSelect': request.headers.get('cvss_modifier'), 'startDate': request.headers.get('time_start'), 'endDate': request.headers.get('time_end'), 'timeSelect': request.headers.get('time_modifier'), 'timeTypeSelect': request.headers.get('time_type'), 'skip': request.headers.get('skip'), 'limit': request.headers.get('limit')} try: skip = int(f['skip']) if f['skip'] else 0 except: raise(APIError('skip must be an int', 400)) try: limit = int(f['limit']) if f['limit'] else 0 except: raise(APIError('limit must be an int', 400)) return self.filter_logic(f, skip, limit) # /api/browse # /api/browse/ # /api/browse/<vendor> @api def api_browse(self, vendor=None): if vendor: vendor = urllib.parse.quote_plus(vendor).lower() try: if vendor: return {'vendor': vendor, 'product': self.db.Redis.products(vendor)} return {'vendor': self.db.Redis.vendors(), 'product': None} except redisExceptions.ConnectionError: raise(APIError("Server could not connect to the browsing repository", 503)) # /api/search/<vendor>/<path:product> @api def api_search(self, vendor=None, product=None): if not (vendor and product): return {} search = vendor + ":" + product return self.db.CVE.forCPE(search) # /api/search/<path:search> @api def api_text_search(self, search): result={'data':[]} links = {'n': 'Link', 'd': self.db.CVE.via4links(search)} try: textsearch={'n': 'Text search', 'd': self.db.CVE.textSearch(search)} except: textsearch={'n': 'Text search', 'd': []} result['errors']=['textsearch'] for collection in [links, textsearch]: for item in collection['d']: # Check if already in result data if not any(item.id==entry.id for entry in result['data']): item.reason=collection['n'] result['data'].append(item) return result # /api/link/<key>/<value> @api def api_link(self, key=None,value=None): key=self.htmlDecode(key) value=self.htmlDecode(value) regex = re.compile(re.escape(value), re.I) data = {'cves': self.db.VIA4.link(key, regex)} cvssList=[float(x.cvss) for x in data['cves'] if x.cvss] if cvssList: data['stats']={'maxCVSS': max(cvssList), 'minCVSS': min(cvssList),'count':len(data['cves'])} else: data['stats']={'maxCVSS': 0, 'minCVSS': 0, 'count':len(data['cves'])} return data # /api/dbInfo @api def api_dbInfo(self): return self.db.db_info() ######################## # Web Server Functions # ######################## # signal handlers def sig_handler(self, sig, frame): print('Caught signal: %s' % sig) IOLoop.instance().add_callback(self.shutdown) def shutdown(self): MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3 print('Stopping http server') self.http_server.stop() print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN) io_loop = IOLoop.instance() deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN def stop_loop(): now = time.time() if now < deadline and (io_loop._callbacks or io_loop._timeouts): io_loop.add_timeout(now + 1, stop_loop) else: io_loop.stop() print('Shutdown') stop_loop() def start(self): # get properties flaskHost = Configuration.getFlaskHost() flaskPort = Configuration.getFlaskPort() flaskDebug = Configuration.getFlaskDebug() # logging if Configuration.getLogging(): logfile = Configuration.getLogfile() pathToLog = logfile.rsplit('/', 1)[0] if not os.path.exists(pathToLog): os.makedirs(pathToLog) maxLogSize = Configuration.getMaxLogSize() backlog = Configuration.getBacklog() file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog) file_handler.setLevel(logging.ERROR) formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(formatter) self.app.logger.addHandler(file_handler) # Placing routes - Doing this later allows us to define multiple API instances, without flask problems for route in web.Routes.get_routes(self): self.addRoute(route) if flaskDebug: # start debug flask server self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug) else: # start asynchronous server using tornado wrapper for flask # ssl connection print("Server starting...") if Configuration.useSSL(): ssl_options = {"certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()), "keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey())} else: ssl_options = None signal.signal(signal.SIGTERM, self.sig_handler) signal.signal(signal.SIGINT, self.sig_handler) self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options) self.http_server.bind(flaskPort, address=flaskHost) self.http_server.start(0) # Forks multiple sub-processes IOLoop.instance().start()
runPath = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(runPath, "..")) from redis import exceptions as redisExceptions from lib.Config import Configuration from lib.DatabaseLayer import DatabaseLayer from lib.Toolkit import pad argParser = argparse.ArgumentParser(description='CPE entries importer in Redis cache') argParser.add_argument('-v', action='store_true', default=False, help='Verbose logging') argParser.add_argument('-o', action='store_true', default=False, help='Import cpeother database in Redis cache') args = argParser.parse_args() if args.o: cpe = DatabaseLayer().CPE.getAllAlternative() else: cpe = DatabaseLayer().CPE.getAll() try: r = Configuration.getRedisVendorConnection() except: sys.exit(1) for e in cpe: try: if not args.o: if e.id.count(':') > 4: value = ":".join(e.id.split(':')[:6]) (prefix, cpeversion, cpetype, vendor, product, version) = pad(value.split(':'),6) else:
print(json.dumps(output, sort_keys=True, indent=4)) elif output_format == "compact": print("\n".join([i.id for i in data])) elif output_format == "expanded": print("\n".join(["%s %s" % (i.id, i.title) for i in data])) if __name__ == "__main__": argParser = argparse.ArgumentParser( description='Search for CPE with a pattern') argParser.add_argument('-s', type=str, required=True, help='search in cpe list') argParser.add_argument( '-o', type=str, default='expanded', help='O = output format [expanded, compact, json] (default: expanded)') argParser.add_argument( '-f', action='store_true', help= 'Enlarge the CPE search to all CPE indexed. Need the cpeother activated.', default=False) args = argParser.parse_args() data = DatabaseLayer().CPE.get_regex(urllib.parse.quote(args.s), args.f) output(data, args.o)
def dump(limit, ranking=False, via4=False, capec=False): db = DatabaseLayer() for cve in db.CVE.last(limit=limit, via4=via4, ranking=ranking): item = cve.dict(capec, human_dates=True) print(json.dumps(item, sort_keys=True))
def searchcve(cpe=None): if cpe is None: return False cve = DatabaseLayer().CVE.forCPE(cpe) return cve
argParser.add_argument('-c', type=str, help='CPE name to add (e.g. google:chrome)') argParser.add_argument('-g', type=str, help='Name of the organization (e.g. mycompany)') argParser.add_argument('-r', type=int, default=1, help='Ranking value (integer) default value is 1') argParser.add_argument('-f', type=str, help='Find ranking based on a CPE name regexp') argParser.add_argument('-l', action='store_true', help='List all ranking') argParser.add_argument('-d', type=str, default=None, help='Remove ranking based on a CPE name regexp') args = argParser.parse_args() if args.c is not None and args.g is not None: DatabaseLayer().Ranking.add(cpe=args.c, key=args.g, rank=args.r) elif args.f is not None: print(DatabaseLayer().CPE.ranking(cpe=args.f)) elif args.l: print(listranking()) elif args.d: DatabaseLayer().Ranking.remove(cpe=args.d) else: argParser.print_help()
def __init__(self, **kwargs): self.methods = [] self._load_methods() self.api_sessions = {} self.db = DatabaseLayer()