Exemplo n.º 1
0
    def getConnection(self,
                      host=None,
                      port=None,
                      db=None,
                      user=None,
                      password=None):
        # Default connection
        _host, _port = conf.getDatabaseServer()
        _user, _password = conf.getDatabaseAuth()
        _db = conf.getDatabaseName()
        # assign needed
        host = host or _host
        port = port or _port
        user = user or _user
        password = password or _password
        db = db or _db

        try:
            if user and password:
                mongoURI = "mongodb://{user}:{pwd}@{host}:{port}/{db}".format(
                    user=urllib.parse.quote(user),
                    password=urllib.parse.quote(password),
                    host=host,
                    port=port,
                    db=db)
                connect = pymongo.MongoClient(mongoURI)
            else:
                connect = pymongo.MongoClient(host, port)[db]
        except Exception as e:
            print(e)
            sys.exit("Unable to connect to Mongo. Is it running on %s:%s?" %
                     (host, port))
        return connect
Exemplo n.º 2
0
 def db_getStats(self, include_admin=False):
     data = {
         'cves': {},
         'cpe': {},
         'cpeOther': {},
         'capec': {},
         'cwe': {},
         'via4': {}
     }
     for key in data.keys():
         data[key] = {
             'size': self._size(self.db[key.lower()]),
             'last_update': self._getUpdate(self.db[key.lower()])
         }
     if include_admin:
         data = {
             'stats': {
                 'size_on_disk': self.db.command("dbstats")['storageSize'],
                 'db_size': self.db.command('dbstats')['dataSize'],
                 'name': conf.getDatabaseName()
             },
             'data': data
         }
     return data
Exemplo n.º 3
0
class API():
    app = Flask(__name__, static_folder='static', static_url_path='/static')
    app.config['MONGO_DBNAME'] = Configuration.getDatabaseName()
    app.config['SECRET_KEY'] = str(random.getrandbits(256))

    def __init__(self):
        self.db = DatabaseLayer()


    def addRoute(self, route):
        self.app.add_url_rule(route['r'], view_func=route['f'], methods=route['m'])


    #############
    # Decorator #
    #############
    def api(funct):
        @wraps(funct)
        def api_wrapper(*args, **kwargs):
            data = error = None
            # Get data (and possibly errors)
            try:
                data = (funct(*args, **kwargs), 200)
            except APIError as e:
                error = ({'status': 'error', 'reason': e.message}, e.status)
            except Exception as e:
                print(e)
                error = ({'status': 'error', 'reason': 'Internal server error'}, 500)
            # Check if data should be returned as html or data
            try:
                legacy = True
                returnType = 'application/json'
                if (request.url_rule.rule.lower().startswith("/api/")):
                    # Support JSONP
                    if request.args.get('callback', False):
                        data="%s(%s)"%(request.args.get('callback'), data)

                    # Check API version for backwards compatibility. We'll call the old API v1.0
                    elif request.headers.get('Version') in ['1.1', '2.0']:
                        # Get the requested return type
                        returnType = request.headers.get('Accept')
                        # Default to JSON
                        if   any(t in returnType for t in ['json', 'application/*', 'text/*', '*/*']):
                            data = error if error else {'status': 'success', 'data': data[0]}
                        elif 'plain' in returnType:
                            pass # No need to do anything, but needs to be accepted
                        else:
                            data = ({'status': 'error', 'reason': 'Unknown Content-type requested'}, 415)
                            returnType = 'application/json'
                        if request.headers.get('Version') in ['2.0']:
                            legacy = False
                    if type(data) is tuple:
                        data = list(data)
                        data[0] = json.dumps(bsonConverter(data[0], legacy), indent=2, sort_keys=True, default=json_util.default)
                    else:
                        data = (json.dumps(bsonConverter(data, legacy), indent=2, sort_keys=True, default=json_util.default), 200)
                    return Response(data[0], mimetype=returnType), data[1]
                else:
                    return data[0]
            except Exception as e:
                if str(e).startswith("Working outside of request context."):
                    return data[0]
                pass
            if error and error[1] == 500: raise(APIError(error[0]['reason']))
        return api_wrapper

    #############
    # FUNCTIONS #
    #############
    def generate_minimal_query(self, f):
        query = []
        # retrieving lists
        if f['rejectedSelect'] == "hide":
            exp = "^(?!\*\* REJECT \*\*\s+DO NOT USE THIS CANDIDATE NUMBER.*)"
            query.append({'summary': re.compile(exp)})

        # cvss logic
        if   f['cvssSelect'] == "above":  query.append({'cvss': {'$gt': float(f['cvss'])}})
        elif f['cvssSelect'] == "equals": query.append({'cvss': float(f['cvss'])})
        elif f['cvssSelect'] == "below":  query.append({'cvss': {'$lt': float(f['cvss'])}})

        # date logic
        if f['timeSelect'] != "all":
            if f['startDate']:
                startDate = parse_datetime(f['startDate'], ignoretz=True, dayfirst=True)
            if f['endDate']:
                endDate   = parse_datetime(f['endDate'],   ignoretz=True, dayfirst=True)

            if   f['timeSelect'] == "from":
                query.append({f['timeTypeSelect']: {'$gt': startDate}})
            elif f['timeSelect'] == "until":
                query.append({f['timeTypeSelect']: {'$lt': endDate}})
            elif f['timeSelect'] == "between":
                query.append({f['timeTypeSelect']: {'$gt': startDate, '$lt': endDate}})
            elif f['timeSelect'] == "outside":
                query.append({'$or': [{f['timeTypeSelect']: {'$lt': startDate}}, {f['timeTypeSelect']: {'$gt': endDate}}]})
        return query

    def filter_logic(self, filters, skip, limit=None):
        query = self.generate_minimal_query(filters)
        limit = limit if limit else self.args['pageLength']
        return self.db.CVE.query(limit=limit, skip=skip, query=query)

    ##########
    # ROUTES #
    ##########
    # /api
    def api_documentation(self):
        return render_template('api.html')

    # /api/cpe2.3/<cpe>
    @api
    def api_cpe23(self, cpe):
        cpe = tk.toStringFormattedCPE(cpe)
        return cpe if cpe else "None"

    # /api/cpe2.2/<cpe>
    @api
    def api_cpe22(self, cpe):
        cpe = tk.toOldCPE(cpe)
        return cpe if cpe else "None"

    # /api/cvefor/<cpe>
    @api
    def api_cvesFor(self, cpe):
        cpe  = urllib.parse.unquote_plus(cpe)
        return self.db.CVE.forCPE(cpe)

    # /api/cve/<cveid>
    @api
    def api_cve(self, cveid):
        cve = self.db.CVE.get(cveid, ranking=True, via4=True)
        if not cve: raise(APIError('cve not found', 404))
        return cve

    # /api/cwe
    # /api/cwe/<cwe_id>
    @api
    def api_cwe(self, cwe_id=None):
        if cwe_id: return self.db.CAPEC.relatedTo(cwe_id)
        else:      return self.db.CWE.getAll()

    # /api/capec/<cweid>
    @api
    def api_capec(self, cweid):
        return self.db.CAPEC.get(cweid)

    # /api/last
    # /api/last/
    # /api/last/<limit>
    @api
    def api_last(self, limit=None):
        limit = limit or 30
        return self.db.CVE.last(limit=limit, ranking=True, via4=True)

    # /query
    @api
    def api_query(self):
        f={'rejectedSelect': request.headers.get('rejected'),
           'cvss':           request.headers.get('cvss_score'),
           'cvssSelect':     request.headers.get('cvss_modifier'),
           'startDate':      request.headers.get('time_start'),
           'endDate':        request.headers.get('time_end'),
           'timeSelect':     request.headers.get('time_modifier'),
           'timeTypeSelect': request.headers.get('time_type'),
           'skip':           request.headers.get('skip'),
           'limit':          request.headers.get('limit')}
        try:
            skip = int(f['skip']) if f['skip'] else 0
        except:
            raise(APIError('skip must be an int', 400))
        try:
            limit = int(f['limit']) if f['limit'] else 0
        except:
            raise(APIError('limit must be an int', 400))
        return self.filter_logic(f, skip, limit)

    # /api/browse
    # /api/browse/
    # /api/browse/<vendor>
    @api
    def api_browse(self, vendor=None):
        if vendor:
            vendor = urllib.parse.quote_plus(vendor).lower()
        try:
            if vendor:
                return {'vendor': vendor, 'product': self.db.Redis.products(vendor)}
            return {'vendor': self.db.Redis.vendors(), 'product': None}
        except redisExceptions.ConnectionError:
            raise(APIError("Server could not connect to the browsing repository", 503))

    # /api/search/<vendor>/<path:product>
    @api
    def api_search(self, vendor=None, product=None):
        if not (vendor and product): return {}
        search = vendor + ":" + product
        return self.db.CVE.forCPE(search)

    # /api/search/<path:search>
    @api
    def api_text_search(self, search):
        result={'data':[]}
        links =      {'n': 'Link',          'd': self.db.CVE.via4links(search)}
        try:
            textsearch={'n': 'Text search', 'd': self.db.CVE.textSearch(search)}
        except:
            textsearch={'n': 'Text search', 'd': []}
            result['errors']=['textsearch']

        for collection in [links, textsearch]:
            for item in collection['d']:
                # Check if already in result data
                if not any(item.id==entry.id for entry in result['data']):
                    item.reason=collection['n']
                    result['data'].append(item)
        return result

    # /api/link/<key>/<value>
    @api
    def api_link(self, key=None,value=None):
        key=self.htmlDecode(key)
        value=self.htmlDecode(value)
        regex = re.compile(re.escape(value), re.I)
        data = {'cves': self.db.VIA4.link(key, regex)}
        cvssList=[float(x.cvss) for x in data['cves'] if x.cvss]
        if cvssList:
            data['stats']={'maxCVSS': max(cvssList), 'minCVSS': min(cvssList),'count':len(data['cves'])}
        else:
            data['stats']={'maxCVSS': 0, 'minCVSS': 0, 'count':len(data['cves'])}
        return data

    # /api/dbInfo
    @api
    def api_dbInfo(self):
        return self.db.db_info()


    ########################
    # Web Server Functions #
    ########################
    # signal handlers
    def sig_handler(self, sig, frame):
        print('Caught signal: %s' % sig)
        IOLoop.instance().add_callback(self.shutdown)

    def shutdown(self):
        MAX_WAIT_SECONDS_BEFORE_SHUTDOWN = 3
        print('Stopping http server')
        self.http_server.stop()

        print('Will shutdown in %s seconds ...' % MAX_WAIT_SECONDS_BEFORE_SHUTDOWN)
        io_loop = IOLoop.instance()
        deadline = time.time() + MAX_WAIT_SECONDS_BEFORE_SHUTDOWN

        def stop_loop():
            now = time.time()
            if now < deadline and (io_loop._callbacks or io_loop._timeouts):
                io_loop.add_timeout(now + 1, stop_loop)
            else:
                io_loop.stop()
                print('Shutdown')
        stop_loop()

    def start(self):
        # get properties
        flaskHost = Configuration.getFlaskHost()
        flaskPort = Configuration.getFlaskPort()
        flaskDebug = Configuration.getFlaskDebug()
        # logging
        if Configuration.getLogging():
            logfile = Configuration.getLogfile()
            pathToLog = logfile.rsplit('/', 1)[0]
            if not os.path.exists(pathToLog):
                os.makedirs(pathToLog)
            maxLogSize = Configuration.getMaxLogSize()
            backlog = Configuration.getBacklog()
            file_handler = RotatingFileHandler(logfile, maxBytes=maxLogSize, backupCount=backlog)
            file_handler.setLevel(logging.ERROR)
            formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
            file_handler.setFormatter(formatter)
            self.app.logger.addHandler(file_handler)

        # Placing routes - Doing this later allows us to define multiple API instances, without flask problems
        for route in web.Routes.get_routes(self): self.addRoute(route)

        if flaskDebug:
            # start debug flask server
            self.app.run(host=flaskHost, port=flaskPort, debug=flaskDebug)
        else:
            # start asynchronous server using tornado wrapper for flask
            # ssl connection
            print("Server starting...")
            if Configuration.useSSL():
                ssl_options = {"certfile": os.path.join(_runPath, "../", Configuration.getSSLCert()),
                                "keyfile": os.path.join(_runPath, "../", Configuration.getSSLKey())}
            else:
                ssl_options = None
            signal.signal(signal.SIGTERM, self.sig_handler)
            signal.signal(signal.SIGINT,  self.sig_handler)

            self.http_server = HTTPServer(WSGIContainer(self.app), ssl_options=ssl_options)
            self.http_server.bind(flaskPort, address=flaskHost)
            self.http_server.start(0)  # Forks multiple sub-processes
            IOLoop.instance().start()