def __init__(self,
mongodb_connection_details=None,
api_connection_details=None):
"""
Create a new instance of CveXplore
:param mongodb_connection_details: Provide the connection details needed to establish a connection to a mongodb
instance. The connection details should be in line with pymongo's
documentation.
:type mongodb_connection_details: dict
:param api_connection_details: Provide the connection details needed to establish a connection to a cve-search
API provider. The cve-search API provider should allow access to the 'query' POST
endpoint; all other API endpoints are not needed for CveXplore to function. For
the connection details supported please check the :ref:`API connection <api>`
documentation.
:type api_connection_details: dict
"""
self.__version = VERSION
os.environ["DOC_BUILD"] = json.dumps({"DOC_BUILD": "NO"})
if (api_connection_details is not None
and mongodb_connection_details is not None):
raise ValueError(
"CveXplore can be used to connect to either a cve-search database OR a cve-search api, not both!"
)
elif api_connection_details is None and mongodb_connection_details is None:
# by default assume we are talking to a database
mongodb_connection_details = {}
os.environ["MONGODB_CON_DETAILS"] = json.dumps(
mongodb_connection_details)
self.datasource = MongoDBConnection(**mongodb_connection_details)
self.database = MainUpdater(datasource=self.datasource)
elif mongodb_connection_details is not None:
os.environ["MONGODB_CON_DETAILS"] = json.dumps(
mongodb_connection_details)
self.datasource = MongoDBConnection(**mongodb_connection_details)
self.database = MainUpdater(datasource=self.datasource)
elif api_connection_details is not None:
api_connection_details["user_agent"] = "CveXplore:{}".format(
self.version)
os.environ["API_CON_DETAILS"] = json.dumps(api_connection_details)
self.datasource = ApiDatabaseSource(**api_connection_details)
self.database_mapping = database_mapping
from CveXplore.database.helpers.generic_db import GenericDatabaseFactory
from CveXplore.database.helpers.specific_db import CvesDatabaseFunctions
for each in self.database_mapping:
if each != "cves":
setattr(self, each, GenericDatabaseFactory(collection=each))
else:
setattr(self, each, CvesDatabaseFunctions(collection=each))
class DatasourceConnection(object):
"""
The DatasourceConnection class handles the connection to the data source and is the base class for the database
objects and generic database functions
"""
__DATA_SOURCE_CONNECTION = (
ApiDatabaseSource(**json.loads(os.getenv("API_CON_DETAILS")))
if os.getenv("API_CON_DETAILS") else MongoDBConnection(
**json.loads(os.getenv("MONGODB_CON_DETAILS"))))
def __init__(self, collection):
"""
Create a DatasourceConnection object
:param collection: Define the collection to connect to
:type collection: str
"""
self.__collection = collection
@property
def _datasource_connection(self):
return DatasourceConnection.__DATA_SOURCE_CONNECTION
@property
def _datasource_collection_connection(self):
return getattr(
DatasourceConnection.__DATA_SOURCE_CONNECTION,
"store_{}".format(self.__collection),
)
@property
def _collection(self):
return self.__collection
def __init__(self, feed_type, prefix=None):
self._end = None
self.feed_type = feed_type
self.prefix = prefix
self.queue = WorkerQueue(name=self.feed_type)
self.file_queue = WorkerQueue(name=f"{self.feed_type}:files")
self.file_queue.clear()
self.progress_bar = None
self.last_modified = None
self.do_process = True
database = MongoDBConnection(
**json.loads(os.getenv("MONGODB_CON_DETAILS")))
self.database = database._dbclient
self.logger = logging.getLogger("DownloadHandler")
self.config = Configuration()
def __init__(self):
with open(os.path.join(runPath, "../../.schema_version")) as f:
self.schema_version = json.loads(f.read())
database = MongoDBConnection(
**json.loads(os.getenv("MONGODB_CON_DETAILS")))
self.dbh = database._dbclient["schema"]
self.logger = logging.getLogger("SchemaChecker")
def __init__(self):
database = MongoDBConnection(
**json.loads(os.getenv("MONGODB_CON_DETAILS")))
self.database = database._dbclient
self.indexes = {
"cpe": [
MongoUniqueIndex(
index=[("id", ASCENDING)],
name="id",
unique=True,
),
MongoAddIndex(index=[("vendor", ASCENDING)], name="vendor"),
MongoAddIndex(
index=[("product", ASCENDING)],
name="product",
),
],
"cpeother": [
MongoUniqueIndex(
index=[("id", ASCENDING)],
name="id",
unique=True,
)
],
"cves": [
MongoAddIndex(index=[("id", ASCENDING)], name="id"),
MongoAddIndex(
index=[("vulnerable_configuration", ASCENDING)],
name="vulnerable_configuration",
),
MongoAddIndex(
index=[("vulnerable_product", ASCENDING)],
name="vulnerable_product",
),
MongoAddIndex(
index=[("Modified", ASCENDING)],
name="Modified",
),
MongoAddIndex(
index=[("Published", ASCENDING)],
name="Published",
),
MongoAddIndex(
index=[("last-modified", ASCENDING)],
name="last-modified",
),
MongoAddIndex(index=[("cvss", ASCENDING)], name="cvss"),
MongoAddIndex(index=[("cvss3", ASCENDING)], name="cvss3"),
MongoAddIndex(index=[("summary", TEXT)], name="summary"),
MongoAddIndex(
index=[("vendors", ASCENDING)],
name="vendors",
),
MongoAddIndex(
index=[("products", ASCENDING)],
name="products",
),
MongoAddIndex(
index=[("vulnerable_product_stems", ASCENDING)],
name="vulnerable_product_stems",
),
MongoAddIndex(
index=[("vulnerable_configuration_stems", ASCENDING)],
name="vulnerable_configuration_stems",
),
],
"via4": [MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"mgmt_whitelist":
[MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"mgmt_blacklist":
[MongoAddIndex(index=[("id", ASCENDING)], name="id")],
"capec": [
MongoAddIndex(
index=[("related_weakness", ASCENDING)],
name="related_weakness",
)
],
}
self.logger = logging.getLogger("DatabaseIndexer")