def test_mongo_connect_store_file():
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
"db": "cuckootest",
},
},
})
mongo.init()
assert mongo.database == "cuckootest"
fd, filepath = tempfile.mkstemp()
os.write(fd, "hello world")
os.close(fd)
f = File(filepath)
r = MongoDB()
r.init_once()
id1 = r.store_file(f, "foobar.txt")
id2 = r.store_file(f, "foobar.txt")
assert id1 == id2
assert mongo.db.fs.files.find_one({
"sha256": f.get_sha256(),
})["_id"] == id1
assert mongo.grid.get(id1).read() == "hello world"
def test_mongo_connect_store_file():
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
"db": "cuckootest",
},
},
})
mongo.init()
assert mongo.database == "cuckootest"
fd, filepath = tempfile.mkstemp()
os.write(fd, "hello world")
os.close(fd)
f = File(filepath)
r = MongoDB()
r.init_once()
id1 = r.store_file(f, "foobar.txt")
id2 = r.store_file(f, "foobar.txt")
assert id1 == id2
assert mongo.db.fs.files.find_one({
"sha256": f.get_sha256(),
})["_id"] == id1
assert mongo.grid.get(id1).read() == "hello world"
def setup_class(cls):
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
"db": "cuckootest",
},
},
})
mongo.init()
mongo.connect()
def setup_class(cls):
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
"db": "cuckootest",
},
},
})
mongo.init()
mongo.connect()
def init_once(cls):
if not mongo.init():
return
mongo.connect()
cls.db = mongo.db
cls.fs = mongo.grid
# Set MongoDB schema version.
if "cuckoo_schema" in mongo.collection_names:
version = mongo.db.cuckoo_schema.find_one()["version"]
if version != cls.SCHEMA_VERSION:
raise CuckooReportError(
"Unknown MongoDB version schema version found. Cuckoo "
"doesn't really know how to proceed now.."
)
else:
mongo.db.cuckoo_schema.save({"version": cls.SCHEMA_VERSION})
# Set an unique index on stored files to avoid duplicates. As per the
# pymongo documentation this is basically a no-op if the index already
# exists. So we don't have to do that check ourselves.
mongo.db.fs.files.ensure_index(
"sha256", unique=True, sparse=True, name="sha256_unique"
)
def init_once(cls):
if not mongo.init():
return
mongo.connect()
cls.db = mongo.db
cls.fs = mongo.grid
# Set MongoDB schema version.
if "cuckoo_schema" in mongo.collection_names:
version = mongo.db.cuckoo_schema.find_one()["version"]
if version != cls.SCHEMA_VERSION:
raise CuckooReportError(
"Unknown MongoDB version schema version found. Cuckoo "
"doesn't really know how to proceed now.."
)
else:
mongo.db.cuckoo_schema.save({"version": cls.SCHEMA_VERSION})
# Set an unique index on stored files to avoid duplicates. As per the
# pymongo documentation this is basically a no-op if the index already
# exists. So we don't have to do that check ourselves.
mongo.db.fs.files.ensure_index(
"sha256", unique=True, sparse=True, name="sha256_unique"
)
def test_mongodb_offline(p, q, r, s):
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
},
},
})
r.side_effect = socket.error("error")
db = p.return_value.__getitem__.return_value
db.collection_names.side_effect = pymongo.errors.PyMongoError("error")
with pytest.raises(CuckooCriticalError) as e:
mongo.init()
mongo.connect()
e.match("Unable to connect to MongoDB")
s.warning.assert_called_once()
def test_mongodb_offline(p, q, r, s):
set_cwd(tempfile.mkdtemp())
cuckoo_create(cfg={
"reporting": {
"mongodb": {
"enabled": True,
},
},
})
r.side_effect = socket.error("error")
db = p.return_value.__getitem__.return_value
db.collection_names.side_effect = pymongo.errors.PyMongoError("error")
with pytest.raises(CuckooCriticalError) as e:
mongo.init()
mongo.connect()
e.match("Unable to connect to MongoDB")
s.warning.assert_called_once()
def mongo_upgrade():
"""Migrate mongodb schema and data."""
if mongo.init():
print "Starting MongoDB migration."
mongo.connect()
# Check for schema version and create it.
if "cuckoo_schema" in mongo.db.collection_names():
print "Mongo schema version not expected"
sys.exit()
else:
mongo.db.cuckoo_schema.save({"version": mongo_revision})
else:
print "Mongo reporting module not enabled, skipping mongo migration."
def mongo_upgrade():
"""Migrate mongodb schema and data."""
if mongo.init():
print "Starting MongoDB migration."
mongo.connect()
# Check for schema version and create it.
if "cuckoo_schema" in mongo.db.collection_names():
print "Mongo schema version not expected"
sys.exit()
else:
mongo.db.cuckoo_schema.save({"version": mongo_revision})
else:
print "Mongo reporting module not enabled, skipping mongo migration."
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured
databases (SQL and MongoDB).
"""
# Init logging (without writing to file).
init_console_logging()
try:
# Initialize the database connection.
db = Database()
db.connect(schema_check=False)
# Drop all tables.
db.drop()
except (CuckooDependencyError, CuckooDatabaseError) as e:
# If something is screwed due to incorrect database migrations or bad
# database SqlAlchemy would be unable to connect and operate.
log.warning(
"Error connecting to database: it is suggested to check "
"the connectivity, apply all migrations if needed or purge "
"it manually. Error description: %s", e)
# Check if MongoDB reporting is enabled and drop the database if it is.
if mongo.init():
try:
mongo.connect()
mongo.drop()
mongo.close()
except Exception as e:
log.warning("Unable to drop MongoDB database: %s", e)
# Check if ElasticSearch reporting is enabled and drop its data if it is.
if elastic.init():
elastic.connect()
# TODO This should be moved to the elastic abstract.
# TODO We should also drop historic data, i.e., from pervious days,
# months, and years.
date_index = datetime.datetime.utcnow().strftime({
"yearly": "%Y",
"monthly": "%Y-%m",
"daily": "%Y-%m-%d",
}[elastic.index_time_pattern])
dated_index = "%s-%s" % (elastic.index, date_index)
elastic.client.indices.delete(index=dated_index, ignore=[400, 404])
template_name = "%s_template" % dated_index
if elastic.client.indices.exists_template(template_name):
elastic.client.indices.delete_template(template_name)
# Paths to clean.
paths = [
cwd("cuckoo.db"),
cwd("log"),
cwd("storage", "analyses"),
cwd("storage", "baseline"),
cwd("storage", "binaries"),
]
# Delete the various files and directories. In case of directories, keep
# the parent directories, so to keep the state of the CWD in tact.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
os.mkdir(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
elif os.path.isfile(path):
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
def cuckoo_clean():
"""Clean up cuckoo setup.
It deletes logs, all stored data from file system and configured
databases (SQL and MongoDB).
"""
# Init logging (without writing to file).
init_console_logging()
try:
# Initialize the database connection.
db = Database()
db.connect(schema_check=False)
# Drop all tables.
db.drop()
except (CuckooDependencyError, CuckooDatabaseError) as e:
# If something is screwed due to incorrect database migrations or bad
# database SqlAlchemy would be unable to connect and operate.
log.warning("Error connecting to database: it is suggested to check "
"the connectivity, apply all migrations if needed or purge "
"it manually. Error description: %s", e)
# Check if MongoDB reporting is enabled and drop the database if it is.
if mongo.init():
try:
mongo.connect()
mongo.drop()
mongo.close()
except Exception as e:
log.warning("Unable to drop MongoDB database: %s", e)
# Check if ElasticSearch reporting is enabled and drop its data if it is.
if elastic.init():
elastic.connect()
# TODO This should be moved to the elastic abstract.
# TODO We should also drop historic data, i.e., from pervious days,
# months, and years.
date_index = datetime.datetime.utcnow().strftime({
"yearly": "%Y",
"monthly": "%Y-%m",
"daily": "%Y-%m-%d",
}[elastic.index_time_pattern])
dated_index = "%s-%s" % (elastic.index, date_index)
elastic.client.indices.delete(
index=dated_index, ignore=[400, 404]
)
template_name = "%s_template" % dated_index
if elastic.client.indices.exists_template(template_name):
elastic.client.indices.delete_template(template_name)
# Paths to clean.
paths = [
cwd("cuckoo.db"),
cwd("log"),
cwd("storage", "analyses"),
cwd("storage", "baseline"),
cwd("storage", "binaries"),
]
# Delete the various files and directories. In case of directories, keep
# the parent directories, so to keep the state of the CWD in tact.
for path in paths:
if os.path.isdir(path):
try:
shutil.rmtree(path)
os.mkdir(path)
except (IOError, OSError) as e:
log.warning("Error removing directory %s: %s", path, e)
elif os.path.isfile(path):
try:
os.unlink(path)
except (IOError, OSError) as e:
log.warning("Error removing file %s: %s", path, e)
import os
import sys
from django.template.base import TemplateSyntaxError
from cuckoo.common.colors import red
from cuckoo.common.elastic import elastic
from cuckoo.common.mongo import mongo
from cuckoo.core.startup import init_rooter, init_routing
from cuckoo.misc import cwd, decide_cwd
if cwd(root=True) is None:
decide_cwd(exists=True)
# Connect to MongoDB (mandatory).
if not mongo.init():
sys.exit(
red("In order to use the Cuckoo Web Interface it is required to have "
"MongoDB up-and-running and enabled in Cuckoo. Please refer to our "
"official documentation as well as the $CWD/conf/reporting.conf file."
))
mongo.connect()
# Connect to ElasticSearch (optional).
elastic.init()
elastic.connect()
# In case we have VPNs enabled we need to initialize through the following
# two methods as they verify the interaction with VPNs as well as gather
# which VPNs are available (for representation upon File/URL submission).
import os
import sys
from django.template.base import TemplateSyntaxError
from cuckoo.common.colors import red
from cuckoo.common.elastic import elastic
from cuckoo.common.mongo import mongo
from cuckoo.core.startup import init_rooter, init_routing
from cuckoo.misc import cwd, decide_cwd
if cwd(root=True) is None:
decide_cwd(exists=True)
# Connect to MongoDB (mandatory).
if not mongo.init():
sys.exit(red(
"In order to use the Cuckoo Web Interface it is required to have "
"MongoDB up-and-running and enabled in Cuckoo. Please refer to our "
"official documentation as well as the $CWD/conf/reporting.conf file."
))
mongo.connect()
# Connect to ElasticSearch (optional).
elastic.init()
elastic.connect()
# In case we have VPNs enabled we need to initialize through the following
# two methods as they verify the interaction with VPNs as well as gather
# which VPNs are available (for representation upon File/URL submission).
