def test_mongo_connect_store_file(): set_cwd(tempfile.mkdtemp()) cuckoo_create(cfg={ "reporting": { "mongodb": { "enabled": True, "db": "cuckootest", }, }, }) mongo.init() assert mongo.database == "cuckootest" fd, filepath = tempfile.mkstemp() os.write(fd, "hello world") os.close(fd) f = File(filepath) r = MongoDB() r.init_once() id1 = r.store_file(f, "foobar.txt") id2 = r.store_file(f, "foobar.txt") assert id1 == id2 assert mongo.db.fs.files.find_one({ "sha256": f.get_sha256(), })["_id"] == id1 assert mongo.grid.get(id1).read() == "hello world"
def setup_class(cls): set_cwd(tempfile.mkdtemp()) cuckoo_create(cfg={ "reporting": { "mongodb": { "enabled": True, "db": "cuckootest", }, }, }) mongo.init() mongo.connect()
def init_once(cls): if not mongo.init(): return mongo.connect() cls.db = mongo.db cls.fs = mongo.grid # Set MongoDB schema version. if "cuckoo_schema" in mongo.collection_names: version = mongo.db.cuckoo_schema.find_one()["version"] if version != cls.SCHEMA_VERSION: raise CuckooReportError( "Unknown MongoDB version schema version found. Cuckoo " "doesn't really know how to proceed now.." ) else: mongo.db.cuckoo_schema.save({"version": cls.SCHEMA_VERSION}) # Set an unique index on stored files to avoid duplicates. As per the # pymongo documentation this is basically a no-op if the index already # exists. So we don't have to do that check ourselves. mongo.db.fs.files.ensure_index( "sha256", unique=True, sparse=True, name="sha256_unique" )
def test_mongodb_offline(p, q, r, s): set_cwd(tempfile.mkdtemp()) cuckoo_create(cfg={ "reporting": { "mongodb": { "enabled": True, }, }, }) r.side_effect = socket.error("error") db = p.return_value.__getitem__.return_value db.collection_names.side_effect = pymongo.errors.PyMongoError("error") with pytest.raises(CuckooCriticalError) as e: mongo.init() mongo.connect() e.match("Unable to connect to MongoDB") s.warning.assert_called_once()
def mongo_upgrade(): """Migrate mongodb schema and data.""" if mongo.init(): print "Starting MongoDB migration." mongo.connect() # Check for schema version and create it. if "cuckoo_schema" in mongo.db.collection_names(): print "Mongo schema version not expected" sys.exit() else: mongo.db.cuckoo_schema.save({"version": mongo_revision}) else: print "Mongo reporting module not enabled, skipping mongo migration."
def cuckoo_clean(): """Clean up cuckoo setup. It deletes logs, all stored data from file system and configured databases (SQL and MongoDB). """ # Init logging (without writing to file). init_console_logging() try: # Initialize the database connection. db = Database() db.connect(schema_check=False) # Drop all tables. db.drop() except (CuckooDependencyError, CuckooDatabaseError) as e: # If something is screwed due to incorrect database migrations or bad # database SqlAlchemy would be unable to connect and operate. log.warning( "Error connecting to database: it is suggested to check " "the connectivity, apply all migrations if needed or purge " "it manually. Error description: %s", e) # Check if MongoDB reporting is enabled and drop the database if it is. if mongo.init(): try: mongo.connect() mongo.drop() mongo.close() except Exception as e: log.warning("Unable to drop MongoDB database: %s", e) # Check if ElasticSearch reporting is enabled and drop its data if it is. if elastic.init(): elastic.connect() # TODO This should be moved to the elastic abstract. # TODO We should also drop historic data, i.e., from pervious days, # months, and years. date_index = datetime.datetime.utcnow().strftime({ "yearly": "%Y", "monthly": "%Y-%m", "daily": "%Y-%m-%d", }[elastic.index_time_pattern]) dated_index = "%s-%s" % (elastic.index, date_index) elastic.client.indices.delete(index=dated_index, ignore=[400, 404]) template_name = "%s_template" % dated_index if elastic.client.indices.exists_template(template_name): elastic.client.indices.delete_template(template_name) # Paths to clean. paths = [ cwd("cuckoo.db"), cwd("log"), cwd("storage", "analyses"), cwd("storage", "baseline"), cwd("storage", "binaries"), ] # Delete the various files and directories. In case of directories, keep # the parent directories, so to keep the state of the CWD in tact. for path in paths: if os.path.isdir(path): try: shutil.rmtree(path) os.mkdir(path) except (IOError, OSError) as e: log.warning("Error removing directory %s: %s", path, e) elif os.path.isfile(path): try: os.unlink(path) except (IOError, OSError) as e: log.warning("Error removing file %s: %s", path, e)
def cuckoo_clean(): """Clean up cuckoo setup. It deletes logs, all stored data from file system and configured databases (SQL and MongoDB). """ # Init logging (without writing to file). init_console_logging() try: # Initialize the database connection. db = Database() db.connect(schema_check=False) # Drop all tables. db.drop() except (CuckooDependencyError, CuckooDatabaseError) as e: # If something is screwed due to incorrect database migrations or bad # database SqlAlchemy would be unable to connect and operate. log.warning("Error connecting to database: it is suggested to check " "the connectivity, apply all migrations if needed or purge " "it manually. Error description: %s", e) # Check if MongoDB reporting is enabled and drop the database if it is. if mongo.init(): try: mongo.connect() mongo.drop() mongo.close() except Exception as e: log.warning("Unable to drop MongoDB database: %s", e) # Check if ElasticSearch reporting is enabled and drop its data if it is. if elastic.init(): elastic.connect() # TODO This should be moved to the elastic abstract. # TODO We should also drop historic data, i.e., from pervious days, # months, and years. date_index = datetime.datetime.utcnow().strftime({ "yearly": "%Y", "monthly": "%Y-%m", "daily": "%Y-%m-%d", }[elastic.index_time_pattern]) dated_index = "%s-%s" % (elastic.index, date_index) elastic.client.indices.delete( index=dated_index, ignore=[400, 404] ) template_name = "%s_template" % dated_index if elastic.client.indices.exists_template(template_name): elastic.client.indices.delete_template(template_name) # Paths to clean. paths = [ cwd("cuckoo.db"), cwd("log"), cwd("storage", "analyses"), cwd("storage", "baseline"), cwd("storage", "binaries"), ] # Delete the various files and directories. In case of directories, keep # the parent directories, so to keep the state of the CWD in tact. for path in paths: if os.path.isdir(path): try: shutil.rmtree(path) os.mkdir(path) except (IOError, OSError) as e: log.warning("Error removing directory %s: %s", path, e) elif os.path.isfile(path): try: os.unlink(path) except (IOError, OSError) as e: log.warning("Error removing file %s: %s", path, e)
import os import sys from django.template.base import TemplateSyntaxError from cuckoo.common.colors import red from cuckoo.common.elastic import elastic from cuckoo.common.mongo import mongo from cuckoo.core.startup import init_rooter, init_routing from cuckoo.misc import cwd, decide_cwd if cwd(root=True) is None: decide_cwd(exists=True) # Connect to MongoDB (mandatory). if not mongo.init(): sys.exit( red("In order to use the Cuckoo Web Interface it is required to have " "MongoDB up-and-running and enabled in Cuckoo. Please refer to our " "official documentation as well as the $CWD/conf/reporting.conf file." )) mongo.connect() # Connect to ElasticSearch (optional). elastic.init() elastic.connect() # In case we have VPNs enabled we need to initialize through the following # two methods as they verify the interaction with VPNs as well as gather # which VPNs are available (for representation upon File/URL submission).
import os import sys from django.template.base import TemplateSyntaxError from cuckoo.common.colors import red from cuckoo.common.elastic import elastic from cuckoo.common.mongo import mongo from cuckoo.core.startup import init_rooter, init_routing from cuckoo.misc import cwd, decide_cwd if cwd(root=True) is None: decide_cwd(exists=True) # Connect to MongoDB (mandatory). if not mongo.init(): sys.exit(red( "In order to use the Cuckoo Web Interface it is required to have " "MongoDB up-and-running and enabled in Cuckoo. Please refer to our " "official documentation as well as the $CWD/conf/reporting.conf file." )) mongo.connect() # Connect to ElasticSearch (optional). elastic.init() elastic.connect() # In case we have VPNs enabled we need to initialize through the following # two methods as they verify the interaction with VPNs as well as gather # which VPNs are available (for representation upon File/URL submission).