def enumerate_plugins(dirpath, module_prefix, namespace, class_, attributes={}, as_dict=False): """Import plugins of type `class` located at `dirpath` into the `namespace` that starts with `module_prefix`. If `dirpath` represents a filepath then it is converted into its containing directory. The `attributes` dictionary allows one to set extra fields for all imported plugins. Using `as_dict` a dictionary based on the module name is returned.""" if os.path.isfile(dirpath): dirpath = os.path.dirname(dirpath) for fname in os.listdir(dirpath): if fname.endswith(".py") and not fname.startswith("__init__"): module_name, _ = os.path.splitext(fname) try: importlib.import_module( "%s.%s" % (module_prefix, module_name) ) except ImportError as e: raise CuckooOperationalError( "Unable to load the Cuckoo plugin at %s: %s. Please " "review its contents and/or validity!" % (fname, e) ) subclasses = class_.__subclasses__()[:] plugins = [] while subclasses: subclass = subclasses.pop(0) # Include subclasses of this subclass (there are some subclasses, e.g., # LibVirtMachinery, that fail the fail the following module namespace # check and as such we perform this logic here). subclasses.extend(subclass.__subclasses__()) # Check whether this subclass belongs to the module namespace that # we're currently importing. It should be noted that parent and child # namespaces should fail the following if-statement. if module_prefix != ".".join(subclass.__module__.split(".")[:-1]): continue namespace[subclass.__name__] = subclass for key, value in attributes.items(): setattr(subclass, key, value) plugins.append(subclass) if as_dict: ret = {} for plugin in plugins: ret[plugin.__module__.split(".")[-1]] = plugin return ret return sorted(plugins, key=lambda x: x.__name__.lower())
def delete(*folder): """Delete a folder and all its subdirectories. @param folder: path or components to path to delete. @raise CuckooOperationalError: if fails to delete folder. """ folder = os.path.join(*folder) if os.path.exists(folder): try: shutil.rmtree(folder) except OSError: raise CuckooOperationalError("Unable to delete folder: %s" % folder)
def _dburi_engine(dburi): # Defaults to a sqlite3 database. if not dburi: dburi = "sqlite:///db/cuckoo.db" try: return sqlalchemy.create_engine(dburi).engine except sqlalchemy.exc.ArgumentError: raise CuckooOperationalError( "Error creating SQL database backup as your SQL database URI " "wasn't understood by us: %r!" % dburi )
def netlog_sanitize_fname(path): """Validate agent-provided path for result files""" path = path.replace("\\", "/") dir_part, name = os.path.split(path) if dir_part not in RESULT_UPLOADABLE: raise CuckooOperationalError( "Netlog client requested banned path: %r" % path) if any(c in BANNED_PATH_CHARS for c in name): for c in BANNED_PATH_CHARS: path = path.replace(c, "X") return path
def fetch_community(branch="master", force=False, filepath=None): if filepath: buf = open(filepath, "rb").read() else: log.info("Downloading.. %s", URL % branch) r = requests.get(URL % branch) if r.status_code != 200: raise CuckooOperationalError( "Error fetching the Cuckoo Community binaries " "(status_code: %d)!" % r.status_code) buf = r.content t = tarfile.TarFile.open(fileobj=io.BytesIO(buf), mode="r:gz") folders = { "modules/signatures": "signatures", "data/monitor": "monitor", "agent": "agent", "analyzer": "analyzer", } members = t.getmembers() directory = members[0].name.split("/")[0] for tarfolder, outfolder in folders.items(): mkdir(cwd(outfolder)) # E.g., "community-master/modules/signatures". name_start = "%s/%s" % (directory, tarfolder) for member in members: if not member.name.startswith(name_start) or \ name_start == member.name: continue filepath = cwd(outfolder, member.name[len(name_start) + 1:]) if member.isdir(): mkdir(filepath) continue # TODO Ask for confirmation as we used to do. if os.path.exists(filepath) and not force: log.debug("Not overwriting file which already exists: %s", member.name[len(name_start) + 1:]) continue if member.issym(): t.makelink(member, filepath) continue log.debug("Extracted %s..", member.name[len(name_start) + 1:]) open(filepath, "wb").write(t.extractfile(member).read())
def connect(self): if not self.enabled: return try: self.client = pymongo.MongoClient(self.hostname, self.port) self.db = self.client[self.database] if self.username and self.password: self.db.authenticate(self.username, self.password) self.grid = gridfs.GridFS(self.db) except pymongo.errors.PyMongoError as e: raise CuckooOperationalError( "Unable to connect to MongoDB: %s" % e )
def _request_hash(self, file_hash, **kwargs): """Wrapper around requesting a hash.""" params = dict(hash=file_hash, apikey=self.apikey) try: r = requests.get(self.HASH_DOWNLOAD, params=params, timeout=self.timeout, **kwargs) r.raise_for_status() # raise an exception for HTTP error codes return r.content except (requests.ConnectionError, ValueError, requests.HTTPError): raise CuckooOperationalError("Could not fetch hash \"%s\" " "from VirusTotal" % file_hash)
def read_newline(self): """Read until the next newline character, but never more than `MAX_NETLOG_LINE`.""" while True: pos = self.buf.find("\n") if pos < 0: if len(self.buf) >= MAX_NETLOG_LINE: raise CuckooOperationalError("Received overly long line") buf = self.read() if buf == "": raise EOFError self.buf += buf continue line, self.buf = self.buf[:pos], self.buf[pos + 1:] return line
def init(self): """ Initialization method to determine if the OpenDXL Cuckoo Reporting Module is enabled and to get the OpenDXL Python Client config file location from the dxl_client_config_file setting under the [dxleventreporting] section of the reporting.conf file. :return: A boolean indicating if the OpenDXL Cuckoo Reporting Module is enabled or not. """ self.enabled = config("reporting:dxleventreporting:enabled") dxl_client_config_file = config( "reporting:dxleventreporting:dxl_client_config_file") if dxl_client_config_file is None: raise CuckooOperationalError( "Missing dxl_client_config_file setting under the " "[dxleventreporting] section in the report.conf file.") self.config = DxlClientConfig.create_dxl_config_from_file( dxl_client_config_file) return self.enabled
def negotiate_protocol(self): protocol = self.read_newline(strip=True) # Command with version number. if " " in protocol: command, version = protocol.split() version = int(version) else: command, version = protocol, None if command == "BSON": self.protocol = BsonParser(self, version) elif command == "FILE": self.protocol = FileUpload(self, version) elif command == "LOG": self.protocol = LogHandler(self, version) else: raise CuckooOperationalError( "Netlog failure, unknown protocol requested.") self.protocol.init()
def handle(self): # Read until newline for file path, e.g., # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin self.handler.sock.settimeout(30) dump_path = netlog_sanitize_fname(self.handler.read_newline()) if self.version and self.version >= 2: # NB: filepath is only used as metadata filepath = self.handler.read_newline() pids = map(int, self.handler.read_newline().split()) else: filepath, pids = None, [] log.debug("Task #%s: File upload for %r", self.task_id, dump_path) file_path = os.path.join(self.storagepath, dump_path.decode("utf-8")) try: self.fd = open_exclusive(file_path) except OSError as e: if e.errno == errno.EEXIST: raise CuckooOperationalError("Analyzer for task #%s tried to " "overwrite an existing file" % self.task_id) raise # Append-writes are atomic with open(self.filelog, "a+b") as f: print(json.dumps({ "path": dump_path, "filepath": filepath, "pids": pids, }), file=f) self.handler.sock.settimeout(None) try: return self.handler.copy_to_fd(self.fd, self.upload_max_size) finally: log.debug("Task #%s uploaded file length: %s", self.task_id, self.fd.tell())
def create(root=".", folders=None): """Creates a directory or multiple directories. @param root: root path. @param folders: folders list to be created. @raise CuckooOperationalError: if fails to create folder. If folders is None, we try to create the folder provided by `root`. """ if isinstance(root, (tuple, list)): root = os.path.join(*root) if folders is None: folders = [""] elif isinstance(folders, basestring): folders = folders, for folder in folders: folder_path = os.path.join(root, folder) if not os.path.isdir(folder_path): try: os.makedirs(folder_path) except OSError: raise CuckooOperationalError( "Unable to create folder: %s" % folder_path)
def connect(self): """ A method to have the OpenDXL Python Client connect to a DXL Broker listed in the OpenDXL Python Client configuration client. """ if not self.enabled: return try: log.info("Creating DXL Client") if not self.client: self.client = DxlClient(self.config) self.client.connect() elif not self.client.connected: self.client.connect() log.info("Connected to a broker") except Exception as ex: log.exception( "Error creating DXL Client and connecting to a DXL Broker.") raise CuckooOperationalError( "Error creating DXL Client and connecting to a DXL Broker: %s" % ex)
def dumpcmd(dburi, dirpath): engine = _dburi_engine(dburi) if engine.name == "sqlite": # If the SQLite3 database filepath is relative, then make it relative # against the old Cuckoo setup. If it's absolute, os.path.join() will # keep it absolute as-is (see also our version 1.1.1 release :-P). return [ "sqlite3", os.path.join(dirpath, engine.url.database), ".dump" ], {} if engine.name == "mysql": args = ["mysqldump"] if engine.url.username: args += ["-u", engine.url.username] if engine.url.password: args.append("-p%s" % engine.url.password) if engine.url.host and engine.url.host != "localhost": args += ["-h", engine.url.host] args.append(engine.url.database) return args, {} if engine.name == "postgresql": args, env = ["pg_dump"], {} if engine.url.username: args += ["-U", engine.url.username] if engine.url.password: env["PGPASSWORD"] = engine.url.password if engine.url.host and engine.url.host != "localhost": args += ["-h", engine.url.host] args.append(engine.url.database) return args, env raise CuckooOperationalError( "Error creating SQL database backup as your SQL database URI " "wasn't understood by us: %r!" % dburi)
def stop(self): """Stop sniffing. @return: operation status. """ # The tcpdump process was never started in the first place. if not self.proc: return # The tcpdump process has already quit, generally speaking this # indicates an error such as "permission denied". if self.proc.poll(): out, err = self.proc.communicate() raise CuckooOperationalError( "Error running tcpdump to sniff the network traffic during " "the analysis; stdout = %r and stderr = %r. Did you enable " "the extra capabilities to allow running tcpdump as non-root " "user and disable AppArmor properly (the latter only applies " "to Ubuntu-based distributions with AppArmor, see also %s)?" % (out, err, faq("permission-denied-for-tcpdump")) ) try: self.proc.terminate() except: try: if not self.proc.poll(): log.debug("Killing sniffer") self.proc.kill() except OSError as e: log.debug("Error killing sniffer: %s. Continue", e) except Exception as e: log.exception("Unable to stop the sniffer with pid %d: %s", self.proc.pid, e) # Ensure expected output was received from tcpdump. out, err = self.proc.communicate() self._check_output(out, err)
def sqldump(dburi, dirpath): args, env = dumpcmd(dburi, dirpath) envargs = " ".join("%s=%s" % (k, v) for k, v in env.items()) cmdline = " ".join('"%s"' % arg if " " in arg else arg for arg in args) cmd = "%s %s" % (envargs, cmdline) if envargs else cmdline print "We can make a SQL database backup as follows:" print "input cmd =>", cmd print "output SQL =>", cwd("backup.sql") if not click.confirm("Would you like to make a backup", default=True): return try: subprocess.check_call(args, stdout=open(cwd("backup.sql"), "wb"), env=dict(os.environ.items() + env.items())) except (subprocess.CalledProcessError, OSError) as e: raise CuckooOperationalError( "Error creating SQL database dump as the command returned an " "error code: %s. Please make sure that the required tooling " "for making a database backup is installed and review the " "database URI to make sure it's correct: %s!" % (e, dburi))
def import_cuckoo(username, mode, dirpath): version = identify(dirpath) if not version: raise CuckooOperationalError( "The path that you specified is not a proper Cuckoo setup. Please " "point the path to the root of your older Cuckoo setup, i.e., to " "the directory containing the cuckoo.py script!") # TODO Copy over the configuration and ignore the database. if version in ("0.4", "0.4.1", "0.4.2"): raise CuckooOperationalError( "Importing from version 0.4, 0.4.1, or 0.4.2 is not supported as " "there are no database migrations for that version. Please start " "from scratch, your configuration would have been obsolete anyway!" ) print "We've identified a Cuckoo Sandbox %s installation!" % version if os.path.isdir(cwd()) and os.listdir(cwd()): raise CuckooOperationalError( "This Cuckoo Working Directory already exists. Please import to " "a new/clean Cuckoo Working Directory.") # Following are various recursive imports. from cuckoo.apps import migrate_database from cuckoo.main import cuckoo_create print "Reading in the old configuration.." # Port the older configuration. cfg = Config.from_confdir(os.path.join(dirpath, "conf"), loose=True) cfg = migrate_conf(cfg, version) print " configuration has been migrated to the latest version!" print # Create a fresh Cuckoo Working Directory. cuckoo_create(username, cfg, quiet=True) dburi = cfg["cuckoo"]["database"]["connection"] # Ask if the user would like to make a backup of the SQL database and in # the case of sqlite3, copy/move/symlink cuckoo.db to the CWD. sqldump(dburi, dirpath) movesql(dburi, mode, dirpath) # Run database migrations. if not migrate_database(): raise CuckooOperationalError( "Error migrating your old Cuckoo database!") # Link or copy all of the older results to the new CWD. import_legacy_analyses(mode, dirpath) # Urge the user to run the community command. print print "You have successfully imported your old version of Cuckoo!" print "However, in order to get up-to-date, you'll probably want to" print yellow("run the community command"), print "by running", red("'cuckoo community'"), "manually." print "The community command will fetch the latest monitoring updates" print "and Cuckoo Signatures."
def import_(self, f, submit_id): """Import an analysis identified by the file(-like) object f.""" try: z = zipfile.ZipFile(f) except zipfile.BadZipfile: raise CuckooOperationalError( "Imported analysis is not a proper .zip file.") # Ensure there are no files with illegal or potentially insecure names. # TODO Keep in mind that if we start to support other archive formats # (e.g., .tar) that those may also support symbolic links. In that case # we should probably start using sflock here. for filename in z.namelist(): if filename.startswith("/") or ".." in filename or ":" in filename: raise CuckooOperationalError( "The .zip file contains a file with a potentially " "incorrect filename: %s" % filename) if "task.json" not in z.namelist(): raise CuckooOperationalError( "The task.json file is required in order to be able to import " "an analysis! This file contains metadata about the analysis.") required_fields = { "options": dict, "route": basestring, "package": basestring, "target": basestring, "category": basestring, "memory": bool, "timeout": (int, long), "priority": (int, long), "custom": basestring, "tags": (tuple, list), } try: info = json.loads(z.read("task.json")) for key, type_ in required_fields.items(): if key not in info: raise ValueError("missing %s" % key) if info[key] is not None and not isinstance(info[key], type_): raise ValueError("%s => %s" % (key, info[key])) except ValueError as e: raise CuckooOperationalError( "The provided task.json file, required for properly importing " "the analysis, is incorrect or incomplete (%s)." % e) if info["category"] == "url": task_id = submit_task.add_url(url=info["target"], package=info["package"], timeout=info["timeout"], options=info["options"], priority=info["priority"], custom=info["custom"], memory=info["memory"], tags=info["tags"], submit_id=submit_id) else: # Users may have the "delete_bin_copy" enabled and in such cases # the binary file won't be included in the .zip file. if "binary" in z.namelist(): filepath = Files.temp_named_put( z.read("binary"), os.path.basename(info["target"])) else: # Generate a temp file as a target if no target is present filepath = Files.temp_put("") # We'll be updating the target shortly. task_id = submit_task.add_path(file_path=filepath, package=info["package"], timeout=info["timeout"], options=info["options"], priority=info["priority"], custom=info["custom"], memory=info["memory"], tags=info["tags"], submit_id=submit_id) if not task_id: raise CuckooOperationalError( "There was an error creating a task for the to-be imported " "analysis in our database.. Can't proceed.") # The constructors currently don't accept this argument. db.set_route(task_id, info["route"]) mkdir(cwd(analysis=task_id)) z.extractall(cwd(analysis=task_id)) # If there's an analysis.json file, load it up to figure out additional # metdata regarding this analysis. if os.path.exists(cwd("analysis.json", analysis=task_id)): try: obj = json.load( open(cwd("analysis.json", analysis=task_id), "rb")) if not isinstance(obj, dict): raise ValueError if "errors" in obj and not isinstance(obj["errors"], list): raise ValueError if "action" in obj and not isinstance(obj["action"], list): raise ValueError except ValueError: log.warning( "An analysis.json file was provided, but wasn't a valid " "JSON object/structure that we can to enhance the " "analysis information.") else: for error in set(obj.get("errors", [])): if isinstance(error, basestring): db.add_error(error, task_id) for action in set(obj.get("action", [])): if isinstance(action, basestring): db.add_error("", task_id, action) # We set this analysis as completed so that it will be processed # automatically (assuming 'cuckoo process' is running). db.set_status(task_id, TASK_COMPLETED) return task_id
def __iter__(self): # Read until newline for file path, e.g., # shots/0001.jpg or files/9498687557/libcurl-4.dll.bin dump_path = self.handler.read_newline(strip=True).replace("\\", "/") if self.version >= 2: filepath = self.handler.read_newline(strip=True) pids = map(int, self.handler.read_newline(strip=True).split()) else: filepath, pids = None, [] log.debug("File upload request for %s", dump_path) dir_part, filename = os.path.split(dump_path) if "./" in dump_path or not dir_part or dump_path.startswith("/"): raise CuckooOperationalError( "FileUpload failure, banned path: %s" % dump_path) for restricted in self.RESTRICTED_DIRECTORIES: if restricted in dir_part: raise CuckooOperationalError( "FileUpload failure, banned path.") try: Folders.create(self.storagepath, dir_part) except CuckooOperationalError: log.error("Unable to create folder %s", dir_part) return file_path = os.path.join(self.storagepath, dump_path) if not file_path.startswith(self.storagepath): raise CuckooOperationalError( "FileUpload failure, path sanitization failed.") if os.path.exists(file_path): log.warning("Analyzer tried to overwrite an existing file, " "closing connection.") return self.fd = open(file_path, "wb") chunk = self.handler.read_any() while chunk: self.fd.write(chunk) if self.fd.tell() >= self.upload_max_size: log.warning( "Uploaded file length larger than upload_max_size, " "stopping upload.") self.fd.write("... (truncated)") break try: chunk = self.handler.read_any() except: break self.lock.acquire() with open(self.filelog, "a+b") as f: f.write("%s\n" % json.dumps({ "path": dump_path, "filepath": filepath, "pids": pids, })) self.lock.release() log.debug("Uploaded file length: %s", self.fd.tell()) return yield
def init_config(self): """Create a volatility configuration.""" if self.config is not None and self.addr_space is not None: return if not self.osprofile: raise CuckooOperationalError( "Can't continue to process the VM memory dump if no OS " "profile has been defined for it. One may define its OS " "profile using the 'osprofile' field for the VM in its " "machinery configuration or set a global default using " "'guest_profile' in memory.conf") if self.osprofile not in self.profiles: raise CuckooOperationalError( "The profile '%s' does not exist! Please pick one of the " "following profiles for your VMs: %s" % (self.osprofile, ", ".join(sorted(self.profiles)))) self.config = conf.ConfObject() self.config.optparser.set_conflict_handler("resolve") registry.register_global_options(self.config, commands.Command) base_conf = { "profile": self.osprofile, "use_old_as": None, "kdbg": None, "help": False, "kpcr": None, "tz": None, "pid": None, "output_file": None, "physical_offset": None, "conf_file": None, "dtb": None, "output": None, "info": None, "location": "file://%s" % self.memdump, "plugins": None, "debug": None, "cache_dtb": True, "filename": None, "cache_directory": None, "verbose": None, "write": False } for key, value in base_conf.items(): self.config.update(key, value) # Deal with Volatility support for KVM/qemu memory dump. # See: #464. try: self.addr_space = utils.load_as(self.config) except exc.AddrSpaceError as e: if self.get_dtb(): self.addr_space = utils.load_as(self.config) elif "No suitable address space mapping found" in e.message: raise CuckooOperationalError( "An incorrect OS has been specified for this machine! " "Please provide the correct one or Cuckoo won't be able " "to provide Volatility-based results for analyses with " "this VM.") else: raise self.plugins = (registry.get_plugin_classes(commands.Command, lower=True))