def _read_queue(self): """ .. py:function:: _read_queue(self) Main loop that processes the match(es) from the :code:`multiprocessing.Queue` instance. :param self: current class instance :type self: class """ while True: item = self.queue.get() if item == _codes.DONE: break self.map[self.target["format"]](item) with self.results[0]: self.results[1].value += 1 self.results[2].append(item["target"]["identifier"]) _log.debug( "Matching signature from rule <{}> on evidence <{}>.".format( item["match"]["rule"], item["target"]["identifier"]))
def __init__(self, arguments): """ .. py:function:: __init__(self, arguments) Initialization method for the class. :param self: current class instance :type self: class :param arguments: :code:`argparse.Parser` instance containing the processed command-line arguments :type arguments: list """ self.arguments = arguments self.name = os.path.basename(self.arguments.output) self.resources = { "case": self.arguments.output, "matches": os.path.join(self.arguments.output, "{}.{}".format(_conf.MATCHES_FILE_BASENAME, self.arguments.format.lower())), "evidences": { "files": [], "processes": [] }, "temporary": [] } _log.debug("Initialized new case <{}> anchored to <{}>.".format(self.name, self.resources["case"]))
def _create_local_directory(self, directory, mask=0o700): """ .. py:function:: _create_local_directory(self, directory, mask=0o700) Creates a directory on the filesystem. :param self: current class instance :type self: class :param directory: absolute path to the directory to create :type directory: str :param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary :type mask: oct :return: random string of :code:`rounds` characters :rtype: str """ try: os.makedirs(directory, mode=mask) _log.debug("Created local directory <{}>.".format(directory)) except FileExistsError: _log.fault("Failed to create local directory due to existing object <{}>.".format(directory), trace=True) except ( OSError, Exception): _log.fault("Failed to create local directory <{}>.".format(directory), trace=True)
def create_local_directory(directory, mask=0o700): """ .. py:function:: create_local_directory(directory, mask=0o700) Creates a local case directory on the filesystem. :param directory: absolute path to the directory to create :type directory: str :param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary :type mask: oct """ try: os.makedirs(directory, mode=mask) _log.debug("Created local directory <{}>.".format(directory)) except FileExistsError: _log.fault("Failed to create local directory due to existing object <{}>.".format(directory), post_mortem=True) except ( OSError, Exception): _log.fault("Failed to create local directory <{}>.".format(directory), post_mortem=True)
def __exit__(self, *args): """ .. py:function:: __exit__(self, *args) Exit method raised when leaving the context manager. :param self: current class instance :type self: class :param *args: list of argument(s) :type *args: class """ _log.debug("Ended <{}> session <{}>.".format(self.module.__class__.__name__, self.module.__name__))
def __init__(self, module): """ .. py:function:: __init__(self, module) Initialization method for the class. :param self: current class instance :type self: class :param module: class inherited from the :code:`models` reference classes :type module: class """ self.module = module _log.debug("Started <{}> session <{}>.".format(self.module.__class__.__name__, self.module.__name__))
def _dispatch_jobs(self): """ .. py:function:: _dispatch_jobs(self) Dispatches the processing task(s) to the subprocess(es). :param self: current class instance :type self: class :return: number of match(es) :rtype: int """ with multiprocessing.Manager() as manager: queue = manager.Queue() results = (multiprocessing.Lock(), multiprocessing.Value(ctypes.c_int, 0), manager.list()) reader = multiprocessing.Process(target=_reader.Reader(queue, results, { "target": self.case.resources["matches"], "storage": self.case.resources["storage"], "format": self.case.arguments.format }).run) reader.daemon = True reader.start() _log.debug("Started reader subprocess to consume queue result(s).") with _magic.Pool(processes=self.case.arguments.processes) as pool: for file in self.case.resources["evidences"]: if os.path.getsize(file) > self.case.arguments.max_size: _log.warning("Evidence <{}> exceeds the maximum size. Ignoring evidence. Try changing --max-size to override this behavior.".format(file)) continue pool.starmap_async( _processors.File(self.case.arguments.hash_algorithms, self.case.arguments.callbacks, queue, self.case.arguments.fast).run, [(file, self.buffers)], error_callback=_log.inner_exception) _log.debug("Mapped concurrent job to consume evidence <{}>.".format(file)) queue.put(_codes.DONE) with _magic.Hole(KeyboardInterrupt, action=lambda:_log.fault("Aborted due to manual user interruption <SIGINT>.")): reader.join() return results[1].value
def __init__(self, processes=(multiprocessing.cpu_count() or _conf.FALLBACK_PROCESSES)): """ .. py:function:: __init__(self, processes=(multiprocessing.cpu_count() or _conf.FALLBACK_PROCESSES)) Initialization method for the class. :param self: current class instance :type self: class :param exception: number of concurrent process(es) to spawn :type exception: int """ self.processes = processes self.pool = multiprocessing.Pool(processes=self.processes, initializer=self._worker_initializer) _log.debug("Initialized pool of <{}> concurrent process(es).".format(self.processes))
def _compile_ruleset(self, name, ruleset): """ .. py:function:: _compile_ruleset(self, name, ruleset) Compiles and saves YARA rule(s) to the dictionary to be passed to the asynchronous job(s). :param self: current class instance :type self: class :param name: name of the ruleset file to compile the rule(s) from :type name: str :param ruleset: absolute path to the ruleset file to compile the rule(s) from :type ruleset: str :return: tuple containing the final status of the compilation and the number of successfully loaded rule(s) :rtype: bool, int """ count = 0 try: buffer = io.BytesIO() rules = yara.compile(ruleset, includes=_conf.YARA_INCLUDES, error_on_warning=(not self.case.arguments.ignore_warnings)) rules.save(file=buffer) self.buffers[ruleset] = buffer count += sum(1 for _ in rules) _log.debug("Precompilated YARA ruleset <{}> in memory with a total of <{}> valid rule(s).".format(name, count)) return True, count except yara.SyntaxError: _log.exception("Syntax error in YARA ruleset <{}>.".format(ruleset)) except ( Exception, yara.Error): _log.exception("Failed to pre-compile ruleset <{}>.".format(ruleset)) return False, count
def track_file(self, evidence): """ .. py:function:: track_file(self, evidence) Checks and registers an evidence file for processing. :param self: current class instance :type self: class :param evidence: absolute path to the evidence file :type evidence: str """ if os.path.isfile(evidence): self.resources["evidences"]["files"].append(evidence) _log.debug("Tracking file <{}>.".format(evidence)) else: _log.warning("Evidence <{}> not found or invalid.".format(evidence))
def _store_matching_evidences(self): """ .. py:function:: _store_matching_evidences(self) Saves the matching evidence(s) to the specified storage directory. :param self: current class instance :type self: class """ for evidence in self.results[2]: if not os.path.isdir(self.target["storage"]): _fs.create_local_directory(self.target["storage"]) try: storage_path = ( os.path.join(self.target["storage"], os.path.basename(evidence)) if not _conf.NEUTRALIZE_MATCHING_EVIDENCES else os.path.join( self.target["storage"], "{}.{}".format( os.path.basename(evidence), _meta.__package__))) shutil.copy2(evidence, storage_path) if _conf.NEUTRALIZE_MATCHING_EVIDENCES: os.chmod( storage_path, stat.S_IMODE(os.lstat(storage_path).st_mode) & ~stat.S_IEXEC) _log.debug("Saved {}matching evidence <{}> as <{}>.".format( "and neutralized " if _conf.NEUTRALIZE_MATCHING_EVIDENCES else "", os.path.basename(evidence), storage_path)) except (OSError, shutil.Error, Exception): _log.exception( "Failed to save matching evidence <{}> as <{}>.".format( os.path.basename(evidence), storage_path))
def run(self): """ .. py:function:: run(self) Main entry point for the module. :param self: current class instance :type self: class """ if self.case.arguments._inline_password: _log.debug( "Using inline password <{}> to unpack archive(s).".format( self.case.arguments._inline_password)) elif self.case.arguments._password: self._password = _interaction.password_prompt( "Unpacking password: "******"Recursive unpacking manually disabled using --no-recursion.") tmp = self.case.require_temporary_directory() for evidence in self.feed: self.recursive_inflate( evidence, tmp, password=(self.case.arguments._inline_password if not hasattr(self, "_password") else self._password)) for evidence in _fs.expand_files([tmp], recursive=True, include=self.case.arguments._include, exclude=self.case.arguments._exclude): print("found {}".format(evidence))
def run(self): """ .. py:function:: run(self) Main entry point for the module. :param self: current class instance :type self: class """ tmp = self.case.require_temporary_directory() for item in self.case.arguments.input: if os.path.isfile(item): _log.debug("Tracking file <{}> to <{}>.".format(file, tmp)) elif os.path.isdir(item): _log.warning( "Directory <{}> is not an archive. Ignoring.".format(item)) else: _log.warning( "Unknown inode type for object <{}>.".format(item))
def _tear_down(self): """ .. py:function:: _tear_down(self) Cleanup method called on class destruction that gets rid of the temporary artifact(s). :param self: current class instance :type self: class """ for artifact in self.resources["temporary"]: try: shutil.rmtree(artifact) _log.debug("Removed temporary artifact <{}>.".format(artifact)) except FileNotFoundError: _log.debug("Temporary artifact not found <{}>.".format(artifact)) except ( OSError, Exception): _log.exception("Failed to remove temporary artifact <{}>.".format(artifact))
def track_process(self, pid): """ .. py:function:: track_process(self, pid) Checks wether a process exists on the local machine and registers it for processing. :param self: current class instance :type self: class :param pid: process identifier :type pid: int """ if not isinstance(pid, int): _log.error("Invalid PID format <{}>.".format(pid)) return if psutil.pid_exists(pid): self.resources["evidences"]["processes"].append(pid) _log.debug("Tracking live process matching PID <{}>.".format(pid)) else: _log.warning("Process <{}> not found.".format(pid))
def track_files(self, evidences, include=[], exclude=[]): """ .. py:function:: track_files(self, evidences) Checks and registers multiple evidence files for processing. :param self: current class instance :type self: class :param evidences: list of absolute path(s) to the evidence file(s) :type evidences: list :param include: list of wildcard pattern(s) to include :type include: list :param exclude: list of wildcard pattern(s) to exclude :type exclude: list """ evidences = [os.path.abspath(evidence) for evidence in evidences] for evidence in self._iterate_existing_files(evidences): if include and not _fs.matches_patterns(os.path.basename(evidence), wildcard_patterns=include): _log.debug( "Ignoring evidence <{}> not matching inclusion pattern(s) <{}>." .format(evidence, include)) continue if exclude and _fs.matches_patterns(os.path.basename(evidence), wildcard_patterns=exclude): _log.debug( "Ignoring evidence <{}> matching exclusion pattern(s) <{}>." .format(evidence, exclude)) continue self.track_file(evidence)
def recursive_inflate(self, archive, output_directory, level=0, password=None): if level > self.case.arguments._level: _log.warning( "Limit unpacking level <{}> exceeded. Stopped unpacking.". format(self.case.arguments._level)) return _log.debug( "Inflating {}archive <{}> to temporary directory <{}>.".format( "level {} sub".format(level) if level else "base ", archive, output_directory)) sub_directory = os.path.join(output_directory, os.path.basename(archive)) try: with zipfile.ZipFile(archive) as z: z.extractall(path=sub_directory, pwd=(password.encode() if password else password)) except zipfile.BadZipFile: _log.error( "Bad file header. Cannot inflate evidence <{}>. Try to filter out non-zip file(s) using --include \"*.zip\" \".*.zip\"." .format(archive)) return except RuntimeError as exc: if "password required" in str(exc): _log.error( "Archive <{}> seems to be encrypted. Please specify a password using --password or --inline-password." .format(archive)) elif "Bad password" in str(exc): _log.error( "Password {}seems to be incorrect for archive <{}>. Please specify another password using --password or --inline-password." .format( "<{}> ".format(self.case.arguments._inline_password) if not hasattr(self, "_password") else "", archive)) else: _log.exception( "Runtime exception raised while unpacking archive <{}>.". format(archive)) return except KeyboardInterrupt: sys.stderr.write("\n") _log.fault("Aborted due to manual user interruption.") except Exception: _log.exception( "Exception raised while unpacking archive <{}>.".format( archive)) if self.case.arguments._no_recursion: return for subarchive in _fs.enumerate_matching_files( sub_directory, wildcard_patterns=([ "*.{}".format(_) for _ in self.__associations__["extensions"] ] + [ ".*.{}".format(_) for _ in self.__associations__["extensions"] ] if hasattr(self, "__associations__") and "extensions" in self.__associations__ else None), mime_types=(self.__associations__["mime"] if hasattr(self, "__associations__") and "mime" in self.__associations__ else None), recursive=True): self.recursive_inflate(subarchive, sub_directory, level=(level + 1), password=password)
def run(self): """ .. py:function:: run(self) Main entry point for the module. :param self: current class instance :type self: class """ tmp = self.case.require_temporary_directory() for evidence in self.feed: try: mail = eml_parser.eml_parser.decode_email( evidence, include_raw_body=True, include_attachment_data=True) _log.info("Extracted <{}> attachment(s) from <{}>.".format( len(mail["attachment"]), evidence)) except Exception: _log.exception( "Failed to extract data from <{}>. Ignoring evidence.". format(evidence)) continue output_directory = os.path.join(tmp, os.path.basename(evidence)) if not os.path.isdir(output_directory): _fs.create_local_directory(output_directory) for attachment in mail["attachment"]: if not attachment["filename"]: attachment["filename"] = idx if not _fs.matches_patterns(attachment["filename"], self.case.arguments._include): _log.warning( "Ignoring attachment <{}> not matching inner inclusion pattern(s)." .format(attachment["filename"])) continue if _fs.matches_patterns(attachment["filename"], self.case.arguments._exclude): _log.warning( "Ignoring attachment <{}> matching inner exclusion pattern(s)." .format(attachment["filename"])) continue output_path = os.path.join(output_directory, attachment["filename"]) with open(output_path, "wb") as out: out.write(base64.b64decode(attachment["raw"])) _log.debug( "Attachment <{}> extracted from <{}> stored locally as <{}>." .format(attachment["filename"], evidence, output_path)) self.case.track_file(output_path)
for file in feed: meta = _fs.guess_file_type(file) if not meta: tasks.setdefault(("raw", modules["raw"]), []).append(file) _log.warning( "Could not determine data type. Added evidence <{}> to the force-feeding list." .format(file)) continue try: name, Module = _find_association(modules, meta) tasks.setdefault((name, Module), []).append(file) _log.debug( "Identified data type <{}> for evidence <{}>. Dispatching to <{}>." .format(meta.mime, file, name)) except _errors.UnsupportedType: tasks.setdefault(("raw", modules["raw"]), []).append(file) _log.warning( "Data type <{}> unsupported. Added evidence <{}> to the force-feeding list." .format(meta.mime, file)) if tasks: for (name, Module), partial_feed in tasks.items(): if _interaction.prompt( "Found <{}> evidence(s) that can be dispatched. Do you want to automatically invoke the <{}> module using default option(s)?" .format(len(partial_feed), name), default_state=True): Module.case = case
def _initialize(container): """ .. py:function:: _initialize(container) Local entry point for the program. :param container: tuple containing the loaded module(s) and processed command-line argument(s) :type container: tuple """ del container[1]._dummy modules = container[0] args = container[1] _log.set_console_level(args.logging.upper()) if not _checker.number_rulesets(): _log.fault("No YARA rulesets found. Nothing to be done.") if args.no_prompt: _conf.DEFAULTS["NO_PROMPT"] = True case = _case.Case(args) case._create_arborescence() if _conf.CASE_WIDE_LOGGING: _log._create_file_logger("case", os.path.join( case.resources["case"], "{}.log".format(_meta.__package__)), level=_conf.CASE_WIDE_LOGGING_LEVEL, encoding=_conf.OUTPUT_CHARACTER_ENCODING) feed = _fs.expand_files(args.input, recursive=args.recursive, include=args.include, exclude=args.exclude) if not feed: _log.fault("No evidence(s) to process. Quitting.") if args._subparser: Module = container[0][args._subparser] Module.case = case Module.feed = feed with _magic.Hole( Exception, action=lambda: _log.fault( "Fatal exception raised within preprocessing module <{}>.". format(args._subparser), post_mortem=True)), _magic.Invocator(Module): Module.run() del Module else: _log.debug("Guessing data type(s).") _dispatch_preprocessing(modules, case, feed) if not case.resources["evidences"]: _log.fault("No evidence(s) to process. Quitting.")
def _dispatch_jobs(self): """ .. py:function:: _dispatch_jobs(self) Dispatches the processing task(s) to the subprocess(es). :param self: current class instance :type self: class :return: number of match(es) :rtype: int """ with multiprocessing.Manager() as manager: queue = manager.Queue() results = (multiprocessing.Lock(), multiprocessing.Value(ctypes.c_int, 0)) reader = multiprocessing.Process(target=_reader.Reader( queue, results, { "target": self.case.resources["matches"], "format": self.case.arguments.format }).run) reader.daemon = True reader.start() _log.debug("Started reader subprocess to process queue result(s).") with _magic.Pool(processes=self.case.arguments.processes) as pool: for file in self.case.resources["evidences"]["files"]: pool.starmap_async(_processors.File( self.case.arguments.hash_algorithms, self.case.arguments.callbacks, queue, self.case.arguments.fast).run, [(file, self.buffers)], error_callback=_log.inner_exception) _log.debug( "Mapped concurrent job to process evidence <{}>.". format(file)) for process in self.case.resources["evidences"]["processes"]: pool.starmap_async(_processors.Process( self.case.arguments.callbacks, queue, self.case.arguments.fast).run, [(process, self.buffers)], error_callback=_log.inner_exception) _log.debug( "Mapped concurrent job to process live process matching PID <{}>." .format(process)) queue.put(_codes.DONE) with _magic.Hole( KeyboardInterrupt, action=lambda: _log.fault( "Aborted due to manual user interruption <SIGINT>.")): reader.join() return results[1].value