def _read_queue(self):
"""
.. py:function:: _read_queue(self)
Main loop that processes the match(es) from the :code:`multiprocessing.Queue` instance.
:param self: current class instance
:type self: class
"""
while True:
item = self.queue.get()
if item == _codes.DONE:
break
self.map[self.target["format"]](item)
with self.results[0]:
self.results[1].value += 1
self.results[2].append(item["target"]["identifier"])
_log.debug(
"Matching signature from rule <{}> on evidence <{}>.".format(
item["match"]["rule"], item["target"]["identifier"]))
def __init__(self, arguments):
"""
.. py:function:: __init__(self, arguments)
Initialization method for the class.
:param self: current class instance
:type self: class
:param arguments: :code:`argparse.Parser` instance containing the processed command-line arguments
:type arguments: list
"""
self.arguments = arguments
self.name = os.path.basename(self.arguments.output)
self.resources = {
"case": self.arguments.output,
"matches": os.path.join(self.arguments.output, "{}.{}".format(_conf.MATCHES_FILE_BASENAME, self.arguments.format.lower())),
"evidences": {
"files": [],
"processes": []
},
"temporary": []
}
_log.debug("Initialized new case <{}> anchored to <{}>.".format(self.name, self.resources["case"]))
def _create_local_directory(self, directory, mask=0o700):
"""
.. py:function:: _create_local_directory(self, directory, mask=0o700)
Creates a directory on the filesystem.
:param self: current class instance
:type self: class
:param directory: absolute path to the directory to create
:type directory: str
:param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary
:type mask: oct
:return: random string of :code:`rounds` characters
:rtype: str
"""
try:
os.makedirs(directory, mode=mask)
_log.debug("Created local directory <{}>.".format(directory))
except FileExistsError:
_log.fault("Failed to create local directory due to existing object <{}>.".format(directory), trace=True)
except (
OSError,
Exception):
_log.fault("Failed to create local directory <{}>.".format(directory), trace=True)
def create_local_directory(directory, mask=0o700):
"""
.. py:function:: create_local_directory(directory, mask=0o700)
Creates a local case directory on the filesystem.
:param directory: absolute path to the directory to create
:type directory: str
:param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary
:type mask: oct
"""
try:
os.makedirs(directory, mode=mask)
_log.debug("Created local directory <{}>.".format(directory))
except FileExistsError:
_log.fault("Failed to create local directory due to existing object <{}>.".format(directory), post_mortem=True)
except (
OSError,
Exception):
_log.fault("Failed to create local directory <{}>.".format(directory), post_mortem=True)
def __exit__(self, *args):
"""
.. py:function:: __exit__(self, *args)
Exit method raised when leaving the context manager.
:param self: current class instance
:type self: class
:param *args: list of argument(s)
:type *args: class
"""
_log.debug("Ended <{}> session <{}>.".format(self.module.__class__.__name__, self.module.__name__))
def __init__(self, module):
"""
.. py:function:: __init__(self, module)
Initialization method for the class.
:param self: current class instance
:type self: class
:param module: class inherited from the :code:`models` reference classes
:type module: class
"""
self.module = module
_log.debug("Started <{}> session <{}>.".format(self.module.__class__.__name__, self.module.__name__))
def _dispatch_jobs(self):
"""
.. py:function:: _dispatch_jobs(self)
Dispatches the processing task(s) to the subprocess(es).
:param self: current class instance
:type self: class
:return: number of match(es)
:rtype: int
"""
with multiprocessing.Manager() as manager:
queue = manager.Queue()
results = (multiprocessing.Lock(), multiprocessing.Value(ctypes.c_int, 0), manager.list())
reader = multiprocessing.Process(target=_reader.Reader(queue, results, {
"target": self.case.resources["matches"],
"storage": self.case.resources["storage"],
"format": self.case.arguments.format
}).run)
reader.daemon = True
reader.start()
_log.debug("Started reader subprocess to consume queue result(s).")
with _magic.Pool(processes=self.case.arguments.processes) as pool:
for file in self.case.resources["evidences"]:
if os.path.getsize(file) > self.case.arguments.max_size:
_log.warning("Evidence <{}> exceeds the maximum size. Ignoring evidence. Try changing --max-size to override this behavior.".format(file))
continue
pool.starmap_async(
_processors.File(self.case.arguments.hash_algorithms, self.case.arguments.callbacks, queue, self.case.arguments.fast).run,
[(file, self.buffers)],
error_callback=_log.inner_exception)
_log.debug("Mapped concurrent job to consume evidence <{}>.".format(file))
queue.put(_codes.DONE)
with _magic.Hole(KeyboardInterrupt, action=lambda:_log.fault("Aborted due to manual user interruption <SIGINT>.")):
reader.join()
return results[1].value
def __init__(self, processes=(multiprocessing.cpu_count() or _conf.FALLBACK_PROCESSES)):
"""
.. py:function:: __init__(self, processes=(multiprocessing.cpu_count() or _conf.FALLBACK_PROCESSES))
Initialization method for the class.
:param self: current class instance
:type self: class
:param exception: number of concurrent process(es) to spawn
:type exception: int
"""
self.processes = processes
self.pool = multiprocessing.Pool(processes=self.processes, initializer=self._worker_initializer)
_log.debug("Initialized pool of <{}> concurrent process(es).".format(self.processes))
def _compile_ruleset(self, name, ruleset):
"""
.. py:function:: _compile_ruleset(self, name, ruleset)
Compiles and saves YARA rule(s) to the dictionary to be passed to the asynchronous job(s).
:param self: current class instance
:type self: class
:param name: name of the ruleset file to compile the rule(s) from
:type name: str
:param ruleset: absolute path to the ruleset file to compile the rule(s) from
:type ruleset: str
:return: tuple containing the final status of the compilation and the number of successfully loaded rule(s)
:rtype: bool, int
"""
count = 0
try:
buffer = io.BytesIO()
rules = yara.compile(ruleset, includes=_conf.YARA_INCLUDES, error_on_warning=(not self.case.arguments.ignore_warnings))
rules.save(file=buffer)
self.buffers[ruleset] = buffer
count += sum(1 for _ in rules)
_log.debug("Precompilated YARA ruleset <{}> in memory with a total of <{}> valid rule(s).".format(name, count))
return True, count
except yara.SyntaxError:
_log.exception("Syntax error in YARA ruleset <{}>.".format(ruleset))
except (
Exception,
yara.Error):
_log.exception("Failed to pre-compile ruleset <{}>.".format(ruleset))
return False, count
def track_file(self, evidence):
"""
.. py:function:: track_file(self, evidence)
Checks and registers an evidence file for processing.
:param self: current class instance
:type self: class
:param evidence: absolute path to the evidence file
:type evidence: str
"""
if os.path.isfile(evidence):
self.resources["evidences"]["files"].append(evidence)
_log.debug("Tracking file <{}>.".format(evidence))
else:
_log.warning("Evidence <{}> not found or invalid.".format(evidence))
def _store_matching_evidences(self):
"""
.. py:function:: _store_matching_evidences(self)
Saves the matching evidence(s) to the specified storage directory.
:param self: current class instance
:type self: class
"""
for evidence in self.results[2]:
if not os.path.isdir(self.target["storage"]):
_fs.create_local_directory(self.target["storage"])
try:
storage_path = (
os.path.join(self.target["storage"],
os.path.basename(evidence)) if
not _conf.NEUTRALIZE_MATCHING_EVIDENCES else os.path.join(
self.target["storage"], "{}.{}".format(
os.path.basename(evidence), _meta.__package__)))
shutil.copy2(evidence, storage_path)
if _conf.NEUTRALIZE_MATCHING_EVIDENCES:
os.chmod(
storage_path,
stat.S_IMODE(os.lstat(storage_path).st_mode)
& ~stat.S_IEXEC)
_log.debug("Saved {}matching evidence <{}> as <{}>.".format(
"and neutralized "
if _conf.NEUTRALIZE_MATCHING_EVIDENCES else "",
os.path.basename(evidence), storage_path))
except (OSError, shutil.Error, Exception):
_log.exception(
"Failed to save matching evidence <{}> as <{}>.".format(
os.path.basename(evidence), storage_path))
def run(self):
"""
.. py:function:: run(self)
Main entry point for the module.
:param self: current class instance
:type self: class
"""
if self.case.arguments._inline_password:
_log.debug(
"Using inline password <{}> to unpack archive(s).".format(
self.case.arguments._inline_password))
elif self.case.arguments._password:
self._password = _interaction.password_prompt(
"Unpacking password: "******"Recursive unpacking manually disabled using --no-recursion.")
tmp = self.case.require_temporary_directory()
for evidence in self.feed:
self.recursive_inflate(
evidence,
tmp,
password=(self.case.arguments._inline_password if
not hasattr(self, "_password") else self._password))
for evidence in _fs.expand_files([tmp],
recursive=True,
include=self.case.arguments._include,
exclude=self.case.arguments._exclude):
print("found {}".format(evidence))
def run(self):
"""
.. py:function:: run(self)
Main entry point for the module.
:param self: current class instance
:type self: class
"""
tmp = self.case.require_temporary_directory()
for item in self.case.arguments.input:
if os.path.isfile(item):
_log.debug("Tracking file <{}> to <{}>.".format(file, tmp))
elif os.path.isdir(item):
_log.warning(
"Directory <{}> is not an archive. Ignoring.".format(item))
else:
_log.warning(
"Unknown inode type for object <{}>.".format(item))
def _tear_down(self):
"""
.. py:function:: _tear_down(self)
Cleanup method called on class destruction that gets rid of the temporary artifact(s).
:param self: current class instance
:type self: class
"""
for artifact in self.resources["temporary"]:
try:
shutil.rmtree(artifact)
_log.debug("Removed temporary artifact <{}>.".format(artifact))
except FileNotFoundError:
_log.debug("Temporary artifact not found <{}>.".format(artifact))
except (
OSError,
Exception):
_log.exception("Failed to remove temporary artifact <{}>.".format(artifact))
def track_process(self, pid):
"""
.. py:function:: track_process(self, pid)
Checks wether a process exists on the local machine and registers it for processing.
:param self: current class instance
:type self: class
:param pid: process identifier
:type pid: int
"""
if not isinstance(pid, int):
_log.error("Invalid PID format <{}>.".format(pid))
return
if psutil.pid_exists(pid):
self.resources["evidences"]["processes"].append(pid)
_log.debug("Tracking live process matching PID <{}>.".format(pid))
else:
_log.warning("Process <{}> not found.".format(pid))
def track_files(self, evidences, include=[], exclude=[]):
"""
.. py:function:: track_files(self, evidences)
Checks and registers multiple evidence files for processing.
:param self: current class instance
:type self: class
:param evidences: list of absolute path(s) to the evidence file(s)
:type evidences: list
:param include: list of wildcard pattern(s) to include
:type include: list
:param exclude: list of wildcard pattern(s) to exclude
:type exclude: list
"""
evidences = [os.path.abspath(evidence) for evidence in evidences]
for evidence in self._iterate_existing_files(evidences):
if include and not _fs.matches_patterns(os.path.basename(evidence),
wildcard_patterns=include):
_log.debug(
"Ignoring evidence <{}> not matching inclusion pattern(s) <{}>."
.format(evidence, include))
continue
if exclude and _fs.matches_patterns(os.path.basename(evidence),
wildcard_patterns=exclude):
_log.debug(
"Ignoring evidence <{}> matching exclusion pattern(s) <{}>."
.format(evidence, exclude))
continue
self.track_file(evidence)
def recursive_inflate(self,
archive,
output_directory,
level=0,
password=None):
if level > self.case.arguments._level:
_log.warning(
"Limit unpacking level <{}> exceeded. Stopped unpacking.".
format(self.case.arguments._level))
return
_log.debug(
"Inflating {}archive <{}> to temporary directory <{}>.".format(
"level {} sub".format(level) if level else "base ", archive,
output_directory))
sub_directory = os.path.join(output_directory,
os.path.basename(archive))
try:
with zipfile.ZipFile(archive) as z:
z.extractall(path=sub_directory,
pwd=(password.encode() if password else password))
except zipfile.BadZipFile:
_log.error(
"Bad file header. Cannot inflate evidence <{}>. Try to filter out non-zip file(s) using --include \"*.zip\" \".*.zip\"."
.format(archive))
return
except RuntimeError as exc:
if "password required" in str(exc):
_log.error(
"Archive <{}> seems to be encrypted. Please specify a password using --password or --inline-password."
.format(archive))
elif "Bad password" in str(exc):
_log.error(
"Password {}seems to be incorrect for archive <{}>. Please specify another password using --password or --inline-password."
.format(
"<{}> ".format(self.case.arguments._inline_password)
if not hasattr(self, "_password") else "", archive))
else:
_log.exception(
"Runtime exception raised while unpacking archive <{}>.".
format(archive))
return
except KeyboardInterrupt:
sys.stderr.write("\n")
_log.fault("Aborted due to manual user interruption.")
except Exception:
_log.exception(
"Exception raised while unpacking archive <{}>.".format(
archive))
if self.case.arguments._no_recursion:
return
for subarchive in _fs.enumerate_matching_files(
sub_directory,
wildcard_patterns=([
"*.{}".format(_)
for _ in self.__associations__["extensions"]
] + [
".*.{}".format(_)
for _ in self.__associations__["extensions"]
] if hasattr(self, "__associations__")
and "extensions" in self.__associations__
else None),
mime_types=(self.__associations__["mime"]
if hasattr(self, "__associations__")
and "mime" in self.__associations__ else None),
recursive=True):
self.recursive_inflate(subarchive,
sub_directory,
level=(level + 1),
password=password)
def run(self):
"""
.. py:function:: run(self)
Main entry point for the module.
:param self: current class instance
:type self: class
"""
tmp = self.case.require_temporary_directory()
for evidence in self.feed:
try:
mail = eml_parser.eml_parser.decode_email(
evidence,
include_raw_body=True,
include_attachment_data=True)
_log.info("Extracted <{}> attachment(s) from <{}>.".format(
len(mail["attachment"]), evidence))
except Exception:
_log.exception(
"Failed to extract data from <{}>. Ignoring evidence.".
format(evidence))
continue
output_directory = os.path.join(tmp, os.path.basename(evidence))
if not os.path.isdir(output_directory):
_fs.create_local_directory(output_directory)
for attachment in mail["attachment"]:
if not attachment["filename"]:
attachment["filename"] = idx
if not _fs.matches_patterns(attachment["filename"],
self.case.arguments._include):
_log.warning(
"Ignoring attachment <{}> not matching inner inclusion pattern(s)."
.format(attachment["filename"]))
continue
if _fs.matches_patterns(attachment["filename"],
self.case.arguments._exclude):
_log.warning(
"Ignoring attachment <{}> matching inner exclusion pattern(s)."
.format(attachment["filename"]))
continue
output_path = os.path.join(output_directory,
attachment["filename"])
with open(output_path, "wb") as out:
out.write(base64.b64decode(attachment["raw"]))
_log.debug(
"Attachment <{}> extracted from <{}> stored locally as <{}>."
.format(attachment["filename"], evidence, output_path))
self.case.track_file(output_path)
for file in feed:
meta = _fs.guess_file_type(file)
if not meta:
tasks.setdefault(("raw", modules["raw"]), []).append(file)
_log.warning(
"Could not determine data type. Added evidence <{}> to the force-feeding list."
.format(file))
continue
try:
name, Module = _find_association(modules, meta)
tasks.setdefault((name, Module), []).append(file)
_log.debug(
"Identified data type <{}> for evidence <{}>. Dispatching to <{}>."
.format(meta.mime, file, name))
except _errors.UnsupportedType:
tasks.setdefault(("raw", modules["raw"]), []).append(file)
_log.warning(
"Data type <{}> unsupported. Added evidence <{}> to the force-feeding list."
.format(meta.mime, file))
if tasks:
for (name, Module), partial_feed in tasks.items():
if _interaction.prompt(
"Found <{}> evidence(s) that can be dispatched. Do you want to automatically invoke the <{}> module using default option(s)?"
.format(len(partial_feed), name),
default_state=True):
Module.case = case
def _initialize(container):
"""
.. py:function:: _initialize(container)
Local entry point for the program.
:param container: tuple containing the loaded module(s) and processed command-line argument(s)
:type container: tuple
"""
del container[1]._dummy
modules = container[0]
args = container[1]
_log.set_console_level(args.logging.upper())
if not _checker.number_rulesets():
_log.fault("No YARA rulesets found. Nothing to be done.")
if args.no_prompt:
_conf.DEFAULTS["NO_PROMPT"] = True
case = _case.Case(args)
case._create_arborescence()
if _conf.CASE_WIDE_LOGGING:
_log._create_file_logger("case",
os.path.join(
case.resources["case"],
"{}.log".format(_meta.__package__)),
level=_conf.CASE_WIDE_LOGGING_LEVEL,
encoding=_conf.OUTPUT_CHARACTER_ENCODING)
feed = _fs.expand_files(args.input,
recursive=args.recursive,
include=args.include,
exclude=args.exclude)
if not feed:
_log.fault("No evidence(s) to process. Quitting.")
if args._subparser:
Module = container[0][args._subparser]
Module.case = case
Module.feed = feed
with _magic.Hole(
Exception,
action=lambda: _log.fault(
"Fatal exception raised within preprocessing module <{}>.".
format(args._subparser),
post_mortem=True)), _magic.Invocator(Module):
Module.run()
del Module
else:
_log.debug("Guessing data type(s).")
_dispatch_preprocessing(modules, case, feed)
if not case.resources["evidences"]:
_log.fault("No evidence(s) to process. Quitting.")
def _dispatch_jobs(self):
"""
.. py:function:: _dispatch_jobs(self)
Dispatches the processing task(s) to the subprocess(es).
:param self: current class instance
:type self: class
:return: number of match(es)
:rtype: int
"""
with multiprocessing.Manager() as manager:
queue = manager.Queue()
results = (multiprocessing.Lock(),
multiprocessing.Value(ctypes.c_int, 0))
reader = multiprocessing.Process(target=_reader.Reader(
queue, results, {
"target": self.case.resources["matches"],
"format": self.case.arguments.format
}).run)
reader.daemon = True
reader.start()
_log.debug("Started reader subprocess to process queue result(s).")
with _magic.Pool(processes=self.case.arguments.processes) as pool:
for file in self.case.resources["evidences"]["files"]:
pool.starmap_async(_processors.File(
self.case.arguments.hash_algorithms,
self.case.arguments.callbacks, queue,
self.case.arguments.fast).run, [(file, self.buffers)],
error_callback=_log.inner_exception)
_log.debug(
"Mapped concurrent job to process evidence <{}>.".
format(file))
for process in self.case.resources["evidences"]["processes"]:
pool.starmap_async(_processors.Process(
self.case.arguments.callbacks, queue,
self.case.arguments.fast).run,
[(process, self.buffers)],
error_callback=_log.inner_exception)
_log.debug(
"Mapped concurrent job to process live process matching PID <{}>."
.format(process))
queue.put(_codes.DONE)
with _magic.Hole(
KeyboardInterrupt,
action=lambda: _log.fault(
"Aborted due to manual user interruption <SIGINT>.")):
reader.join()
return results[1].value