def _create_local_directory(self, directory, mask=0o700):
"""
.. py:function:: _create_local_directory(self, directory, mask=0o700)
Creates a directory on the filesystem.
:param self: current class instance
:type self: class
:param directory: absolute path to the directory to create
:type directory: str
:param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary
:type mask: oct
:return: random string of :code:`rounds` characters
:rtype: str
"""
try:
os.makedirs(directory, mode=mask)
_log.debug("Created local directory <{}>.".format(directory))
except FileExistsError:
_log.fault("Failed to create local directory due to existing object <{}>.".format(directory), trace=True)
except (
OSError,
Exception):
_log.fault("Failed to create local directory <{}>.".format(directory), trace=True)
def run(self):
"""
.. py:function:: run(self)
Main entry point for the class.
:param self: current class instance
:type self: class
"""
loaded = {
"rulesets": 0,
"rules": 0
}
for name, ruleset in _loader.iterate_rulesets():
status, count = self._compile_ruleset(name, ruleset)
if status:
loaded["rulesets"] += 1
loaded["rules"] += count
if not loaded["rulesets"]:
_log.fault("No YARA ruleset(s) loaded. Quitting.")
_log.info("Applying a total of <{}> YARA rule(s) from <{}> ruleset(s).".format(loaded["rules"], loaded["rulesets"]))
del loaded
if not self._dispatch_jobs():
_log.warning("Skipping <{}> module(s) invocation.".format(_models.Post.__name__))
return
self._invoke_post_modules()
def _open_output_file(self,
mode="a",
character_encoding=_conf.OUTPUT_CHARACTER_ENCODING):
"""
.. py:function:: _open_output_file(self, mode="a", character_encoding=conf.OUTPUT_CHARACTER_ENCODING)
Opens the output stream.
:param self: current class instance
:type self: class
:param mode: file opening mode to use
:type mode: str
:param character_encoding: character encoding to use
:type character_encoding: str
:return: descriptor for the newly opened file stream
:rtype: class
"""
try:
return open(self.target["target"],
mode=mode,
encoding=character_encoding)
except (OSError, Exception):
_log.fault("Failed to open <{}> for writing.".format(
self.target["target"]),
post_mortem=True)
def run(self):
"""
.. py:function:: run(self)
Main entry point for the class.
:param self: current class instance
:type self: class
"""
count = 0
for name, ruleset in _loader.iterate_rulesets():
if self._compile_ruleset(name, ruleset):
count += 1
if not count:
_log.fault("No YARA rulesets loaded. Quitting.")
_log.info("Applying a total of <{}> YARA ruleset(s).".format(count))
del count
if not self._dispatch_jobs():
_log.warning("Skipping <{}> module(s) invocation.".format(
_models.Post.__name__))
return
self._invoke_post_modules()
def _create_arborescence(self):
"""
.. py:function:: _create_arborescence(self)
Creates the base arborescence for the current case.
:param self: current class instance
:type self: class
"""
if os.path.exists(self.resources["case"]):
if not self.arguments.overwrite:
if not _interaction.prompt(
"Overwrite existing object <{}>?".format(
self.resources["case"])):
_log.fault("Aborted due to manual user interruption.")
try:
send2trash.send2trash(self.resources["case"])
_log.warning("Overwritten existing object <{}>.".format(
self.resources["case"]))
except (send2trash.TrashPermissionError, OSError, Exception):
_log.fault("Failed to overwrite existing object <{}>.".format(
self.resources["case"]),
post_mortem=True)
_fs.create_local_directory(self.resources["case"])
def create_local_directory(directory, mask=0o700):
"""
.. py:function:: create_local_directory(directory, mask=0o700)
Creates a local case directory on the filesystem.
:param directory: absolute path to the directory to create
:type directory: str
:param mask: permissions bit mask to apply for the newly created :code:`directory` and its parents if necessary
:type mask: oct
"""
try:
os.makedirs(directory, mode=mask)
_log.debug("Created local directory <{}>.".format(directory))
except FileExistsError:
_log.fault("Failed to create local directory due to existing object <{}>.".format(directory), post_mortem=True)
except (
OSError,
Exception):
_log.fault("Failed to create local directory <{}>.".format(directory), post_mortem=True)
def create_arborescence(self):
"""
.. py:function:: create_arborescence(self)
Creates the base arborescence for the current case.
:param self: current class instance
:type self: class
"""
if os.path.exists(self.resources["case"]):
if not self.arguments.overwrite:
self._prompt("Overwrite existing object <{}> ? [y/N] ".format(self.resources["case"]))
try:
shutil.rmtree(self.resources["case"])
_log.warning("Overwritten existing object <{}>.".format(self.resources["case"]))
except (
OSError,
Exception):
_log.fault("Failed to overwrite existing object <{}>.".format(self.resources["case"]), trace=True)
self._create_local_directory(self.resources["case"])
def main():
"""
.. py:function:: main()
Main entry point for the program.
"""
try:
_initialize(_argparser(_parser.Parser()))
except SystemExit:
pass
except:
_log.fault("Unhandled exception trapped. Quitting.", post_mortem=True)
def render_modules(package, model):
"""
.. py:function:: render_modules(package, model)
Renders available module(s) name(s) as a list.
:param package: package handle to import module(s) from
:type package: class
:param model: reference module class handle
:type model: class
:return: available module(s) in :code:`package`
:rtype: list
"""
try:
_checker.check_package(package)
except _errors.InvalidPackage:
_log.fault("Invalid package <{}>.".format(package), post_mortem=True)
return [os.path.splitext(name)[0] for name, _ in Loader.iterate_modules(package, model, silent=True)]
def _dispatch_jobs(self):
"""
.. py:function:: _dispatch_jobs(self)
Dispatches the processing task(s) to the subprocess(es).
:param self: current class instance
:type self: class
:return: number of match(es)
:rtype: int
"""
with multiprocessing.Manager() as manager:
queue = manager.Queue()
results = (multiprocessing.Lock(), multiprocessing.Value(ctypes.c_int, 0), manager.list())
reader = multiprocessing.Process(target=_reader.Reader(queue, results, {
"target": self.case.resources["matches"],
"storage": self.case.resources["storage"],
"format": self.case.arguments.format
}).run)
reader.daemon = True
reader.start()
_log.debug("Started reader subprocess to consume queue result(s).")
with _magic.Pool(processes=self.case.arguments.processes) as pool:
for file in self.case.resources["evidences"]:
if os.path.getsize(file) > self.case.arguments.max_size:
_log.warning("Evidence <{}> exceeds the maximum size. Ignoring evidence. Try changing --max-size to override this behavior.".format(file))
continue
pool.starmap_async(
_processors.File(self.case.arguments.hash_algorithms, self.case.arguments.callbacks, queue, self.case.arguments.fast).run,
[(file, self.buffers)],
error_callback=_log.inner_exception)
_log.debug("Mapped concurrent job to consume evidence <{}>.".format(file))
queue.put(_codes.DONE)
with _magic.Hole(KeyboardInterrupt, action=lambda:_log.fault("Aborted due to manual user interruption <SIGINT>.")):
reader.join()
return results[1].value
def load_module(name, model, silent=False):
"""
.. py:function:: load_module(name, model)
Dynamically loads a registered module.
:param name: name of the module to load
:type name: str
:param model: reference class handle
:type model: class
:param silent: suppress the warning message
:type silent: bool
:return: module class handle
:rtype: class
"""
try:
module = importlib.import_module("framework.modules.{}.{}".format(model.__name__.lower(), name))
except ModuleNotFound as exc:
_log.fault("Missing dependency <{0}>. Try <pip install {0}> or manually build the required module to fix the issue.".format(exc.name))
try:
_checker.check_module(module, model)
except _errors.NotFound:
_log.fault("No subclass found in module <{}.{}>. Quitting.".format(model.__name__.lower(), name), post_mortem=True)
except _errors.ModuleInheritance:
_log.fault("Module <{}.{}> not inheriting from the base class. Quitting.".format(model.__name__.lower(), name), post_mortem=True)
except _errors.SystemNotSupported:
if model.__name__ == _models.Pre.__name__:
_log.fault("Module <{}.{}> does not support the current system <{}>. Quitting.".format(model.__name__.lower(), name, platform.system()))
elif not silent:
_log.warning("Module <{}.{}> incompatible with the current system <{}>. Ignoring.".format(model.__name__.lower(), name, platform.system()))
return None
return getattr(module, model.__name__)
def _prompt(self, message, rounds=_conf.PROMPT_ROUNDS, harsh_escape=_conf.PROMPT_HARSH_ESCAPE):
"""
.. py:function:: _prompt(self, message, rounds=_conf.PROMPT_ROUNDS, harsh_escape=_conf.PROMPT_HARSH_ESCAPE)
Prompts the user with a yes/no question and wait for a valid answer.
:param self: current class instance
:type self: class
:param message: question to print
:type message: str
:param rounds: number of times to repeat the question
:type rounds: int
:param harsh_escape: exit the program if :code:`rounds` has been reached
:type harsh_escape: bool
"""
for _ in range(rounds):
try:
answer = _checker.sanitize_data(input(message))
except _errors.MalformatedDataError:
_log.fault("Malformated input.")
except KeyboardInterrupt:
sys.stderr.write("\n")
_log.fault("SIGINT trapped.")
if not answer or answer in "nN":
_log.fault("Aborted due to manual user interruption.")
elif answer in "yY":
return
if harsh_escape:
_log.fault("No valid answer provided.")
def recursive_inflate(self,
archive,
output_directory,
level=0,
password=None):
if level > self.case.arguments._level:
_log.warning(
"Limit unpacking level <{}> exceeded. Stopped unpacking.".
format(self.case.arguments._level))
return
_log.debug(
"Inflating {}archive <{}> to temporary directory <{}>.".format(
"level {} sub".format(level) if level else "base ", archive,
output_directory))
sub_directory = os.path.join(output_directory,
os.path.basename(archive))
try:
with zipfile.ZipFile(archive) as z:
z.extractall(path=sub_directory,
pwd=(password.encode() if password else password))
except zipfile.BadZipFile:
_log.error(
"Bad file header. Cannot inflate evidence <{}>. Try to filter out non-zip file(s) using --include \"*.zip\" \".*.zip\"."
.format(archive))
return
except RuntimeError as exc:
if "password required" in str(exc):
_log.error(
"Archive <{}> seems to be encrypted. Please specify a password using --password or --inline-password."
.format(archive))
elif "Bad password" in str(exc):
_log.error(
"Password {}seems to be incorrect for archive <{}>. Please specify another password using --password or --inline-password."
.format(
"<{}> ".format(self.case.arguments._inline_password)
if not hasattr(self, "_password") else "", archive))
else:
_log.exception(
"Runtime exception raised while unpacking archive <{}>.".
format(archive))
return
except KeyboardInterrupt:
sys.stderr.write("\n")
_log.fault("Aborted due to manual user interruption.")
except Exception:
_log.exception(
"Exception raised while unpacking archive <{}>.".format(
archive))
if self.case.arguments._no_recursion:
return
for subarchive in _fs.enumerate_matching_files(
sub_directory,
wildcard_patterns=([
"*.{}".format(_)
for _ in self.__associations__["extensions"]
] + [
".*.{}".format(_)
for _ in self.__associations__["extensions"]
] if hasattr(self, "__associations__")
and "extensions" in self.__associations__
else None),
mime_types=(self.__associations__["mime"]
if hasattr(self, "__associations__")
and "mime" in self.__associations__ else None),
recursive=True):
self.recursive_inflate(subarchive,
sub_directory,
level=(level + 1),
password=password)
def _initialize(container):
"""
.. py:function:: _initialize(container)
Local entry point for the program.
:param container: tuple containing the loaded module(s) and processed command-line argument(s)
:type container: tuple
"""
del container[1]._dummy
modules = container[0]
args = container[1]
_log.set_console_level(args.logging.upper())
if not _checker.number_rulesets():
_log.fault("No YARA rulesets found. Nothing to be done.")
if args.no_prompt:
_conf.DEFAULTS["NO_PROMPT"] = True
case = _case.Case(args)
case._create_arborescence()
if _conf.CASE_WIDE_LOGGING:
_log._create_file_logger("case",
os.path.join(
case.resources["case"],
"{}.log".format(_meta.__package__)),
level=_conf.CASE_WIDE_LOGGING_LEVEL,
encoding=_conf.OUTPUT_CHARACTER_ENCODING)
feed = _fs.expand_files(args.input,
recursive=args.recursive,
include=args.include,
exclude=args.exclude)
if not feed:
_log.fault("No evidence(s) to process. Quitting.")
if args._subparser:
Module = container[0][args._subparser]
Module.case = case
Module.feed = feed
with _magic.Hole(
Exception,
action=lambda: _log.fault(
"Fatal exception raised within preprocessing module <{}>.".
format(args._subparser),
post_mortem=True)), _magic.Invocator(Module):
Module.run()
del Module
else:
_log.debug("Guessing data type(s).")
_dispatch_preprocessing(modules, case, feed)
if not case.resources["evidences"]:
_log.fault("No evidence(s) to process. Quitting.")
"Data type <{}> unsupported. Added evidence <{}> to the force-feeding list."
.format(meta.mime, file))
if tasks:
for (name, Module), partial_feed in tasks.items():
if _interaction.prompt(
"Found <{}> evidence(s) that can be dispatched. Do you want to automatically invoke the <{}> module using default option(s)?"
.format(len(partial_feed), name),
default_state=True):
Module.case = case
Module.feed = partial_feed
with _magic.Hole(
Exception,
action=lambda: _log.fault(
"Fatal exception raised within preprocessing module <{}>."
.format(name),
post_mortem=True)), _magic.Invocator(Module):
Module.run()
del Module
def _argparser(parser):
"""
.. py:function:: _argparser(parser)
Command-line argument parsing function.
:param parser: :code:`argparse.Parser` instance
:type parser: class
from framework.contexts.logger import Logger as _log
import sys
try:
from pygments import highlight
from pygments.formatters import TerminalFormatter
from pygments.lexers import JsonLexer
import simplejson as json
except (
ImportError,
Exception):
_log.fault("Import error.", trace=True)
__all__ = [
"Callback"
]
class Callback(_models.Callback):
__author__ = "sk4la"
__description__ = "Simple callback tailing and beautifying match(es)."
__license__ = "MIT <https://github.com/sk4la/plast/blob/master/LICENSE.adoc>"
__maintainer__ = ["sk4la"]
__system__ = ["Darwin", "Linux", "Windows"]
__version__ = "0.1"
def run(self, data):
sys.stdout.write(highlight(json.dumps(data, indent=4, sort_keys=True), JsonLexer(), TerminalFormatter()))
from framework.contexts import errors as _errors
from framework.contexts import models as _models
from framework.contexts.logger import Logger as _log
from framework.contexts.configuration import Configuration as _conf
from framework.contexts.meta import Meta as _meta
import importlib
import os.path
import pkgutil
import platform
try:
import yara
except ImportError as exc:
_log.fault("Missing dependency <{0}>. Try <pip install {0}> or manually build the required module to fix the issue.".format(exc.name))
__all__ = [
"Loader"
]
class Loader:
"""Assists modules load."""
@staticmethod
def load_module(name, model, silent=False):
"""
.. py:function:: load_module(name, model)
Dynamically loads a registered module.
def _dispatch_jobs(self):
"""
.. py:function:: _dispatch_jobs(self)
Dispatches the processing task(s) to the subprocess(es).
:param self: current class instance
:type self: class
:return: number of match(es)
:rtype: int
"""
with multiprocessing.Manager() as manager:
queue = manager.Queue()
results = (multiprocessing.Lock(),
multiprocessing.Value(ctypes.c_int, 0))
reader = multiprocessing.Process(target=_reader.Reader(
queue, results, {
"target": self.case.resources["matches"],
"format": self.case.arguments.format
}).run)
reader.daemon = True
reader.start()
_log.debug("Started reader subprocess to process queue result(s).")
with _magic.Pool(processes=self.case.arguments.processes) as pool:
for file in self.case.resources["evidences"]["files"]:
pool.starmap_async(_processors.File(
self.case.arguments.hash_algorithms,
self.case.arguments.callbacks, queue,
self.case.arguments.fast).run, [(file, self.buffers)],
error_callback=_log.inner_exception)
_log.debug(
"Mapped concurrent job to process evidence <{}>.".
format(file))
for process in self.case.resources["evidences"]["processes"]:
pool.starmap_async(_processors.Process(
self.case.arguments.callbacks, queue,
self.case.arguments.fast).run,
[(process, self.buffers)],
error_callback=_log.inner_exception)
_log.debug(
"Mapped concurrent job to process live process matching PID <{}>."
.format(process))
queue.put(_codes.DONE)
with _magic.Hole(
KeyboardInterrupt,
action=lambda: _log.fault(
"Aborted due to manual user interruption <SIGINT>.")):
reader.join()
return results[1].value
from framework.api.external import filesystem as _fs
from framework.contexts import models as _models
from framework.contexts.configuration import Configuration as _conf
from framework.contexts.logger import Logger as _log
import base64
import multiprocessing
import os.path
try:
import eml_parser
except ImportError as exc:
_log.fault(
"Missing dependency <{0}>. Try <pip install {0}> or manually build the required module to fix the issue. Module <{0}> may require the installation of the <{1}> library."
.format(exc.name, "libmagic"))
__all__ = ["Pre"]
class Pre(_models.Pre):
__author__ = "sk4la"
__description__ = "Parses .eml file(s), extracts attachment(s) and feeds the resulting evidence(s) to the engine."
__license__ = "GNU GPLv3 <https://github.com/sk4la/plast/blob/master/LICENSE>"
__maintainer__ = ["sk4la"]
__system__ = ["Darwin", "Linux", "Windows"]
__version__ = "0.1"
__associations__ = {"extensions": ["eml"]}
def __init__(self, parser):
# -*- coding: utf-8 -*-
from framework.contexts import models as _models
from framework.contexts.logger import Logger as _log
try:
import ExtractMsg
except ImportError as exc:
_log.fault(
"Missing dependency <{0}>. Try <pip install https://github.com/mattgwwalker/msg-extractor/zipball/master> or manually build the required module to fix the issue."
.format(exc.name))
__all__ = ["Pre"]
class Pre(_models.Pre):
__author__ = "sk4la"
__description__ = "Parses .msg file(s), extracts attachment(s) and feeds the resulting evidence(s) to the engine."
__license__ = "GNU GPLv3 <https://github.com/sk4la/plast/blob/master/LICENSE>"
__maintainer__ = ["sk4la"]
__system__ = ["Darwin", "Linux", "Windows"]
__version__ = "0.1"
__associations__ = {
"extensions": ["msg"],
"mime": ["application/vnd.ms-outlook"]
}
def run(self):
"""
.. py:function:: run(self)
