def test_run_zipextractor(self):
conf = WhatTheFileConfiguration()
output_safe_directory = "./tests/examples/safe_directory"
conf.parse_string("""
[whatthefile]
modules_package = src.modules
safe_output_path = """ + output_safe_directory + """
output = list
log_output = stdout
[module.zipextractor]
active = true
""")
final_file = os.path.abspath(output_safe_directory) + \
"/1/zipextractor/" + \
os.path.abspath("./tests/examples/collie.jpg.zip/collie.jpg").replace(":", "")
final_file = os.path.abspath(final_file)
if os.path.exists(final_file):
os.remove(final_file)
self._remove_test_folders(output_safe_directory)
self.assertFalse(os.path.exists(final_file))
path = "./tests/examples/collie.jpg.zip"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
self.assertTrue(os.path.exists(final_file))
paths = []
for element in output.get_list():
paths.append(element["path"])
self.assertTrue(os.path.abspath(final_file) in paths)
os.remove(final_file)
self._remove_test_folders(output_safe_directory)
def a_test_run_all(self):
conf = WhatTheFileConfiguration()
conf.parse_dict({
"whatthefile": {
"modules_package": "src.modules",
"output": "list",
"log_output": "stdout",
"safe_output_path": "./tests/examples/safe_directory"
},
"module.commentextractor": {
"active": True
},
"module.entropy": {
"active": True
},
"module.hashes": {
"active": True,
'hashes_to_calculate': "MD5,SHA1,SHA256"
},
"module.imagerecognitiontensorflow": {
"active": True
},
"module.metadata": {
"active": True
},
"module.ocrtesseract": {
"active": True
},
"module.qrbcreader": {
"active": True
},
"module.strings": {
"active": True,
"char_min": 4
},
"module.virustotal": {
"active": True
},
"module.zipextractor": {
"active": True
},
"module.tikaparser": {
"active": True
},
"module.certificatereader": {
"active": True
},
"module.browserhisstory": {
"active": True
}
})
path = "./tests/examples/collie.jpg"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
self.assertEqual(
"collie" in output.get_list()[0]["imagerecognitiontensorflow"])
def run(self, arguments):
if len(arguments) != 3:
Console.print_help(arguments)
sys.exit()
else:
conf = WhatTheFileConfiguration()
conf.parse_file(arguments[1])
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(arguments[2])
def test_run_hashes(self):
conf = WhatTheFileConfiguration()
conf.parse_dict({
"whatthefile": {
"modules_package": "src.modules",
"output": "list",
"log_output": "stdout",
"safe_output_path": "./tests/examples/safe_directory"
},
"module.commentextractor": {
"active": True
},
"module.entropy": {
"active": False
},
"module.hashes": {
"active": True,
'hashes_to_calculate': "MD5,SHA1,SHA256"
},
"module.imagerecognitiontensorflow": {
"active": False
},
"module.metadata": {
"active": False
},
"module.ocrtesseract": {
"active": False
},
"module.qrbcreader": {
"active": False
},
"module.strings": {
"active": False,
"char_min": 10
},
"module.virustotal": {
"active": False
},
"module.zipextractor": {
"active": False
}
})
path = "./tests/examples/collie.jpg.zip"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
self.assertTrue("SHA256" in output.get_list()[0]["hashes"])
self.assertTrue("start_module" in output.get_list()[0]["hashes"])
self.assertTrue("end_module" in output.get_list()[0]["hashes"])
self.assertTrue("begin_analysis" in output.get_list()[0])
self.assertTrue("end_analysis" in output.get_list()[0])
def _import_module(conf: WhatTheFileConfiguration, module_section_name,
position: int) -> Module:
module_name = module_section_name.split(".")[1]
try:
mod = importlib.import_module(
conf.get_property("whatthefile", "modules_package") + "." +
module_name + ".main")
py_mod = getattr(mod, "Constructor")()
configuration = conf.get_whatthefile_secction()
configuration.update(conf.get_section(module_section_name))
py_mod.set_params(configuration)
return Module(module_name, position, py_mod)
except:
traceback.print_exc()
return None
def test_run_directory(self):
conf = WhatTheFileConfiguration()
conf.parse_dict({
"whatthefile": {
"modules_package": "src.modules",
"output": "list",
"log_output": "stdout",
"safe_output_path": "./tests/examples/safe_directory"
},
"module.commentextractor": {
"active": True
},
"module.entropy": {
"active": True
},
"module.hashes": {
"active": True,
'hashes_to_calculate': "MD5,SHA1,SHA256"
},
"module.imagerecognitiontensorflow": {
"active": True
},
"module.metadata": {
"active": True
},
"module.ocrtesseract": {
"active": True
},
"module.qrbcreader": {
"active": True
},
"module.strings": {
"active": True,
"char_min": 4
},
"module.virustotal": {
"active": False
},
"module.zipextractor": {
"active": True
}
})
path = "./tests/examples/testdirectorydonotinsertmoreitems"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
self.assertEqual(len(output.get_list()), 3)
def configure(conf: WhatTheFileConfiguration):
try:
Time._output_format = conf.get_property("whatthefile",
"output_date_format")
except:
pass
"using default value"
def test_ignore(self):
conf = WhatTheFileConfiguration()
conf.parse_dict({
"whatthefile": {
"modules_package": "src.modules",
"output": "list",
"log_output": "stdout",
"safe_output_path": "./tests/examples/safe_directory"
},
"module.hashes": {
"active": True,
'hashes_to_calculate': "MD5,SHA1,SHA256"
},
"module.ignore": {
"active":
True,
'file_hashes_md5_to_ignore':
'./tests/examples/ignoredhashesmd5.txt'
},
"module.imagerecognitiontensorflow": {
"active": False
},
"module.metadata": {
"active": False
},
"module.ocrtesseract": {
"active": False
},
"module.qrbcreader": {
"active": False
},
"module.strings": {
"active": True,
"char_min": 10
},
"module.virustotal": {
"active": False
},
"module.zipextractor": {
"active": False
}
})
path = "./tests/examples/collie.jpg"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
self.assertEqual(len(output.get_list()), 0)
def configure(conf: WhatTheFileConfiguration):
output = conf.get_property("whatthefile", "log_output")
switcher = {
"stdout":
sys.stdout,
"stderr":
sys.stderr,
"file":
conf.get_section("log_output.file")["filename"] +
str("_{time}.log") if "filename"
in conf.get_section("log_output.file") else sys.stderr
}
logger.remove()
logger.add(switcher.get(output, sys.stderr),
format="{time} {level} {message}",
level=conf.get_section("whatthefile")["log_level"] if
"log_level" in conf.get_section("whatthefile") else "DEBUG")
def configure(conf: WhatTheFileConfiguration):
try:
Safe.i = 0
Safe.safe_output_path = os.path.abspath(
os.path.join(
conf.get_property("whatthefile", "safe_output_path"),
"./" + str(Safe.i)).replace("/./",
"/").replace("/../", "/"))
Safe.next_rotation()
except:
raise BaseException("safe_output_path is required")
def _load_modules(self, conf: WhatTheFileConfiguration) -> None:
module_index = 0
for module_section_name in conf.get_modules_section_names():
try:
active = conf.get_property_boolean(module_section_name,
"active")
except:
active = False
if active:
Log.info("Loading module: [" + module_section_name + "]")
mod = self._import_module(conf, module_section_name,
module_index)
if mod is not None:
module_index += 1
self.__modules.append(mod)
Log.info("Module loaded successfully: [" +
str(mod.get_index()) + "] - " + mod.get_name())
else:
Log.error("Error loading module: [" + module_section_name +
"]")
def test_zipextractor_unzip_with_zip_inside(self):
output_safe_directory = "./tests/examples/safe_directory"
final_file = os.path.abspath(output_safe_directory) + \
"/2/zipextractor/" + \
os.path.abspath(output_safe_directory).replace(":", "") + \
"/1/zipextractor/" + \
os.path.abspath("./tests/examples/folderzip.zip/folderzip/Surprisezip.txt.zip/Surprisezip.txt").replace(":", "")
final_file = os.path.abspath(final_file)
temporal_zip = os.path.abspath(output_safe_directory) + \
"/1/zipextractor/" + \
os.path.abspath("./tests/examples/folderzip.zip/folderzip/Surprisezip.txt.zip").replace(":", "")
temporal_zip = os.path.abspath(temporal_zip)
conf = WhatTheFileConfiguration()
conf.parse_string("""
[whatthefile]
modules_package = src.modules
safe_output_path = """ + output_safe_directory + """
output = list
log_output = stdout
[module.zipextractor]
active = true
""")
if os.path.exists(final_file):
os.remove(final_file)
self._remove_test_folders(output_safe_directory)
self.assertFalse(os.path.exists(final_file))
path = "./tests/examples/folderzip.zip"
output = OutputFactory.get_output_by_conf(conf)
core = Core(conf, output)
core.run(path)
paths = []
for element in output.get_list():
paths.append(os.path.abspath(element["path"]))
self.assertTrue(temporal_zip in paths)
self._remove_test_folders(output_safe_directory)
def importRoutes(rootpath, app, config_object: Config):
"""Add user routes to app."""
conf = WhatTheFileConfiguration()
conf.parse_file(config_object.WHATTHEFILECONFIGFILE)
output = ListOutput()
core = Core(conf, output)
@app.route(rootpath, methods=['GET', 'POST'])
def index_or_upload_file():
if request.method == 'GET':
return send_file("pages/index.html", mimetype='text/html')
else:
if 'fileToUpload' not in request.files:
abort(404)
else:
file = request.files['fileToUpload']
binary = file.read()
if len(binary) != 0:
path = _write_file(config_object, binary,
os.path.basename(file.filename))
output.get_list().clear()
core.run(path)
_remove_file(path)
core.clean_safe_output_path()
result = output.get_list()
remove_internal_info(result)
return Response(json.dumps(result, default=str),
200,
mimetype='application/json')
else:
return Response(json.dumps({"error": "invalid file"},
default=str),
400,
mimetype='application/json')
@app.route(rootpath + "favicon.ico", methods=['GET'])
def get_favicon():
return send_file("images/favicon.png", mimetype='image/png')
def get_output_by_conf(conf: WhatTheFileConfiguration):
if "output" in conf.get_whatthefile_secction():
output = conf.get_whatthefile_secction()["output"]
if output == "print":
return OutputFactory._get_print_output(conf.get_section("output.print"))
if output == "list":
return OutputFactory._get_list_output(conf.get_section("output.list"))
if output == "file":
return OutputFactory._get_file_output(conf.get_section("output.file"))
if output == "elasticsearch":
return OutputFactory._get_elasticsearch_output(conf.get_section("output.elasticsearch"))
return OutputFactory._get_print_output(conf.get_section("output.print"))
def test_load_conf_dict(self):
conf = WhatTheFileConfiguration()
conf.parse_dict(self.get_conf_dict())
self.assertEqual(len(conf.get_modules_names()), 10)
self.assertTrue(conf.get_property_boolean("module.hashes", "active"))
self.assertTrue("active" in conf.get_section("module.hashes"))
def test_load_modules(self):
config = WhatTheFileConfiguration()
config.parse_dict(self.get_config_dict())
modules = LoaderModules(config).get_modules()
self.assertEqual(len(modules), len(config.get_modules_names()))
def test_load_simple_modules(self):
config = WhatTheFileConfiguration()
config.parse_dict(self.get_simple_config_dict())
modules = LoaderModules(config).get_modules()
self.assertEqual(len(modules), 1)
self.assertEqual(modules[0].get_name(), "entropy")
def reset(conf: WhatTheFileConfiguration):
safe_output_path = conf.get_property("whatthefile", "safe_output_path")
for element in os.listdir(safe_output_path):
Safe._delete(os.path.join(safe_output_path, "./" + str(element)))
Safe.configure(conf)
def test_load_conf_file(self):
conf = WhatTheFileConfiguration()
conf.parse_file('./tests/examples/whatthefile.ini')
self.assertEqual(len(conf.get_modules_names()), 10)
self.assertTrue(conf.get_property_boolean("module.hashes", "active"))
self.assertTrue("active" in conf.get_section("module.hashes"))
