def main(self) -> list:
# Perform Operations on self.data to unpack the sample
pe = pefile.PE(data=self.data)
extractedPayload, extractedDecryptionSection, extractedValue = self.selectingSections(
pe)
decrementationCounter = extractedValue // 512 # that's how it is calculated
obfuscatedPayload = self.payloadDecrypt(
self.payloadDecode(extractedPayload), decrementationCounter)
deobfuscatedPayload = self.runObfuscationCode(obfuscatedPayload)
unpackedExecutable = self.decryptSecondStage(
deobfuscatedPayload, extractedDecryptionSection)
task = Task(
headers={
'type': 'sample',
'kind': 'runnable',
'stage': 'recognized'
},
payload={
'parent':
Resource(name='sample',
content=self.data), # Set Parent Data (Packed Sample)
'sample': Resource(name='unpacked', content=unpackedExecutable
) # Set Child Data (Unpacked Sample)
})
# A list of tasks must be returned, as there can be more than one unpacked child
return [task]
def build_profile_payload(self) -> Dict[str, LocalResource]:
with tempfile.TemporaryDirectory() as tmp_path:
tmp_dir = Path(tmp_path)
for profile in dll_file_list:
fpath = Path(PROFILE_DIR) / f"{profile.dest}.json"
if fpath.is_file():
shutil.copy(fpath, tmp_dir / fpath.name)
return Resource.from_directory(name="profiles", directory_path=tmp_dir)
def test_pass(self) -> None:
res = Resource("sample", b"z")
task = Task(
{
"type": "sample",
"stage": "recognized",
"kind": "runnable"
},
payload={"sample": res},
)
res_tasks = self.run_task(task)
self.assertTasksEqual(res_tasks, [])
def get_tasks(self):
global memory_dumps
self.memory_dump_cleanup()
tasks = []
if len(memory_dumps) <= 0:
return tasks
log.info(
"Successfully Extracted {memory_dumps_count} Suspicious Memory Dump(s)"
.format(memory_dumps_count=len(memory_dumps)))
for memory_dump in memory_dumps:
if self.config['debug'] is True:
log.debug('EXTRACTED_PAYLOAD:')
hexdump.hexdump(memory_dump)
tasks.append(
Task(headers=self.get_headers(memory_dump),
payload={
'parent': Resource(name='sample', content=self.data),
'sample': Resource(name='unpacked',
content=memory_dump)
}))
return tasks
def upload():
producer = Producer(conf)
with NamedTemporaryFile() as f:
request.files["file"].save(f.name)
with open(f.name, "rb") as fr:
sample = Resource("sample", fr.read())
task = Task({"type": "sample", "stage": "recognized", "platform": "win32"})
task.add_payload("override_uid", task.uid)
# Add analysis timeout to task
timeout = request.form.get("timeout")
if timeout:
task.add_payload("timeout", int(timeout))
# Add filename override to task
if request.form.get("file_name"):
filename = request.form.get("file_name")
else:
filename = request.files["file"].filename
if not re.fullmatch(
r"^((?![\\/><|:&])[\x20-\xfe])+\.(?:dll|exe|ps1|bat|doc|docm|docx|dotm|xls|xlsx|xlsm|xltx|xltm|ppt|pptx|vbs|js|jse|hta|html|htm)$",
filename,
flags=re.IGNORECASE,
):
return jsonify({"error": "invalid file_name"}), 400
task.add_payload("file_name", os.path.splitext(filename)[0])
# Extract and add extension
extension = os.path.splitext(filename)[1][1:]
if extension:
task.headers["extension"] = extension
# Add startup command to task
start_command = request.form.get("start_command")
if start_command:
task.add_payload("start_command", start_command)
# Add plugins to task
plugins = request.form.get("plugins")
if plugins:
plugins = json.loads(plugins)
task.add_payload("plugins", plugins)
task.add_resource("sample", sample)
producer.send_task(task)
return jsonify({"task_uid": task.uid})
def main(self) -> list:
log.info(f"upx unpacking {self.name}")
upx = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'upx')
sample_unpacked = tempfile.mktemp()
sample_packed = tempfile.mktemp()
f = open(sample_packed, 'wb')
f.write(self.data)
f.close()
command = [upx, '-d', sample_packed, '-o', sample_unpacked]
command = subprocess.list2cmdline(command)
out = ''
with timeout(self.config['timeout']):
try:
out = subprocess.getoutput(command)
except Exception as error:
log.error(error)
return []
if os.path.exists(sample_packed):
os.remove(sample_packed)
if "Unpacked 1 file".lower() in out.lower():
log.info(f"saved unpacked upx executable to {sample_unpacked}")
f = open(sample_unpacked, 'rb')
child_resource = Resource(name='unpacked', content=f.read())
f.close()
if os.path.exists(sample_unpacked):
os.remove(sample_unpacked)
task = Task(headers={
"type": "sample",
"kind": "raw"
},
payload={
"parent": Resource(name='sample',
content=self.data),
"sample": child_resource
})
return [task]
log.error(f"failed to unpack: {self.name}")
return []
def main(self):
config_encrypted = self.extract_encrypted_config()
config_encrypted = re.sub(br'\x00+', b'\x00', config_encrypted)
config_encrypted = config_encrypted.split(b'\x00')
self.setup_emulator()
log.debug('emulation started')
self.mu.emu_start(self.BASE_ADDRESS,
self.BASE_ADDRESS + len(self.CODE))
log.debug('emulation completed')
unpacked = self.mu.mem_read(self.DATA_ADDRESS, len(self.PACKED_CODE))
if unpacked is not None:
log.debug('succesfully unpacked binary')
task = Task(headers={
"type": "sample",
"kind": "raw"
},
payload={
"parent":
Resource(name='sample', content=self.data),
"sample":
Resource(name='unpacked', content=bytes(unpacked))
})
return [task]
return []
def test_process_unknown_file(self):
resource = Resource("file.txt", b"\x00", sha256="sha256")
res = self.run_task(mock_task(resource))
expected = Task(
headers={
"type": "sample",
"stage": "unrecognized",
"origin": "karton.classifier",
"kind": "unknown",
"quality": "high",
},
payload={
"sample": resource,
},
)
self.assertTasksEqual(res, [expected])
def submit_main(cls):
parser = cls.args_parser()
args = parser.parse_args()
conf_path = os.path.join(ETC_DIR, "config.ini")
config = patch_config(Config(conf_path))
with open(args.tests) as tests:
testcases = [TestCase(**case) for case in json.load(tests)]
root_uids = []
for test in testcases:
sample = test.get_sample()
sys.stderr.write(f"Submitting {test.sha256}\n")
t = Task(headers=dict(type="sample-test", platform="win64"))
t.add_payload("sample", Resource("malwar", sample))
t.add_payload("testcase", test.to_json())
if args.timeout:
t.add_payload("timeout", args.timeout)
p = Producer(config)
p.send_task(t)
root_uids.append(t.root_uid)
consumer = RegressionTester(config)
results = {}
with tqdm(total=len(root_uids)) as pbar:
while len(results) != len(root_uids):
for root_uid in cls.get_finished_tasks(consumer.backend,
root_uids):
if root_uid not in results:
res = json.load(
consumer.backend.minio.get_object(
"draktestd", root_uid))
results[root_uid] = res
print(json.dumps(results[root_uid]))
pbar.update(1)
time.sleep(1)
print(json.dumps(list(results.values())))
def send_file_to_karton(file) -> str:
from mwdb.model.file import File
tmpfile = None
try:
# TODO: Use file.open() directly when Resource(fd=...)
# is implemented in Karton
try:
# If file contents are available via path: just use the path
path = file.get_path()
except (ValueError, IOError):
# If get_path doesn't work: download content to NamedTemporaryFile
tmpfile = tempfile.NamedTemporaryFile()
file_stream = file.open()
shutil.copyfileobj(file_stream, tmpfile)
File.close(file_stream)
path = tmpfile.name
producer = get_karton_producer()
feed_quality = g.auth_user.feed_quality
task_priority = (TaskPriority.NORMAL
if feed_quality == "high" else TaskPriority.LOW)
task = Task(
headers={
"type": "sample",
"kind": "raw",
"quality": feed_quality
},
payload={
"sample":
Resource(file.file_name, path=path, sha256=file.sha256),
"attributes":
file.get_attributes(as_dict=True, check_permissions=False),
},
priority=task_priority,
)
producer.send_task(task)
finally:
if tmpfile is not None:
tmpfile.close()
logger.info("File sent to Karton with %s", task.root_uid)
return task.root_uid
def main():
parser = argparse.ArgumentParser(description="Push sample to the karton")
parser.add_argument("sample", help="Path to the sample")
parser.add_argument(
"--start_command",
help="e.g. start %f, %f will be replaced by file name",
required=False,
)
parser.add_argument(
"--timeout",
default=600,
type=int,
help="analysis timeout in seconds",
required=False,
)
args = parser.parse_args()
conf = patch_config(Config(os.path.join(ETC_DIR, "config.ini")))
producer = Producer(conf)
task = Task({"type": "sample", "stage": "recognized", "platform": "win32"})
with open(args.sample, "rb") as f:
sample = Resource("sample", f.read())
task.add_resource("sample", sample)
# Add filename
filename = os.path.basename(args.sample)
task.add_payload("file_name", os.path.splitext(filename)[0])
# Extract and add extension
extension = os.path.splitext(filename)[1][1:]
if extension:
task.headers["extension"] = extension
if args.start_command is not None:
task.add_payload("start_command", args.start_command)
if args.timeout is not None:
task.add_payload("timeout", args.timeout)
producer.send_task(task)
def test_process(self):
resource = Resource("file.txt",
b"ffafafffa\nfafafafa",
sha256="sha256")
res = self.run_task(mock_task(resource))
expected = Task(
headers={
"type": "sample",
"stage": "recognized",
"origin": "karton.classifier",
"quality": "high",
"kind": "ascii",
"mime": "text/plain",
},
payload={
"sample": resource,
"tags": ["misc:ascii"],
"magic": "ASCII text",
},
)
self.assertTasksEqual(res, [expected])
def send_file_to_karton(file: File) -> str:
try:
path = file.get_path()
tmpfile = None
except Exception:
# If get_path doesn't work: download content to NamedTemporaryFile
# It won't work if we use S3 storage and try to reanalyze
# existing file (not uploaded within the same request).
tmpfile = tempfile.NamedTemporaryFile()
file_stream = file.open()
shutil.copyfileobj(file_stream, tmpfile)
File.close(file_stream)
path = tmpfile.name
producer = Producer(identity="karton.mwdb",
config=KartonConfig(config.karton.config_path))
feed_quality = g.auth_user.feed_quality
task_priority = TaskPriority.NORMAL if feed_quality == "high" else TaskPriority.LOW
task = Task(headers={
"type": "sample",
"kind": "raw",
"quality": feed_quality
},
payload={
"sample":
Resource(file.file_name, path=path, sha256=file.sha256),
"attributes":
file.get_metakeys(as_dict=True, check_permissions=False)
},
priority=task_priority)
producer.send_task(task)
if tmpfile is not None:
tmpfile.close()
file.add_metakey("karton", task.root_uid, check_permissions=False)
logger.info("File sent to karton with %s", task.root_uid)
return task.root_uid
def test_process_error(self):
m = MagicMock()
m.side_effect = Exception("unknown error")
self.karton = Classifier(magic=m,
config=self.config,
backend=self.backend)
resource = Resource("file.txt", b"ffafafffa", sha256="sha256")
res = self.run_task(mock_task(resource))
expected = Task(
headers={
"type": "sample",
"stage": "unrecognized",
"origin": "karton.classifier",
"kind": "unknown",
"quality": "high",
},
payload={
"sample": resource,
},
)
self.assertTasksEqual(res, [expected])
def test_match_2(self) -> None:
res = Resource("sample", b"ab")
input_task = Task(
{
"type": "sample",
"stage": "recognized",
"kind": "runnable"
},
payload={"sample": res},
)
expected_task = Task(
{
"type": "sample",
"origin": "karton.yaramatcher",
"stage": "analyzed"
},
payload={
"sample": res,
"tags": ["yara:a", "yara:b"]
},
)
res_tasks = self.run_task(input_task)
self.assertTasksEqual(res_tasks, [expected_task])
def process(self, task: Task) -> None:
# Get the incoming sample
sample_resource = task.get_resource("sample")
# Log with self.log
self.log.info(f"Hi {sample_resource.name}, let me analyse you!")
# Download the resource to a temporary file
with sample_resource.download_temporary_file() as sample_file:
# And run `strings` on it
strings = subprocess.check_output(["strings", sample_file.name])
# Send our results for further processing or reporting
task = Task(
{
"type": "sample",
"stage": "analyzed"
},
payload={
"parent": sample_resource,
"sample": Resource("string", strings)
},
)
self.send_task(task)
def process(self, task: Task) -> None: # type: ignore
sample = task.get_resource("sample")
ascii_content = sample.content
classifier = AsciiClassifier(ascii_content)
classifier.classify()
decoder = Decoder(ascii_content, classifier.verdict)
try:
decoder.decode()
except binascii.Error:
logging.warning("Error why trying to decode base64.")
return
if decoder.decoded:
self.log.info("Decoded possible executable")
if decoder.decoded[:2] == b"MZ":
task_params = {
"type": "sample",
"kind": "runnable",
"stage": "recognized",
"platform": "win32",
"extension": "exe",
}
else:
task_params = {"type": "sample", "kind": "raw"}
new_sample = Resource(
sample.name,
decoder.decoded,
)
task = Task(task_params,
payload={
"sample": new_sample,
"parent": sample
})
self.send_task(task)
return [task]
return []
if __name__ in '__main__':
parser = argparse.ArgumentParser(
prog='fatalrat.py',
description=
f'Karton Unpacker Service Example Module v{__version__} (CLI Test Utility)',
epilog=f'Author: {__author__}')
parser.add_argument('-i',
'--input',
help='Input File',
type=str,
required=True)
parser.add_argument('--debug',
help='Debug',
action="store_true",
default=False,
required=False)
args = parser.parse_args()
f = open(args.input, 'rb')
sample = Resource(name=args.input, content=f.read())
f.close()
module = KartonUnpackerModule(sample, {'debug': args.debug})
if module.enabled is True:
log.debug('unpacking started')
task = module.main()
data = json.loads(str(task))
print(json.dumps(data, indent=4))
def mock_resource(filename: str, with_name=False) -> Resource:
filepath = tests_dir / "testdata" / filename
return Resource(
filename if with_name else "file", filepath.read_bytes(), sha256="sha256"
)
def process(self, task: Task) -> None:
sample = task.get_resource("sample")
task_password = task.get_payload("password", default=None)
attributes = task.get_payload("attributes", default={})
if not task_password and attributes.get("password"):
self.log.info("Accepting password from attributes")
task_password = attributes.get("password")[0]
try:
if sample.name:
fname = sample.name.encode("utf8")
classifier_extension = "." + task.headers.get("extension")
if classifier_extension and not fname.endswith(
classifier_extension.encode("utf-8")):
fname += classifier_extension.encode("utf-8")
except Exception as e:
self.log.warning("Exception during extraction: %r", e)
fname = None
extraction_level = task.get_payload("extraction_level", 0)
if extraction_level > self.max_depth:
self.log.warning(
"Maximum extraction depth exceeded. Can't extract this archive."
)
return
with tempfile.TemporaryDirectory() as dir_name:
filepath = f"{dir_name}/{fname}"
with open(filepath, "wb") as f:
f.write(sample.content)
archive_password = None
if task_password is not None:
archive_password = task_password.encode()
unpacked = unpack(
filename=fname,
filepath=filepath.encode("utf-8"),
password=archive_password,
)
try:
fname = (unpacked.filename
and unpacked.filename.decode("utf8")) or unpacked.sha256
except Exception as e:
self.log.warning("Exception during extraction: %r", e)
fname = "(unknown)"
self.log.info("Got archive {}".format(fname))
if not unpacked.children:
self.log.warning("Don't know how to unpack this archive")
return
for child in unpacked.children:
fname = (child.filename
and child.filename.decode("utf8")) or child.sha256
self.log.info("Unpacked child {}".format(fname))
if not child.contents:
self.log.warning(
"Child has no contents or protected by unknown password")
continue
if len(child.contents) > self.max_size:
self.log.warning("Child is too big for further processing")
continue
task = Task(
headers={
"type": "sample",
"kind": "raw",
"quality": task.headers.get("quality", "high"),
},
payload={
"sample": Resource(fname, child.contents),
"parent": sample,
"extraction_level": extraction_level + 1,
},
)
self.send_task(task)
def analyze_dumps(self, sample, dump_infos):
"""
Analyse multiple dumps from given sample. There can be more than one
dump from which we managed to extract config from – try to find the best
candidate for each family.
"""
extractor = create_extractor(self)
dump_candidates = {}
results = {
"analysed": 0,
"crashed": 0,
}
for i, dump_info in enumerate(dump_infos):
dump_basename = os.path.basename(dump_info.path)
results["analysed"] += 1
self.log.debug("Analyzing dump %d/%d %s", i, len(dump_infos),
str(dump_basename))
with open(dump_info.path, "rb") as f:
dump_data = f.read()
if not dump_data:
self.log.warning("Dump {} is empty".format(dump_basename))
continue
try:
family = extractor.push_file(dump_info.path,
base=dump_info.base)
if family:
self.log.info("Found better %s config in %s", family,
dump_basename)
dump_candidates[family] = (dump_basename, dump_data)
except Exception:
self.log.exception(
"Error while extracting from {}".format(dump_basename))
results["crashed"] += 1
self.log.debug("Finished analysing dump no. %d", i)
self.log.info("Merging and reporting extracted configs")
for family, config in extractor.configs.items():
dump_basename, dump_data = dump_candidates[family]
self.log.info("* (%s) %s => %s", family, dump_basename,
json.dumps(config))
parent = Resource(name=dump_basename, content=dump_data)
task = Task(
{
"type": "sample",
"stage": "analyzed",
"kind": "dump",
"platform": "win32",
"extension": "exe",
},
payload={
"sample": parent,
"parent": sample,
"tags": ["dump:win32:exe"],
},
)
self.send_task(task)
self.report_config(config, sample, parent=parent)
self.log.info("done analysing, results: {}".format(
json.dumps(results)))
def process(self, task: Task) -> None:
sample = task.get_resource("sample")
resources = None
m = self.yara.match(data=sample.content)
if "autoit_v3_00" in m:
self.log.info("Found a possible autoit v3.00 binary")
resources = extract(data=sample.content,
version=AutoItVersion.EA05)
elif "autoit_v3_26" in m:
self.log.info("Found a possible autoit v3.26+ binary")
resources = extract(data=sample.content,
version=AutoItVersion.EA06)
if resources:
self.log.info("Found embedded data, reporting!")
for res_name, res_data in resources:
if res_name.endswith(".dll") or res_name.endswith(".exe"):
task_params = {
"type": "sample",
"kind": "raw",
}
elif res_name == "script.au3":
task_params = {
"type": "sample",
"kind": "script",
"stage": "analyzed",
"extension": "au3",
}
else:
continue
self.log.info("Sending a task with %s", res_name)
script = Resource(res_name, res_data)
self.send_task(
Task(task_params,
payload={
"sample": script,
"parent": sample
}))
if res_name == "script.au3":
self.log.info(
"Looking for a binary embedded in the script")
drop = extract_binary(res_data.decode())
if drop:
self.log.info("Found an embedded binary")
self.send_task(
Task(
{
"type": "sample",
"kind": "raw"
},
payload={
"sample":
Resource(name="autoit_drop.exe",
content=drop),
"parent":
script,
},
))
def mock_resource(filename: str) -> Resource:
return Resource(filename, b"feeddecaf\n", sha256="sha256")