def analyse_class_file(
self, file_res, cf, cur_file, cur_file_path, start_bytes, imp_res_list, supplementary_files, decompiled_dir,
extract_dir):
if start_bytes[:4] == b"\xCA\xFE\xBA\xBE":
cur_file.seek(0)
cur_file_full_data = cur_file.read()
# Analyse file for suspicious functions
if self.do_class_analysis(cur_file_full_data):
self.decompile_class(cur_file_path, supplementary_files, decompiled_dir, extract_dir)
else:
# Could not deobfuscate
cur_file.seek(0)
first_256 = cur_file.read(256)
ob_res = dict(
title_text=f"Class file {cf} doesn't have the normal class files magic bytes. "
"The file was re-submitted for analysis. Here are the first 256 bytes:",
body=hexdump(first_256),
body_format=BODY_FORMAT.MEMORY_DUMP,
heur_id=3,
tags=[('file.behavior', "Suspicious Java Class")],
files=[cur_file_path],
)
imp_res_list.append(ob_res)
def get_file_hex(sha256, **kwargs):
"""
Returns the file hex representation
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
None
Data Block:
None
API call example:
/api/v4/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
if not file_obj:
return make_api_response({}, "The file was not found in the system.", 404)
if user and Classification.is_accessible(user['classification'], file_obj['classification']):
with forge.get_filestore() as f_transport:
data = f_transport.get(sha256)
if not data:
return make_api_response({}, "This file was not found in the system.", 404)
return make_api_response(hexdump(data))
else:
return make_api_response({}, "You are not allowed to view this file.", 403)
def test_hexdump():
data = bytes([random.randint(1, 255) for _ in range(10000)])
dumped = hexdump(data)
line = dumped.splitlines()[random.randint(1, 200)]
_ = int(line[:8], 16)
assert len(line) == 77
assert line[8:11] == ": "
for c in chunk(line[11:59], 3):
assert c[0] in "abcdef1234567890"
assert c[1] in "abcdef1234567890"
assert c[2] == " "
assert line[59:59 + 2] == " "
def get_file_hex(sha256, **kwargs):
"""
Returns the file hex representation
Variables:
sha256 => A resource locator for the file (sha256)
Arguments:
bytes_only => Only return bytes with no formatting
length => Number of bytes per lines
Data Block:
None
API call example:
/api/v4/file/hex/123456...654321/
Result example:
<THE FILE HEX REPRESENTATION>
"""
user = kwargs['user']
file_obj = STORAGE.file.get(sha256, as_obj=False)
bytes_only = request.args.get('bytes_only',
'false').lower() in ['true', '']
length = int(request.args.get('length', '16'))
if not file_obj:
return make_api_response({}, "The file was not found in the system.",
404)
if file_obj['size'] > API_MAX_SIZE:
return make_api_response(
{}, "This file is too big to be seen through this API.", 403)
if user and Classification.is_accessible(user['classification'],
file_obj['classification']):
data = FILESTORE.get(sha256)
if not data:
return make_api_response({},
"This file was not found in the system.",
404)
if bytes_only:
return make_api_response(dump(data).decode())
else:
return make_api_response(hexdump(data, length=length))
else:
return make_api_response({}, "You are not allowed to view this file.",
403)
# print slack space if it exists
if (self.print_slack and self.filesize_from_peheader > 0 and (
len(self.pe_file.__data__) > self.filesize_from_peheader)):
length_to_display = PEFILE_SLACK_LENGTH_TO_DISPLAY
if length_to_display > 0:
length_display_str = ""
slack_size = len(self.pe_file.__data__) - self.filesize_from_peheader
if slack_size > length_to_display:
length_display_str = "- displaying first %d bytes" % length_to_display
pe_slack_space_res = ResultSection(SCORE['NULL'],
"PE: SLACK SPACE (The file contents after the PE file size ends) "
"[%d bytes] %s" % (
len(self.pe_file.__data__) - self.filesize_from_peheader,
length_display_str),
body_format=TEXT_FORMAT['MEMORY_DUMP'])
pe_slack_space_res.add_line(hexdump(
self.pe_file.__data__[self.filesize_from_peheader:self.filesize_from_peheader + length_to_display]))
self.file_res.add_section(pe_slack_space_res)
def _init_section_list(self):
# Lazy init
if self._sect_list is None:
self._sect_list = []
try:
for section in self.pe_file.sections:
zero_idx = section.Name.find(chr(0x0))
if not zero_idx == -1:
sname = section.Name[:zero_idx]
else:
sname = safe_str(section.Name)
entropy = section.get_entropy()
self._sect_list.append((sname, section, section.get_hash_md5(), entropy))
def peepdf_analysis(self, temp_filename, file_content, request):
file_res = Result()
try:
res_list = []
# js_stream = []
f_list = []
js_dump = []
pdf_parser = PDFParser()
ret, pdf_file = pdf_parser.parse(temp_filename, True, False, file_content)
if ret == 0:
stats_dict = pdf_file.getStats()
if ", ".join(stats_dict['Errors']) == "Bad PDF header, %%EOF not found, PDF sections not found, No " \
"indirect objects found in the body":
# Not a PDF
return
json_body = dict(
version=stats_dict['Version'],
binary=stats_dict['Binary'],
linearized=stats_dict['Linearized'],
encrypted=stats_dict['Encrypted'],
)
if stats_dict['Encryption Algorithms']:
temp = []
for algorithmInfo in stats_dict['Encryption Algorithms']:
temp.append(f"{algorithmInfo[0]} {str(algorithmInfo[1])} bits")
json_body["encryption_algorithms"] = temp
json_body.update(dict(
updates=stats_dict['Updates'],
objects=stats_dict['Objects'],
streams=stats_dict['Streams'],
comments=stats_dict['Comments'],
errors={True: ", ".join(stats_dict['Errors']),
False: "None"}[len(stats_dict['Errors']) != 0]
))
res = ResultSection("PDF File Information", body_format=BODY_FORMAT.KEY_VALUE,
body=json.dumps(json_body))
for version in range(len(stats_dict['Versions'])):
stats_version = stats_dict['Versions'][version]
v_json_body = dict(
catalog=stats_version['Catalog'] or "no",
info=stats_version['Info'] or "no",
objects=self.list_first_x(stats_version['Objects'][1]),
)
if stats_version['Compressed Objects'] is not None:
v_json_body['compressed_objects'] = self.list_first_x(stats_version['Compressed Objects'][1])
if stats_version['Errors'] is not None:
v_json_body['errors'] = self.list_first_x(stats_version['Errors'][1])
v_json_body['streams'] = self.list_first_x(stats_version['Streams'][1])
if stats_version['Xref Streams'] is not None:
v_json_body['xref_streams'] = self.list_first_x(stats_version['Xref Streams'][1])
if stats_version['Object Streams'] is not None:
v_json_body['object_streams'] = self.list_first_x(stats_version['Object Streams'][1])
if int(stats_version['Streams'][0]) > 0:
v_json_body['encoded'] = self.list_first_x(stats_version['Encoded'][1])
if stats_version['Decoding Errors'] is not None:
v_json_body['decoding_errors'] = self.list_first_x(stats_version['Decoding Errors'][1])
if stats_version['Objects with JS code'] is not None:
v_json_body['objects_with_js_code'] = \
self.list_first_x(stats_version['Objects with JS code'][1])
# js_stream.extend(stats_version['Objects with JS code'][1])
res_version = ResultSection(f"Version {str(version)}", parent=res,
body_format=BODY_FORMAT.KEY_VALUE, body=json.dumps(v_json_body))
actions = stats_version['Actions']
events = stats_version['Events']
vulns = stats_version['Vulns']
elements = stats_version['Elements']
is_suspicious = False
if events is not None or actions is not None or vulns is not None or elements is not None:
res_suspicious = ResultSection('Suspicious elements', parent=res_version)
if events is not None:
for event in events:
res_suspicious.add_line(f"{event}: {self.list_first_x(events[event])}")
is_suspicious = True
if actions is not None:
for action in actions:
res_suspicious.add_line(f"{action}: {self.list_first_x(actions[action])}")
is_suspicious = True
if vulns is not None:
for vuln in vulns:
if vuln in vulnsDict:
temp = [vuln, ' (']
for vulnCVE in vulnsDict[vuln]:
if len(temp) != 2:
temp.append(',')
vulnCVE = "".join(vulnCVE) if isinstance(vulnCVE, list) else vulnCVE
temp.append(vulnCVE)
cve_found = re.search("CVE-[0-9]{4}-[0-9]{4}", vulnCVE)
if cve_found:
res_suspicious.add_tag('attribution.exploit',
vulnCVE[cve_found.start():cve_found.end()])
res_suspicious.add_tag('file.behavior',
vulnCVE[cve_found.start():cve_found.end()])
temp.append('): ')
temp.append(str(vulns[vuln]))
res_suspicious.add_line(temp)
else:
res_suspicious.add_line(f"{vuln}: {str(vulns[vuln])}")
is_suspicious = True
if elements is not None:
for element in elements:
if element in vulnsDict:
temp = [element, ' (']
for vulnCVE in vulnsDict[element]:
if len(temp) != 2:
temp.append(',')
vulnCVE = "".join(vulnCVE) if isinstance(vulnCVE, list) else vulnCVE
temp.append(vulnCVE)
cve_found = re.search("CVE-[0-9]{4}-[0-9]{4}", vulnCVE)
if cve_found:
res_suspicious.add_tag('attribution.exploit',
vulnCVE[cve_found.start():cve_found.end()])
res_suspicious.add_tag('file.behavior',
vulnCVE[cve_found.start():cve_found.end()])
temp.append('): ')
temp.append(str(elements[element]))
res_suspicious.add_line(temp)
is_suspicious = True
else:
res_suspicious.add_line(f"\t\t{element}: {str(elements[element])}")
is_suspicious = True
res_suspicious.set_heuristic(8) if is_suspicious else None
urls = stats_version['URLs']
if urls is not None:
res.add_line("")
res_url = ResultSection('Found URLs', parent=res)
for url in urls:
res_url.add_line(f"\t\t{url}")
res_url.set_heuristic(9)
for obj in stats_version['Objects'][1]:
cur_obj = pdf_file.getObject(obj, version)
if cur_obj.containsJScode:
cur_res = ResultSection(f"Object [{obj} {version}] contains {len(cur_obj.JSCode)} "
f"block of JavaScript")
score_modifier = 0
js_idx = 0
for js in cur_obj.JSCode:
sub_res = ResultSection('Block of JavaScript', parent=cur_res)
js_idx += 1
js_score = 0
js_code, unescaped_bytes, _, _, _ = analyseJS(js)
js_dump += [x for x in js_code]
# Malicious characteristics
big_buffs = self.get_big_buffs("".join(js_code))
if len(big_buffs) == 1:
js_score += 500 * len(big_buffs)
if len(big_buffs) > 0:
js_score += 500 * len(big_buffs)
has_eval, has_unescape = self.check_dangerous_func("".join(js_code))
if has_unescape:
js_score += 100
if has_eval:
js_score += 100
js_cmt = ""
if has_eval or has_unescape or len(big_buffs) > 0:
score_modifier += js_score
js_cmt = "Suspiciously malicious "
cur_res.add_tag('file.behavior', "Suspicious JavaScript in PDF")
sub_res.set_heuristic(7)
js_res = ResultSection(f"{js_cmt}JavaScript Code (block: {js_idx})", parent=sub_res)
if js_score > 0:
temp_js_outname = f"object{obj}-{version}_{js_idx}.js"
temp_js_path = os.path.join(self.working_directory, temp_js_outname)
temp_js_bin = "".join(js_code).encode("utf-8")
f = open(temp_js_path, "wb")
f.write(temp_js_bin)
f.close()
f_list.append(temp_js_path)
js_res.add_line(f"The JavaScript block was saved as {temp_js_outname}")
if has_eval or has_unescape:
analysis_res = ResultSection("[Suspicious Functions]", parent=js_res)
if has_eval:
analysis_res.add_line("eval: This JavaScript block uses eval() function "
"which is often used to launch deobfuscated "
"JavaScript code.")
analysis_res.set_heuristic(3)
if has_unescape:
analysis_res.add_line("unescape: This JavaScript block uses unescape() "
"function. It may be legitimate but it is definitely "
"suspicious since malware often use this to "
"deobfuscate code blocks.")
analysis_res.set_heuristic(3)
buff_idx = 0
for buff in big_buffs:
buff_idx += 1
error, new_buff = unescape(buff)
if error == 0:
buff = new_buff
if buff not in unescaped_bytes:
temp_path_name = None
if ";base64," in buff[:100] and "data:" in buff[:100]:
temp_path_name = f"obj{obj}_unb64_{buff_idx}.buff"
try:
buff = b64decode(buff.split(";base64,")[1].strip())
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(buff)
f.close()
f_list.append(temp_path)
except Exception:
self.log.error("Found 'data:;base64, ' buffer "
"but failed to base64 decode.")
temp_path_name = None
if temp_path_name is not None:
buff_cond = f" and was resubmitted as {temp_path_name}"
else:
buff_cond = ""
buff_res = ResultSection(
f"A {len(buff)} bytes buffer was found in the JavaScript "
f"block{buff_cond}. Here are the first 256 bytes.",
parent=js_res, body=hexdump(bytes(buff[:256], "utf-8")),
body_format=BODY_FORMAT.MEMORY_DUMP)
buff_res.set_heuristic(2)
processed_sc = []
sc_idx = 0
for sc in unescaped_bytes:
if sc not in processed_sc:
sc_idx += 1
processed_sc.append(sc)
try:
sc = sc.decode("hex")
except Exception:
pass
shell_score = 500
temp_path_name = f"obj{obj}_unescaped_{sc_idx}.buff"
shell_res = ResultSection(f"Unknown unescaped {len(sc)} bytes JavaScript "
f"buffer (id: {sc_idx}) was resubmitted as "
f"{temp_path_name}. Here are the first 256 bytes.",
parent=js_res)
shell_res.set_body(hexdump(sc[:256]), body_format=BODY_FORMAT.MEMORY_DUMP)
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(sc)
f.close()
f_list.append(temp_path)
cur_res.add_tag('file.behavior', "Unescaped JavaScript Buffer")
shell_res.set_heuristic(6)
score_modifier += shell_score
if score_modifier > 0:
res_list.append(cur_res)
elif cur_obj.type == "stream":
if cur_obj.isEncodedStream and cur_obj.filter is not None:
data = cur_obj.decodedStream
encoding = cur_obj.filter.value.replace("[", "").replace("]", "").replace("/",
"").strip()
val = cur_obj.rawValue
otype = cur_obj.elements.get("/Type", None)
sub_type = cur_obj.elements.get("/Subtype", None)
length = cur_obj.elements.get("/Length", None)
else:
data = cur_obj.rawStream
encoding = None
val = cur_obj.rawValue
otype = cur_obj.elements.get("/Type", None)
sub_type = cur_obj.elements.get("/Subtype", None)
length = cur_obj.elements.get("/Length", None)
if otype:
otype = otype.value.replace("/", "").lower()
if sub_type:
sub_type = sub_type.value.replace("/", "").lower()
if length:
length = length.value
if otype == "embeddedfile":
if len(data) > 4096:
if encoding is not None:
temp_encoding_str = f"_{encoding}"
else:
temp_encoding_str = ""
cur_res = ResultSection(
f'Embedded file found ({length} bytes) [obj: {obj} {version}] '
f'and dumped for analysis {f"(Type: {otype}) " if otype is not None else ""}'
f'{f"(SubType: {sub_type}) " if sub_type is not None else ""}'
f'{f"(Encoded with {encoding})" if encoding is not None else ""}'
)
temp_path_name = f"EmbeddedFile_{obj}{temp_encoding_str}.obj"
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(data)
f.close()
f_list.append(temp_path)
cur_res.add_line(f"The EmbeddedFile object was saved as {temp_path_name}")
res_list.append(cur_res)
elif otype not in BANNED_TYPES:
cur_res = ResultSection(
f'Unknown stream found [obj: {obj} {version}] '
f'{f"(Type: {otype}) " if otype is not None else ""}'
f'{f"(SubType: {sub_type}) " if sub_type is not None else ""}'
f'{f"(Encoded with {encoding})" if encoding is not None else ""}'
)
for line in val.splitlines():
cur_res.add_line(line)
emb_res = ResultSection('First 256 bytes', parent=cur_res)
first_256 = data[:256]
if isinstance(first_256, str):
first_256 = first_256.encode()
emb_res.set_body(hexdump(first_256), BODY_FORMAT.MEMORY_DUMP)
res_list.append(cur_res)
else:
pass
file_res.add_section(res)
for results in res_list:
file_res.add_section(results)
if js_dump:
js_dump_res = ResultSection('Full JavaScript dump')
temp_js_dump = "javascript_dump.js"
temp_js_dump_path = os.path.join(self.working_directory, temp_js_dump)
try:
temp_js_dump_bin = "\n\n----\n\n".join(js_dump).encode("utf-8")
except UnicodeDecodeError:
temp_js_dump_bin = "\n\n----\n\n".join(js_dump)
temp_js_dump_sha1 = hashlib.sha1(temp_js_dump_bin).hexdigest()
f = open(temp_js_dump_path, "wb")
f.write(temp_js_dump_bin)
f.flush()
f.close()
f_list.append(temp_js_dump_path)
js_dump_res.add_line(f"The JavaScript dump was saved as {temp_js_dump}")
js_dump_res.add_line(f"The SHA-1 for the JavaScript dump is {temp_js_dump_sha1}")
js_dump_res.add_tag('file.pdf.javascript.sha1', temp_js_dump_sha1)
file_res.add_section(js_dump_res)
for filename in f_list:
request.add_extracted(filename, os.path.basename(filename),
f"Dumped from {os.path.basename(temp_filename)}")
else:
res = ResultSection("ERROR: Could not parse file with PeePDF.")
file_res.add_section(res)
finally:
request.result = file_res
try:
del pdf_file
except Exception:
pass
try:
del pdf_parser
except Exception:
pass
gc.collect()
if length_to_display > 0:
length_display_str = ""
slack_size = len(
self.pe_file.__data__) - self.filesize_from_peheader
if slack_size > length_to_display:
length_display_str = "- displaying first %d bytes" % length_to_display
pe_slack_space_res = ResultSection(
SCORE['NULL'],
"PE: SLACK SPACE (The file contents after the PE file size ends) "
"[%d bytes] %s" %
(len(self.pe_file.__data__) - self.filesize_from_peheader,
length_display_str),
body_format=TEXT_FORMAT['MEMORY_DUMP'])
pe_slack_space_res.add_line(
hexdump(
self.pe_file.__data__[self.filesize_from_peheader:self.
filesize_from_peheader +
length_to_display]))
self.file_res.add_section(pe_slack_space_res)
def _init_section_list(self):
# Lazy init
if self._sect_list is None:
self._sect_list = []
try:
for section in self.pe_file.sections:
zero_idx = section.Name.find(chr(0x0))
if not zero_idx == -1:
sname = section.Name[:zero_idx]
else:
sname = safe_str(section.Name)
entropy = section.get_entropy()
def peepdf_analysis(self, temp_filename, file_content, request):
file_res = request.result
try:
res_list = []
js_stream = []
f_list = []
js_dump = []
pdf_parser = PDFParser()
ret, pdf_file = pdf_parser.parse(temp_filename, True, False, file_content)
if ret == 0:
stats_dict = pdf_file.getStats()
if ", ".join(stats_dict['Errors']) == "Bad PDF header, %%EOF not found, PDF sections not found, No " \
"indirect objects found in the body":
# Not a PDF
return
res = ResultSection(SCORE['NULL'], "PDF File information")
res.add_line('File: ' + stats_dict['File'])
res.add_line(['MD5: ', stats_dict['MD5']])
res.add_line(['SHA1: ', stats_dict['SHA1']])
res.add_line('SHA256: ' + stats_dict['SHA256'])
res.add_line(['Size: ', stats_dict['Size'], ' bytes'])
res.add_line('Version: ' + stats_dict['Version'])
res.add_line('Binary: ' + stats_dict['Binary'])
res.add_line('Linearized: ' + stats_dict['Linearized'])
res.add_line('Encrypted: ' + stats_dict['Encrypted'])
if stats_dict['Encryption Algorithms']:
temp = ' ('
for algorithmInfo in stats_dict['Encryption Algorithms']:
temp += algorithmInfo[0] + ' ' + str(algorithmInfo[1]) + ' bits, '
temp = temp[:-2] + ')'
res.add_line(temp)
res.add_line('Updates: ' + stats_dict['Updates'])
res.add_line('Objects: ' + stats_dict['Objects'])
res.add_line('Streams: ' + stats_dict['Streams'])
res.add_line('Comments: ' + stats_dict['Comments'])
res.add_line('Errors: ' + {True: ", ".join(stats_dict['Errors']),
False: "None"}[len(stats_dict['Errors']) != 0])
res.add_line("")
for version in range(len(stats_dict['Versions'])):
stats_version = stats_dict['Versions'][version]
res_version = ResultSection(SCORE['NULL'], 'Version ' + str(version), parent=res)
if stats_version['Catalog'] is not None:
res_version.add_line('Catalog: ' + stats_version['Catalog'])
else:
res_version.add_line('Catalog: ' + 'No')
if stats_version['Info'] is not None:
res_version.add_line('Info: ' + stats_version['Info'])
else:
res_version.add_line('Info: ' + 'No')
res_version.add_line('Objects (' + stats_version['Objects'][0] + '): ' +
self.list_first_x(stats_version['Objects'][1]))
if stats_version['Compressed Objects'] is not None:
res_version.add_line('Compressed objects (' + stats_version['Compressed Objects'][0] + '): ' +
self.list_first_x(stats_version['Compressed Objects'][1]))
if stats_version['Errors'] is not None:
res_version.add_line('Errors (' + stats_version['Errors'][0] + '): ' +
self.list_first_x(stats_version['Errors'][1]))
res_version.add_line('Streams (' + stats_version['Streams'][0] + '): ' +
self.list_first_x(stats_version['Streams'][1]))
if stats_version['Xref Streams'] is not None:
res_version.add_line('Xref streams (' + stats_version['Xref Streams'][0] + '): ' +
self.list_first_x(stats_version['Xref Streams'][1]))
if stats_version['Object Streams'] is not None:
res_version.add_line('Object streams (' + stats_version['Object Streams'][0] + '): ' +
self.list_first_x(stats_version['Object Streams'][1]))
if int(stats_version['Streams'][0]) > 0:
res_version.add_line('Encoded (' + stats_version['Encoded'][0] + '): ' +
self.list_first_x(stats_version['Encoded'][1]))
if stats_version['Decoding Errors'] is not None:
res_version.add_line('Decoding errors (' + stats_version['Decoding Errors'][0] + '): ' +
self.list_first_x(stats_version['Decoding Errors'][1]))
if stats_version['Objects with JS code'] is not None:
res_version.add_line('Objects with JS '
'code (' + stats_version['Objects with JS code'][0] + '): ' +
self.list_first_x(stats_version['Objects with JS code'][1]))
js_stream.extend(stats_version['Objects with JS code'][1])
suspicious_score = SCORE['NULL']
actions = stats_version['Actions']
events = stats_version['Events']
vulns = stats_version['Vulns']
elements = stats_version['Elements']
if events is not None or actions is not None or vulns is not None or elements is not None:
res_suspicious = ResultSection(SCORE['NULL'], 'Suspicious elements', parent=res_version)
if events is not None:
for event in events:
res_suspicious.add_line(event + ': ' + self.list_first_x(events[event]))
suspicious_score += SCORE['LOW']
if actions is not None:
for action in actions:
res_suspicious.add_line(action + ': ' + self.list_first_x(actions[action]))
suspicious_score += SCORE['LOW']
if vulns is not None:
for vuln in vulns:
if vuln in vulnsDict:
temp = [vuln, ' (']
for vulnCVE in vulnsDict[vuln]:
if len(temp) != 2:
temp.append(',')
temp.append(vulnCVE)
cve_found = re.search("CVE-[0-9]{4}-[0-9]{4}", vulnCVE)
if cve_found:
file_res.add_tag(TAG_TYPE['EXPLOIT_NAME'],
vulnCVE[cve_found.start():cve_found.end()],
TAG_WEIGHT['MED'],
usage='IDENTIFICATION')
file_res.add_tag(TAG_TYPE['FILE_SUMMARY'],
vulnCVE[cve_found.start():cve_found.end()],
TAG_WEIGHT['MED'],
usage='IDENTIFICATION')
temp.append('): ')
temp.append(str(vulns[vuln]))
res_suspicious.add_line(temp)
else:
res_suspicious.add_line(vuln + ': ' + str(vulns[vuln]))
suspicious_score += SCORE['HIGH']
if elements is not None:
for element in elements:
if element in vulnsDict:
temp = [element, ' (']
for vulnCVE in vulnsDict[element]:
if len(temp) != 2:
temp.append(',')
temp.append(vulnCVE)
cve_found = re.search("CVE-[0-9]{4}-[0-9]{4}", vulnCVE)
if cve_found:
file_res.add_tag(TAG_TYPE['EXPLOIT_NAME'],
vulnCVE[cve_found.start():cve_found.end()],
TAG_WEIGHT['MED'],
usage='IDENTIFICATION')
file_res.add_tag(TAG_TYPE['FILE_SUMMARY'],
vulnCVE[cve_found.start():cve_found.end()],
TAG_WEIGHT['MED'],
usage='IDENTIFICATION')
temp.append('): ')
temp.append(str(elements[element]))
res_suspicious.add_line(temp)
suspicious_score += SCORE['HIGH']
else:
res_suspicious.add_line('\t\t' + element + ': ' + str(elements[element]))
suspicious_score += SCORE['LOW']
res_suspicious.change_score(suspicious_score)
url_score = SCORE['NULL']
urls = stats_version['URLs']
if urls is not None:
res.add_line("")
res_url = ResultSection(SCORE['NULL'], 'Found URLs', parent=res)
for url in urls:
res_url.add_line('\t\t' + url)
url_score += SCORE['MED']
res_url.change_score(url_score)
for obj in stats_version['Objects'][1]:
cur_obj = pdf_file.getObject(obj, version)
if cur_obj.containsJScode:
cur_res = ResultSection(SCORE['NULL'], 'Object [%s %s] contains %s block of Javascript' %
(obj, version, len(cur_obj.JSCode)))
score_modifier = SCORE['NULL']
js_idx = 0
for js in cur_obj.JSCode:
js_idx += 1
js_score = 0
js_code, unescaped_bytes, _, _ = analyseJS(js)
js_dump += [x for x in js_code if not isPostscript(x)]
# Malicious characteristics
big_buffs = self.get_big_buffs("".join(js_code))
if len(big_buffs) > 0:
js_score += SCORE['VHIGH'] * len(big_buffs)
has_eval, has_unescape = self.check_dangerous_func("".join(js_code))
if has_unescape:
js_score += SCORE['HIGH']
if has_eval:
js_score += SCORE['HIGH']
js_cmt = ""
if has_eval or has_unescape or len(big_buffs) > 0:
score_modifier += js_score
js_cmt = "Suspiciously malicious "
file_res.add_tag(TAG_TYPE['FILE_SUMMARY'], "Suspicious javascript in PDF",
TAG_WEIGHT['MED'], usage='IDENTIFICATION')
file_res.report_heuristic(PeePDF.AL_PeePDF_007)
js_res = ResultSection(0, "%sJavascript Code (block: %s)" % (js_cmt, js_idx),
parent=cur_res)
if js_score > SCORE['NULL']:
temp_js_outname = "object%s-%s_%s.js" % (obj, version, js_idx)
temp_js_path = os.path.join(self.working_directory, temp_js_outname)
temp_js_bin = "".join(js_code).encode("utf-8")
f = open(temp_js_path, "wb")
f.write(temp_js_bin)
f.close()
f_list.append(temp_js_path)
js_res.add_line(["The javascript block was saved as ", temp_js_outname])
if has_eval or has_unescape:
analysis_score = SCORE['NULL']
analysis_res = ResultSection(analysis_score, "[Suspicious Functions]",
parent=js_res)
if has_eval:
analysis_res.add_line("eval: This javascript block uses eval() function"
" which is often used to launch deobfuscated"
" javascript code.")
analysis_score += SCORE['HIGH']
file_res.report_heuristic(PeePDF.AL_PeePDF_003)
if has_unescape:
analysis_res.add_line("unescape: This javascript block uses unescape() "
"function. It may be legitimate but it is definitely"
" suspicious since malware often use this to "
"deobfuscate code blocks.")
analysis_score += SCORE['HIGH']
file_res.report_heuristic(PeePDF.AL_PeePDF_004)
analysis_res.change_score(analysis_score)
buff_idx = 0
for buff in big_buffs:
buff_idx += 1
error, new_buff = unescape(buff)
if error == 0:
buff = new_buff
if buff not in unescaped_bytes:
temp_path_name = None
if ";base64," in buff[:100] and "data:" in buff[:100]:
temp_path_name = "obj%s_unb64_%s.buff" % (obj, buff_idx)
try:
buff = b64decode(buff.split(";base64,")[1].strip())
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(buff)
f.close()
f_list.append(temp_path)
except:
self.log.error("Found 'data:;base64, ' buffer "
"but failed to base64 decode.")
temp_path_name = None
ResultSection(SCORE['VHIGH'],
"A %s bytes buffer was found in the javascript "
"block%s. Here are the first 256 bytes." %
(len(buff), {True: " and was resubmitted as %s" %
temp_path_name,
False: ""}[temp_path_name is not None]),
parent=js_res, body=hexdump(buff[:256]),
body_format=TEXT_FORMAT.MEMORY_DUMP)
file_res.report_heuristic(PeePDF.AL_PeePDF_002)
processed_sc = []
sc_idx = 0
for sc in unescaped_bytes:
if sc not in processed_sc:
sc_idx += 1
processed_sc.append(sc)
try:
sc = sc.decode("hex")
except:
pass
shell_score = SCORE['VHIGH']
temp_path_name = "obj%s_unescaped_%s.buff" % (obj, sc_idx)
shell_res = ResultSection(shell_score,
"Unknown unescaped %s bytes "
"javascript buffer (id: %s) was resubmitted as %s. "
"Here are the first 256 bytes." % (len(sc),
sc_idx,
temp_path_name),
parent=js_res)
shell_res.set_body(hexdump(sc[:256]), TEXT_FORMAT.MEMORY_DUMP)
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(sc)
f.close()
f_list.append(temp_path)
file_res.add_tag(TAG_TYPE['FILE_SUMMARY'], "Unescaped Javascript Buffer",
TAG_WEIGHT['MED'],
usage='IDENTIFICATION')
file_res.report_heuristic(PeePDF.AL_PeePDF_006)
score_modifier += shell_score
if score_modifier > SCORE['NULL']:
res_list.append(cur_res)
elif cur_obj.type == "stream":
if cur_obj.isEncodedStream and cur_obj.filter is not None:
data = cur_obj.decodedStream
encoding = cur_obj.filter.value.replace("[", "").replace("]", "").replace("/",
"").strip()
val = cur_obj.rawValue
otype = cur_obj.elements.get("/Type", None)
sub_type = cur_obj.elements.get("/Subtype", None)
length = cur_obj.elements.get("/Length", None)
else:
data = cur_obj.rawStream
encoding = None
val = cur_obj.rawValue
otype = cur_obj.elements.get("/Type", None)
sub_type = cur_obj.elements.get("/Subtype", None)
length = cur_obj.elements.get("/Length", None)
if otype:
otype = otype.value.replace("/", "").lower()
if sub_type:
sub_type = sub_type.value.replace("/", "").lower()
if length:
length = length.value
if otype == "embeddedfile":
if len(data) > 4096:
# TODO: we might have to be smarter here.
cur_res = ResultSection(SCORE['NULL'], 'Embedded file found (%s bytes) [obj: %s %s]'
' and dumped for analysis %s%s%s' %
(length, obj, version, {True: "(Type: %s) " % otype,
False: ""}[otype is not None],
{True: "(SubType: %s) " % sub_type,
False: ""}[sub_type is not None],
{True: "(Encoded with %s)" % encoding,
False: ""}[encoding is not None]))
temp_path_name = "EmbeddedFile_%s%s.obj" % (obj, {True: "_%s" % encoding,
False: ""}[encoding is not None])
temp_path = os.path.join(self.working_directory, temp_path_name)
f = open(temp_path, "wb")
f.write(data)
f.close()
f_list.append(temp_path)
cur_res.add_line(["The EmbeddedFile object was saved as ", temp_path_name])
res_list.append(cur_res)
elif otype not in BANNED_TYPES:
cur_res = ResultSection(SCORE['NULL'], 'Unknown stream found [obj: %s %s] %s%s%s' %
(obj, version, {True: "(Type: %s) " % otype,
False: ""}[otype is not None],
{True: "(SubType: %s) " % sub_type,
False: ""}[sub_type is not None],
{True: "(Encoded with %s)" % encoding,
False: ""}[encoding is not None]))
for line in val.splitlines():
cur_res.add_line(line)
emb_res = ResultSection(SCORE.NULL, 'First 256 bytes', parent=cur_res)
emb_res.set_body(hexdump(data[:256]), TEXT_FORMAT.MEMORY_DUMP)
res_list.append(cur_res)
else:
pass
file_res.add_section(res)
for results in res_list:
file_res.add_section(results)
if js_dump:
js_dump_res = ResultSection(SCORE['NULL'], 'Full Javascript dump')
temp_js_dump = "javascript_dump.js"
temp_js_dump_path = os.path.join(self.working_directory, temp_js_dump)
try:
temp_js_dump_bin = "\n\n----\n\n".join(js_dump).encode("utf-8")
except UnicodeDecodeError:
temp_js_dump_bin = "\n\n----\n\n".join(js_dump)
temp_js_dump_sha1 = hashlib.sha1(temp_js_dump_bin).hexdigest()
f = open(temp_js_dump_path, "wb")
f.write(temp_js_dump_bin)
f.flush()
f.close()
f_list.append(temp_js_dump_path)
js_dump_res.add_line(["The javascript dump was saved as ", temp_js_dump])
js_dump_res.add_line(["The sha1 for the javascript dump is ", temp_js_dump_sha1])
file_res.add_tag(TAG_TYPE['PDF_JAVASCRIPT_SHA1'], temp_js_dump_sha1, TAG_WEIGHT['HIGH'],
usage='CORRELATION')
file_res.add_section(js_dump_res)
for filename in f_list:
request.add_extracted(filename, "Dumped from %s" % os.path.basename(temp_filename))
else:
res = ResultSection(SCORE['INFO'], "ERROR: Could not parse file with peepdf.")
file_res.add_section(res)
finally:
try:
del pdf_file
except:
pass
try:
del pdf_parser
except:
pass
gc.collect()
def execute(self, request):
# ==================================================================
# Execute a request:
# Every time your service receives a new file to scan, the execute function is called
# This is where you should execute your processing code.
# For the purpose of this example, we will only generate results ...
# You should run your code here...
# ==================================================================
# Check if we're scanning an embedded file
# This service always drop 3 embedded file which two generates random results and the other empty results
# We're making a check to see if we're scanning the embedded file.
# In a normal service this is not something you would do at all but since we are using this
# service in our unit test to test all features of our report generator, we have to do this
if request.sha256 not in ['d729ecfb2cf40bc4af8038dac609a57f57dbe6515d35357af973677d5e66417a',
'5ce5ae8ef56a54af2c44415800a81ecffd49a33ae8895dfe38fc1075d3f619ec',
'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06']:
# Main file results...
# ==================================================================
# Write the results:
# First, create a result object where all the result sections will be saved to
result = Result()
# ==================================================================
# Standard text section: BODY_FORMAT.TEXT - DEFAULT
# Text sections basically just dumps the text to the screen...
# All sections scores will be SUMed in the service result
# The Result classification will be the highest classification found in the sections
text_section = ResultSection('Example of a default section')
# You can add lines to your section one at a time
# Here we will generate a random line
text_section.add_line(get_random_phrase())
# Or your can add them from a list
# Here we will generate random amount of random lines
text_section.add_lines([get_random_phrase() for _ in range(random.randint(1, 5))])
# If the section needs to affect the score of the file you need to set a heuristics
# Here we will pick one at random
# In addition to add a heuristic, we will associated a signature with the heuristic,
# we're doing this by adding the signature name to the heuristic. (Here we generating a random name)
text_section.set_heuristic(3, signature="sig_one")
# You can attach attack ids to heuristics after they where defined
text_section.heuristic.add_attack_id("T1066")
# Same thing for the signatures, they can be added to heuristic after the fact and you can even say how
# many time the signature fired by setting its frequency. If you call add_signature_id twice with the
# same signature, this will effectively increase the frequency of the signature.
text_section.heuristic.add_signature_id("sig_two", score=20, frequency=2)
text_section.heuristic.add_signature_id("sig_two", score=20, frequency=3)
text_section.heuristic.add_signature_id("sig_three")
text_section.heuristic.add_signature_id("sig_three")
text_section.heuristic.add_signature_id("sig_four", score=0)
# The heuristic for text_section should have the following properties
# 1. 1 attack ID: T1066
# 2. 4 signatures: sig_one, sig_two, sig_three and sig_four
# 3. Signature frequencies are cumulative therefor they will be as follow:
# - sig_one = 1
# - sig_two = 5
# - sig_three = 2
# - sig_four = 1
# 4. The score used by each heuristic is driven by the following rules: signature_score_map is higher
# priority, then score value for the add_signature_id is in second place and finally the default
# heuristic score is use. Therefor the score used to calculate the total score for the text_section is
# as follow:
# - sig_one: 10 -> heuristic default score
# - sig_two: 20 -> score provided by the function add_signature_id
# - sig_three: 30 -> score provided by the heuristic map
# - sig_four: 40 -> score provided by the heuristic map because it's higher priority than the
# function score
# 5. Total section score is then: 1x10 + 5x20 + 2x30 + 1x40 = 210
# Make sure you add your section to the result
result.add_section(text_section)
# ==================================================================
# Color map Section: BODY_FORMAT.GRAPH_DATA
# Creates a color map bar using a minimum and maximum domain
# e.g. We are using this section to display the entropy distribution in some services
cmap_min = 0
cmap_max = 20
color_map_data = {
'type': 'colormap',
'data': {
'domain': [cmap_min, cmap_max],
'values': [random.random() * cmap_max for _ in range(50)]
}
}
# The classification of a section can be set to any valid classification for your system
section_color_map = ResultSection("Example of colormap result section", body_format=BODY_FORMAT.GRAPH_DATA,
body=json.dumps(color_map_data), classification=cl_engine.RESTRICTED)
result.add_section(section_color_map)
# ==================================================================
# URL section: BODY_FORMAT.URL
# Generate a list of clickable urls using a json encoded format
# As you can see here, the body of the section can be set directly instead of line by line
random_host = get_random_host()
url_section = ResultSection('Example of a simple url section', body_format=BODY_FORMAT.URL,
body=json.dumps({"name": "Random url!", "url": f"https://{random_host}/"}))
# Since urls are very important features we can tag those features in the system so they are easy to find
# Tags are defined by a type and a value
url_section.add_tag("network.static.domain", random_host)
# You may also want to provide a list of url!
# Also, No need to provide a name, the url link will be displayed
host1 = get_random_host()
host2 = get_random_host()
ip1 = get_random_ip()
ip2 = get_random_ip()
ip3 = get_random_ip()
urls = [
{"url": f"https://{host1}/"},
{"url": f"https://{host2}/"},
{"url": f"https://{ip1}/"},
{"url": f"https://{ip2}/"},
{"url": f"https://{ip3}/"}]
# A heuristic can fire more then once without being associated to a signature
url_heuristic = Heuristic(4, frequency=len(urls))
url_sub_section = ResultSection('Example of a url section with multiple links',
body=json.dumps(urls), body_format=BODY_FORMAT.URL,
heuristic=url_heuristic)
url_sub_section.add_tag("network.static.ip", ip1)
url_sub_section.add_tag("network.static.ip", ip2)
url_sub_section.add_tag("network.static.ip", ip3)
url_sub_section.add_tag("network.static.domain", host1)
url_sub_section.add_tag("network.dynamic.domain", host2)
# Since url_sub_section is a sub-section of url_section
# we will add it as a sub-section of url_section not to the main result itself
url_section.add_subsection(url_sub_section)
result.add_section(url_section)
# ==================================================================
# Memory dump section: BODY_FORMAT.MEMORY_DUMP
# Dump whatever string content you have into a <pre/> html tag so you can do your own formatting
data = hexdump(b"This is some random text that we will format as an hexdump and you'll see "
b"that the hexdump formatting will be preserved by the memory dump section!")
memdump_section = ResultSection('Example of a memory dump section', body_format=BODY_FORMAT.MEMORY_DUMP,
body=data)
memdump_section.set_heuristic(random.randint(1, 4))
result.add_section(memdump_section)
# ==================================================================
# KEY_VALUE section:
# This section allows the service writer to list a bunch of key/value pairs to be displayed in the UI
# while also providing easy to parse data for auto mated tools.
# NB: You should definitely use this over a JSON body type since this one will be displayed correctly
# in the UI for the user
# The body argument must be a json dumps of a dictionary (only str, int, and booleans are allowed)
kv_body = {
"a_str": "Some string",
"a_bool": False,
"an_int": 102,
}
kv_section = ResultSection('Example of a KEY_VALUE section', body_format=BODY_FORMAT.KEY_VALUE,
body=json.dumps(kv_body))
result.add_section(kv_section)
# ==================================================================
# JSON section:
# Re-use the JSON editor we use for administration (https://github.com/josdejong/jsoneditor)
# to display a tree view of JSON results.
# NB: Use this sparingly! As a service developer you should do your best to include important
# results as their own result sections.
# The body argument must be a json dump of a python dictionary
json_body = {
"a_str": "Some string",
"a_list": ["a", "b", "c"],
"a_bool": False,
"an_int": 102,
"a_dict": {
"list_of_dict": [
{"d1_key": "val", "d1_key2": "val2"},
{"d2_key": "val", "d2_key2": "val2"}
],
"bool": True
}
}
json_section = ResultSection('Example of a JSON section', body_format=BODY_FORMAT.JSON,
body=json.dumps(json_body))
result.add_section(json_section)
# ==================================================================
# PROCESS_TREE section:
# This section allows the service writer to list a bunch of dictionary objects that have nested lists
# of dictionaries to be displayed in the UI. Each dictionary object represents a process, and therefore
# each dictionary must have be of the following format:
# {
# "process_pid": int,
# "process_name": str,
# "command_line": str,
# "children": [] NB: This list either is empty or contains more dictionaries that have the same
# structure
# }
nc_body = [
{
"process_pid": 123,
"process_name": "evil.exe",
"command_line": "C:\\evil.exe",
"signatures": {},
"children": [
{
"process_pid": 321,
"process_name": "takeovercomputer.exe",
"command_line": "C:\\Temp\\takeovercomputer.exe -f do_bad_stuff",
"signatures": {"one":250},
"children": [
{
"process_pid": 456,
"process_name": "evenworsethanbefore.exe",
"command_line": "C:\\Temp\\evenworsethanbefore.exe -f change_reg_key_cuz_im_bad",
"signatures": {"one":10, "two":10, "three":10},
"children": []
},
{
"process_pid": 234,
"process_name": "badfile.exe",
"command_line": "C:\\badfile.exe -k nothing_to_see_here",
"signatures": {"one":1000, "two":10, "three":10, "four":10, "five":10},
"children": []
}
]
},
{
"process_pid": 345,
"process_name": "benignexe.exe",
"command_line": "C:\\benignexe.exe -f \"just kidding, i'm evil\"",
"signatures": {"one": 2000},
"children": []
}
]
},
{
"process_pid": 987,
"process_name": "runzeroday.exe",
"command_line": "C:\\runzeroday.exe -f insert_bad_spelling",
"signatures": {},
"children": []
}
]
nc_section = ResultSection('Example of a PROCESS_TREE section',
body_format=BODY_FORMAT.PROCESS_TREE,
body=json.dumps(nc_body))
result.add_section(nc_section)
# ==================================================================
# TABLE section:
# This section allows the service writer to have their content displayed in a table format in the UI
# The body argument must be a list [] of dict {} objects. A dict object can have a key value pair
# where the value is a flat nested dictionary, and this nested dictionary will be displayed as a nested
# table within a cell.
table_body = [
{
"a_str": "Some string1",
"extra_column_here": "confirmed",
"a_bool": False,
"an_int": 101,
},
{
"a_str": "Some string2",
"a_bool": True,
"an_int": 102,
},
{
"a_str": "Some string3",
"a_bool": False,
"an_int": 103,
},
{
"a_str": "Some string4",
"a_bool": None,
"an_int": -1000000000000000000,
"extra_column_there": "confirmed",
"nested_table": {
"a_str": "Some string3",
"a_bool": False,
"nested_table_thats_too_deep": {
"a_str": "Some string3",
"a_bool": False,
"an_int": 103,
},
},
},
]
table_section = ResultSection('Example of a TABLE section',
body_format=BODY_FORMAT.TABLE,
body=json.dumps(table_body))
result.add_section(table_section)
# ==================================================================
# Re-Submitting files to the system
# Adding extracted files will have them resubmitted to the system for analysis
# This file will generate random results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(data.encode())
request.add_extracted(temp_path, "file.txt", "Extracted by some magic!")
# Embedded files can also have their own classification!
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(b"CLASSIFIED!!!__"+data.encode())
request.add_extracted(temp_path, "classified.doc", "Classified file ... don't look",
classification=cl_engine.RESTRICTED)
# This file will generate empty results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(b"EMPTY")
request.add_extracted(temp_path, "empty.txt", "Extracted empty resulting file")
# ==================================================================
# Supplementary files
# Adding supplementary files will save them on the datastore for future
# reference but wont reprocess those files.
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(json.dumps(urls))
request.add_supplementary(temp_path, "urls.json", "These are urls as a JSON file")
# like embedded files, you can add more then one supplementary files
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(json.dumps(json_body))
request.add_supplementary(temp_path, "json_body.json", "This is the json_body as a JSON file")
# ==================================================================
# Wrap-up:
# Save your result object back into the request
request.result = result
# ==================================================================
# Empty results file
elif request.sha256 == 'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06':
# Creating and empty result object
request.result = Result()
# ==================================================================
# Randomized results file
else:
# For the randomized results file, we will completely randomize the results
# The content of those results do not matter since we've already showed you
# all the different result sections, tagging, heuristics and file upload functions
embedded_result = Result()
# random number of sections
for _ in range(1, 3):
embedded_result.add_section(self._create_random_section())
request.result = embedded_result
def execute(self, request):
# Create a result object where all the sections will be stored
result = Result()
# ==================================================================
# Default Section:
# Default section basically just dumps the text to the screen...
# All sections scores will be SUMed in the service result
# The Result classification will be the highest classification found in the sections
default_section = ResultSection(SCORE.LOW,
'Example of a default section',
Classification.RESTRICTED)
default_section.add_line("You can add line by line!")
default_section.add_lines(["Or", "Multiple lines", "Inside a list!"])
# ==================================================================
# Color map Section:
# Creates a color map bar using a minimum and maximum domain
cmap_min = 0
cmap_max = 20
color_map_data = {
'type': 'colormap',
'data': {
'domain': [cmap_min, cmap_max],
'values': [random.random() * cmap_max for _ in xrange(50)]
}
}
section_color_map = ResultSection(SCORE.NULL,
"Example of colormap result section",
self.SERVICE_CLASSIFICATION,
body_format=TEXT_FORMAT.GRAPH_DATA,
body=json.dumps(color_map_data))
# ==================================================================
# URL section:
# Generate a list of clickable urls using a json encoded format
url_section = ResultSection(SCORE.NULL,
'Example of a simple url section',
self.SERVICE_CLASSIFICATION,
body_format=TEXT_FORMAT.URL,
body=json.dumps({
"name":
"Google",
"url":
"https://www.google.com/"
}))
# You can add tags to any section although those tag will be brought up to the result object
# Tags are defined by a type, value and weight (confidence lvl)
# you can also add a classification and context if needed
url_section.add_tag(TAG_TYPE.NET_DOMAIN_NAME, "google.com",
TAG_WEIGHT.LOW)
url_section.add_tag(TAG_TYPE.NET_DOMAIN_NAME,
"bob.com",
TAG_WEIGHT.LOW,
classification=Classification.RESTRICTED)
url_section.add_tag(TAG_TYPE.NET_DOMAIN_NAME,
"baddomain.com",
TAG_WEIGHT.LOW,
context=Context.BEACONS)
# You may also want to provide a list of url! Also, No need to provide a name, the url link will be displayed
urls = [{
"url": "https://google.com/"
}, {
"url": "https://google.ca/"
}, {
"url": "https://microsoft.com/"
}]
url_section2 = ResultSection(
SCORE.MED,
'Example of a url section with multiple links',
self.SERVICE_CLASSIFICATION,
body_format=TEXT_FORMAT.URL,
body=json.dumps(urls))
# Add url_section2 as a subsection of url section
# The score of the subsections will automatically be added to the parent section
url_section.add_section(url_section2)
# ==================================================================
# Memory dump section:
# Dump whatever string content you have into a <pre/> html tag so you can do your own formatting
data = hexdump(
"This is some random text that we will format as an hexdump and you'll see "
"that the hexdump formatting will be preserved by the memory dump section!"
)
memdump_section = ResultSection(SCORE.NULL,
'Example of a memory dump section',
self.SERVICE_CLASSIFICATION,
body_format=TEXT_FORMAT.MEMORY_DUMP,
body=data)
# ==================================================================
# Re-Submitting files to the system
# Adding extracted files will have them resubmitted to the system for analysis
if request.srl != '8cf8277a71e85122bf7ea4610c7cfcc0bfb6dee799be50a41b2f4b1321b3317f':
# This IF just prevents resubmitting the same file in a loop for this exemple...
temp_path = tempfile.mktemp(dir=self.working_directory)
with open(temp_path, "w") as myfile:
myfile.write(data)
request.add_extracted(temp_path,
"Extracted by some random magic!",
display_name="file.txt")
# ==================================================================
# Supplementary files
# Adding supplementary files will save them on the datastore for future
# reference but wont reprocess those files.
temp_path = tempfile.mktemp(dir=self.working_directory)
with open(temp_path, "w") as myfile:
myfile.write(json.dumps(urls))
request.add_supplementary(temp_path,
"These are urls as a JSON",
display_name="urls.json")
# ==================================================================
# Wrap-up:
# Add all sections to the Result object
result.add_section(default_section)
result.add_section(section_color_map)
result.add_section(url_section)
result.add_section(memdump_section)
request.result = result
def execute(self, request):
# ==================================================================
# Execute a request:
# Every time your service receives a new file to scan, the execute function is called
# This is where you should execute your processing code.
# For the purpose of this example, we will only generate results ...
# You should run your code here...
# ==================================================================
# Check if we're scanning an embedded file
# This service always drop 3 embedded file which two generates random results and the other empty results
# We're making a check to see if we're scanning the embedded file.
# In a normal service this is not something you would do at all but since we are using this
# service in our unit test to test all features of our report generator, we have to do this
if request.sha256 not in [
'd729ecfb2cf40bc4af8038dac609a57f57dbe6515d35357af973677d5e66417a',
'5ce5ae8ef56a54af2c44415800a81ecffd49a33ae8895dfe38fc1075d3f619ec',
'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06'
]:
# Main file results...
# ==================================================================
# Write the results:
# First, create a result object where all the result sections will be saved to
result = Result()
# ==================================================================
# Standard text section: BODY_FORMAT.TEXT - DEFAULT
# Text sections basically just dumps the text to the screen...
# All sections scores will be SUMed in the service result
# The Result classification will be the highest classification found in the sections
text_section = ResultTextSection('Example of a default section')
# You can add lines to your section one at a time
# Here we will generate a random line
text_section.add_line(get_random_phrase())
# Or your can add them from a list
# Here we will generate random amount of random lines
text_section.add_lines(
[get_random_phrase() for _ in range(random.randint(1, 5))])
# You can tag data to a section, tagging is used to to quickly find defining information about a file
text_section.add_tag("attribution.implant", "ResultSample")
# If the section needs to affect the score of the file you need to set a heuristics
# Here we will pick one at random
# In addition to add a heuristic, we will associated a signature with the heuristic,
# we're doing this by adding the signature name to the heuristic. (Here we generating a random name)
text_section.set_heuristic(3, signature="sig_one")
# You can attach attack ids to heuristics after they where defined
text_section.heuristic.add_attack_id(
random.choice(list(software_map.keys())))
text_section.heuristic.add_attack_id(
random.choice(list(attack_map.keys())))
text_section.heuristic.add_attack_id(
random.choice(list(group_map.keys())))
text_section.heuristic.add_attack_id(
random.choice(list(revoke_map.keys())))
# Same thing for the signatures, they can be added to heuristic after the fact and you can even say how
# many time the signature fired by setting its frequency. If you call add_signature_id twice with the
# same signature, this will effectively increase the frequency of the signature.
text_section.heuristic.add_signature_id("sig_two",
score=20,
frequency=2)
text_section.heuristic.add_signature_id("sig_two",
score=20,
frequency=3)
text_section.heuristic.add_signature_id("sig_three")
text_section.heuristic.add_signature_id("sig_three")
text_section.heuristic.add_signature_id("sig_four", score=0)
# The heuristic for text_section should have the following properties
# 1. 1 attack ID: T1066
# 2. 4 signatures: sig_one, sig_two, sig_three and sig_four
# 3. Signature frequencies are cumulative therefor they will be as follow:
# - sig_one = 1
# - sig_two = 5
# - sig_three = 2
# - sig_four = 1
# 4. The score used by each heuristic is driven by the following rules: signature_score_map is higher
# priority, then score value for the add_signature_id is in second place and finally the default
# heuristic score is use. Therefor the score used to calculate the total score for the text_section is
# as follow:
# - sig_one: 10 -> heuristic default score
# - sig_two: 20 -> score provided by the function add_signature_id
# - sig_three: 30 -> score provided by the heuristic map
# - sig_four: 40 -> score provided by the heuristic map because it's higher priority than the
# function score
# 5. Total section score is then: 1x10 + 5x20 + 2x30 + 1x40 = 210
# Make sure you add your section to the result
result.add_section(text_section)
# Even if the section was added to the results you can still modify it by adding a subsection for example
ResultSection(
"Example of sub-section without a body added later in processing",
parent=text_section)
# ==================================================================
# Color map Section: BODY_FORMAT.GRAPH_DATA
# Creates a color map bar using a minimum and maximum domain
# e.g. We are using this section to display the entropy distribution in some services
cmap_min = 0
cmap_max = 20
cmap_values = [random.random() * cmap_max for _ in range(50)]
# The classification of a section can be set to any valid classification for your system
section_color_map = ResultGraphSection(
"Example of colormap result section",
classification=cl_engine.RESTRICTED)
section_color_map.set_colormap(cmap_min, cmap_max, cmap_values)
result.add_section(section_color_map)
# ==================================================================
# URL section: BODY_FORMAT.URL
# Generate a list of clickable urls using a json encoded format
# As you can see here, the body of the section can be set directly instead of line by line
random_host = get_random_host()
url_section = ResultURLSection('Example of a simple url section')
url_section.add_url(f"https://{random_host}/", name="Random url!")
# Since urls are very important features we can tag those features in the system so they are easy to find
# Tags are defined by a type and a value
url_section.add_tag("network.static.domain", random_host)
# You may also want to provide a list of url!
# Also, No need to provide a name, the url link will be displayed
hosts = [get_random_host() for _ in range(2)]
# A heuristic can fire more then once without being associated to a signature
url_heuristic = Heuristic(4, frequency=len(hosts))
url_sub_section = ResultURLSection(
'Example of a url sub-section with multiple links',
heuristic=url_heuristic,
classification=cl_engine.RESTRICTED)
for host in hosts:
url_sub_section.add_url(f"https://{host}/")
url_sub_section.add_tag("network.static.domain", host)
# You can keep nesting sections if you really need to
ips = [get_random_ip() for _ in range(3)]
url_sub_sub_section = ResultURLSection(
'Exemple of a two level deep sub-section')
for ip in ips:
url_sub_sub_section.add_url(f"https://{ip}/")
url_sub_sub_section.add_tag("network.static.ip", ip)
# Since url_sub_sub_section is a sub-section of url_sub_section
# we will add it as a sub-section of url_sub_section not to the main result itself
url_sub_section.add_subsection(url_sub_sub_section)
# Invalid sections will be ignored, and an error will apear in the logs
# Sub-sections of invalid sections will be ignored too
invalid_section = ResultSection("")
ResultSection(
"I won't make it to the report because my parent is invalid :(",
parent=invalid_section)
url_sub_section.add_subsection(invalid_section)
# Since url_sub_section is a sub-section of url_section
# we will add it as a sub-section of url_section not to the main result itself
url_section.add_subsection(url_sub_section)
result.add_section(url_section)
# ==================================================================
# Memory dump section: BODY_FORMAT.MEMORY_DUMP
# Dump whatever string content you have into a <pre/> html tag so you can do your own formatting
data = hexdump(
b"This is some random text that we will format as an hexdump and you'll see "
b"that the hexdump formatting will be preserved by the memory dump section!"
)
memdump_section = ResultMemoryDumpSection(
'Example of a memory dump section', body=data)
memdump_section.set_heuristic(random.randint(1, 4))
result.add_section(memdump_section)
# ==================================================================
# KEY_VALUE section:
# This section allows the service writer to list a bunch of key/value pairs to be displayed in the UI
# while also providing easy to parse data for auto mated tools.
# NB: You should definitely use this over a JSON body type since this one will be displayed correctly
# in the UI for the user
# The body argument must be a dictionary (only str, int, and booleans are allowed)
kv_section = ResultKeyValueSection(
'Example of a KEY_VALUE section')
# You can add items individually
kv_section.set_item('key', "value")
# Or simply add them in bulk
kv_section.update_items({
"a_str": "Some string",
"a_bool": False,
"an_int": 102,
})
result.add_section(kv_section)
# ==================================================================
# ORDERED_KEY_VALUE section:
# This section provides the same functionality as the KEY_VALUE section except the order of the fields
# are garanteed to be preserved in the order in which the fields are added to the section. Also with
# this section, you can repeat the same key name multiple times
oredered_kv_section = ResultOrderedKeyValueSection(
'Example of an ORDERED_KEY_VALUE section')
# You can add items individually
for x in range(random.randint(3, 6)):
oredered_kv_section.add_item(f'key{x}', f"value{x}")
result.add_section(oredered_kv_section)
# ==================================================================
# JSON section:
# Re-use the JSON editor we use for administration (https://github.com/josdejong/jsoneditor)
# to display a tree view of JSON results.
# NB: Use this sparingly! As a service developer you should do your best to include important
# results as their own result sections.
# The body argument must be a python dictionary
json_body = {
"a_str": "Some string",
"a_list": ["a", "b", "c"],
"a_bool": False,
"an_int": 102,
"a_dict": {
"list_of_dict": [{
"d1_key": "val",
"d1_key2": "val2"
}, {
"d2_key": "val",
"d2_key2": "val2"
}],
"bool":
True
}
}
json_section = ResultJSONSection('Example of a JSON section')
# You can set the json result to a specific value
json_section.set_json(json_body)
# You can also update specific parts after the fact
json_section.update_json({
'an_int': 1000,
'updated_key': 'updated_value'
})
result.add_section(json_section)
# ==================================================================
# PROCESS_TREE section:
# This section allows the service writer to list a bunch of dictionary objects that have nested lists
# of dictionaries to be displayed in the UI. Each dictionary object represents a process, and therefore
# each dictionary must have be of the following format:
# {
# "process_pid": int,
# "process_name": str,
# "command_line": str,
# "signatures": {} This dict has the signature name as a key and the score as it's value
# "children": [] NB: This list either is empty or contains more dictionaries that have the same
# structure
# }
process_tree_section = ResultProcessTreeSection(
'Example of a PROCESS_TREE section')
# You can use the ProcessItem class to create the processes to add to the result section
evil_process = ProcessItem(123, "evil.exe", "c:\\evil.exe")
evil_process_child_1 = ProcessItem(
321, "takeovercomputer.exe",
"C:\\Temp\\takeovercomputer.exe -f do_bad_stuff")
# You can add child processes to the ProcessItem objects
evil_process_child_1.add_child_process(
ProcessItem(
456,
"evenworsethanbefore.exe",
"C:\\Temp\\evenworsethanbefore.exe -f change_reg_key_cuz_im_bad",
signatures={
"one": 10,
"two": 10,
"three": 10
}))
evil_process_child_1.add_child_process(
ProcessItem(234,
"badfile.exe",
"C:\\badfile.exe -k nothing_to_see_here",
signatures={
"one": 1000,
"two": 10,
"three": 10,
"four": 10,
"five": 10
}))
# You can add signatures that hit on a ProcessItem Object
evil_process_child_1.add_signature('one', 250)
# Or even directly create the ProcessItem object with the signature in it
evil_process_child_2 = ProcessItem(
345,
"benignexe.exe",
"C:\\benignexe.exe -f \"just kidding, i'm evil\"",
signatures={"one": 2000})
# You can also add counts for network, file and registry events to a ProcessItem object
evil_process_child_2.add_network_events(4)
evil_process_child_2.add_file_events(7000)
evil_process_child_2.add_registry_events(10)
# You can also indicate if the process tree item has been safelisted
benign_process = ProcessItem(678, "trustme.exe", "C:\\trustme.exe")
benign_process.safelist()
evil_process.add_child_process(evil_process_child_1)
evil_process.add_child_process(evil_process_child_2)
# Add your processes to the result section via the add_process function
process_tree_section.add_process(evil_process)
process_tree_section.add_process(
ProcessItem(987, "runzeroday.exe",
"C:\\runzeroday.exe -f insert_bad_spelling"))
process_tree_section.add_process(benign_process)
result.add_section(process_tree_section)
# ==================================================================
# TABLE section:
# This section allows the service writer to have their content displayed in a table format in the UI
# The body argument must be a list [] of dict {} objects. A dict object can have a key value pair
# where the value is a flat nested dictionary, and this nested dictionary will be displayed as a nested
# table within a cell.
table_section = ResultTableSection('Example of a TABLE section')
# Use the TableRow class to help adding row to the Table section
table_section.add_row(
TableRow(a_str="Some string1",
extra_column_here="confirmed",
a_bool=False,
an_int=101))
table_section.add_row(
TableRow(
{
"a_str": "Some string2",
"a_bool": True,
"an_int": "to_be_overriden_by_kwargs"
},
an_int=102))
table_section.add_row(
TableRow(a_str="Some string3", a_bool=False, an_int=103))
# Valid values for the items in the TableRow are: str, int, bool, None, or dict of those values
table_section.add_row(
TableRow(
{
"a_str": "Some string4",
"a_bool": None,
"an_int": -1000000000000000000
}, {
"extra_column_there": "confirmed",
"nested_key_value_pair": {
"a_str": "Some string3",
"a_bool": False,
"nested_kv_thats_too_deep": {
"a_str": "Some string3",
"a_bool": False,
"an_int": 103,
},
}
}))
result.add_section(table_section)
# ==================================================================
# Re-Submitting files to the system
# Adding extracted files will have them resubmitted to the system for analysis
# This file will generate random results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(data.encode())
request.add_extracted(temp_path, "file.txt",
"Extracted by some magic!")
# Embedded files can also have their own classification!
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(b"CLASSIFIED!!!__" + data.encode())
request.add_extracted(temp_path,
"classified.doc",
"Classified file ... don't look",
classification=cl_engine.RESTRICTED)
# This file will generate empty results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(b"EMPTY")
request.add_extracted(temp_path, "empty.txt",
"Extracted empty resulting file")
# ==================================================================
# Supplementary files
# Adding supplementary files will save them on the datastore for future
# reference but wont reprocess those files.
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(url_sub_section.body)
request.add_supplementary(temp_path, "urls.json",
"These are urls as a JSON file")
# like embedded files, you can add more then one supplementary files
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(json.dumps(json_body))
request.add_supplementary(temp_path, "json_body.json",
"This is the json_body as a JSON file")
# ==================================================================
# Zeroize on safe tags
# When this feature is turned on, the section will get its score set to zero if all its tags
# were safelisted by the safelisting engine
zero_section = ResultSection('Example of zeroize-able section',
zeroize_on_tag_safe=True)
zero_section.set_heuristic(2)
zero_section.add_line(
"This section will have a zero score if all tags are safelisted."
)
zero_section.add_tag('network.static.ip', '127.0.0.1')
result.add_section(zero_section)
# ==================================================================
# Auto-collapse
# When this feature is turned on, the section will be collapsed when first displayed
collapse_section = ResultSection(
'Example of auto-collapse section', auto_collapse=True)
collapse_section.set_heuristic(2)
collapse_section.add_line(
"This section was collapsed when first loaded in the UI")
result.add_section(collapse_section)
# ==================================================================
# Image Section
# This type of section allows the service writer to display images to the user
image_section = ResultImageSection(request,
'Example of Image section')
for x in range(6):
image_section.add_image(f'data/000{x+1}.jpg',
f'000{x+1}.jpg',
f'ResultSample screenshot 000{x+1}',
ocr_heuristic_id=6)
result.add_section(image_section)
# ==================================================================
# Multi Section
# This type of section allows the service writer to display multiple section types
# in the same result section. Here's a concrete exemple of this:
multi_section = ResultMultiSection(
'Example of Multi-typed section')
multi_section.add_section_part(
TextSectionBody(
body="We have detected very high entropy multiple sections "
"of your file, this section is most-likely packed or "
"encrypted.\n\nHere are affected sections:"))
section_count = random.randint(1, 4)
for x in range(section_count):
multi_section.add_section_part(
KVSectionBody(section_name=f".UPX{x}",
offset=f'0x00{8+x}000',
size='4196 bytes'))
graph_part = GraphSectionBody()
graph_part.set_colormap(
0, 8, [7 + random.random() for _ in range(20)])
multi_section.add_section_part(graph_part)
if x != section_count - 1:
multi_section.add_section_part(DividerSectionBody())
multi_section.add_tag("file.pe.sections.name", f".UPX{x}")
multi_section.set_heuristic(5)
result.add_section(multi_section)
# ==================================================================
# Propagate temporary submission data to other services
# Sometimes two service can work in tandem were one extra some piece of information the other
# one uses to do it's work. This is how a service can set temporary data that other
# services that subscribe to can use.
request.temp_submission_data['kv_section'] = kv_section.body
request.temp_submission_data[
'process_tree_section'] = process_tree_section.body
request.temp_submission_data['url_section'] = url_sub_section.body
# ==================================================================
# Wrap-up:
# Save your result object back into the request
request.result = result
# ==================================================================
# Empty results file
elif request.sha256 == 'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06':
# Creating and empty result object
request.result = Result()
# ==================================================================
# Randomized results file
else:
# For the randomized results file, we will completely randomize the results
# The content of those results do not matter since we've already showed you
# all the different result sections, tagging, heuristics and file upload functions
embedded_result = Result()
# random number of sections
for _ in range(1, 3):
embedded_result.add_section(self._create_random_section())
request.result = embedded_result
def execute(self, request):
# ==================================================================
# Execute a request:
# Every time your service receives a new file to scan, the execute function is called
# This is where you should execute your processing code.
# For the purpose of this example, we will only generate results ...
# You should run your code here...
# ==================================================================
# Check if we're scanning an embedded file
# This service always drop two embedded file which one generates random results and the other empty results
# We're making a check to see if we're scanning the embedded file.
# In a normal service this is not something you would do at all but since we are using this
# service in our unit test to test all features of our report generator, we have to do this
if request.sha256 not in [
'd729ecfb2cf40bc4af8038dac609a57f57dbe6515d35357af973677d5e66417a',
'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06'
]:
# Main file results...
# ==================================================================
# Write the results:
# First, create a result object where all the result sections will be saved to
result = Result()
# ==================================================================
# Standard text section: BODY_FORMAT.TEXT - DEFAULT
# Text sections basically just dumps the text to the screen...
# All sections scores will be SUMed in the service result
# The Result classification will be the highest classification found in the sections
text_section = ResultSection('Example of a default section')
# You can add lines to your section one at a time
# Here we will generate a random line
text_section.add_line(get_random_phrase())
# Or your can add them from a list
# Here we will generate random amount of random lines
text_section.add_lines(
[get_random_phrase() for _ in range(random.randint(1, 5))])
# If the section needs to affect the score of the file you need to set a heuristics
# Here we will pick one at random
# In addition to add a heuristic, we will associated a signature with the heuristic,
# we're doing this by adding the signature name to the heuristic. (Here we generating a random name)
text_section.set_heuristic(random.randint(1, 4),
signature=get_random_phrase(
1, 4).lower().replace(" ", "_"))
# Make sure you add your section to the result
result.add_section(text_section)
# ==================================================================
# Color map Section: BODY_FORMAT.GRAPH_DATA
# Creates a color map bar using a minimum and maximum domain
# e.g. We are using this section to display the entropy distribution in some services
cmap_min = 0
cmap_max = 20
color_map_data = {
'type': 'colormap',
'data': {
'domain': [cmap_min, cmap_max],
'values': [random.random() * cmap_max for _ in range(50)]
}
}
section_color_map = ResultSection(
"Example of colormap result section",
body_format=BODY_FORMAT.GRAPH_DATA,
body=json.dumps(color_map_data))
result.add_section(section_color_map)
# ==================================================================
# URL section: BODY_FORMAT.URL
# Generate a list of clickable urls using a json encoded format
# As you can see here, the body of the section can be set directly instead of line by line
random_host = get_random_host()
url_section = ResultSection('Example of a simple url section',
body_format=BODY_FORMAT.URL,
body=json.dumps({
"name":
"Random url!",
"url":
f"https://{random_host}/"
}))
# Since urls are very important features we can tag those features in the system so they are easy to find
# Tags are defined by a type and a value
url_section.add_tag("network.static.domain", random_host)
# You may also want to provide a list of url!
# Also, No need to provide a name, the url link will be displayed
host1 = get_random_host()
host2 = get_random_host()
ip1 = get_random_ip()
urls = [{
"url": f"https://{host1}/"
}, {
"url": f"https://{host2}/"
}, {
"url": f"https://{ip1}/"
}]
url_sub_section = ResultSection(
'Example of a url section with multiple links',
body_format=BODY_FORMAT.URL,
body=json.dumps(urls))
url_sub_section.set_heuristic(random.randint(1, 4))
url_sub_section.add_tag("network.static.ip", ip1)
url_sub_section.add_tag("network.static.domain", host1)
url_sub_section.add_tag("network.dynamic.domain", host2)
# Since url_sub_section is a sub-section of url_section
# we will add it as a sub-section of url_section not to the main result itself
url_section.add_subsection(url_sub_section)
result.add_section(url_section)
# ==================================================================
# Memory dump section: BODY_FORMAT.MEMORY_DUMP
# Dump whatever string content you have into a <pre/> html tag so you can do your own formatting
data = hexdump(
b"This is some random text that we will format as an hexdump and you'll see "
b"that the hexdump formatting will be preserved by the memory dump section!"
)
memdump_section = ResultSection(
'Example of a memory dump section',
body_format=BODY_FORMAT.MEMORY_DUMP,
body=data)
memdump_section.set_heuristic(random.randint(1, 4))
result.add_section(memdump_section)
# ==================================================================
# KEY_VALUE section:
# This section allows the service writer to list a bunch of key/value pairs to be displayed in the UI
# while also providing easy to parse data for auto mated tools.
# NB: You should definitely use this over a JSON body type since this one will be displayed correctly
# in the UI for the user
# The body argument must be a json dumps of a dictionary (only str, int, and booleans are allowed)
kv_body = {
"a_str": "Some string",
"a_bool": False,
"an_int": 102,
}
kv_section = ResultSection('Example of a KEY_VALUE section',
body_format=BODY_FORMAT.KEY_VALUE,
body=json.dumps(kv_body))
result.add_section(kv_section)
# ==================================================================
# JSON section:
# Re-use the JSON editor we use for administration (https://github.com/josdejong/jsoneditor)
# to display a tree view of JSON results.
# NB: Use this sparingly! As a service developer you should do your best to include important
# results as their own result sections.
# The body argument must be a json dump of a python dictionary
json_body = {
"a_str": "Some string",
"a_list": ["a", "b", "c"],
"a_bool": False,
"an_int": 102,
"a_dict": {
"list_of_dict": [{
"d1_key": "val",
"d1_key2": "val2"
}, {
"d2_key": "val",
"d2_key2": "val2"
}],
"bool":
True
}
}
json_section = ResultSection('Example of a JSON section',
body_format=BODY_FORMAT.JSON,
body=json.dumps(json_body))
result.add_section(json_section)
# ==================================================================
# Re-Submitting files to the system
# Adding extracted files will have them resubmitted to the system for analysis
# This file will generate random results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(data.encode())
request.add_extracted(temp_path, "file.txt",
"Extracted by some magic!")
# This file will generate empty results on the next run
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "wb") as myfile:
myfile.write(b"EMPTY")
request.add_extracted(temp_path, "empty.txt",
"Extracted empty resulting file")
# ==================================================================
# Supplementary files
# Adding supplementary files will save them on the datastore for future
# reference but wont reprocess those files.
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(json.dumps(urls))
request.add_supplementary(temp_path, "urls.json",
"These are urls as a JSON file")
# like embedded files, you can add more then one supplementary files
fd, temp_path = tempfile.mkstemp(dir=self.working_directory)
with os.fdopen(fd, "w") as myfile:
myfile.write(json.dumps(json_body))
request.add_supplementary(temp_path, "json_body.json",
"This is the json_body as a JSON file")
# ==================================================================
# Wrap-up:
# Save your result object back into the request
request.result = result
# ==================================================================
# Empty results file
elif request.sha256 == 'cc1d2f838445db7aec431df9ee8a871f40e7aa5e064fc056633ef8c60fab7b06':
# Creating and empty result object
request.result = Result()
# ==================================================================
# Randomized results file
else:
# For the randomized results file, we will completely randomize the results
# The content of those results do not matter since we've already showed you
# all the different result sections, tagging, heuristics and file upload functions
embedded_result = Result()
# random number of sections
for _ in range(1, 3):
embedded_result.add_section(self._create_random_section())
request.result = embedded_result
