def check_url(url) -> bool:
'''
check if url or not
'''
with ignore_excpetion(Exception):
if not url.startswith(("http://", "https://", "ftp://")):
url = "http://" + url
if get_tld(url, fail_silently=True):
root = None
with ignore_excpetion(Exception):
root = get_fld(url, fix_protocol=True)
if root:
return True
return False
def do_analyze(self, line, silent=False):
good_exec = False
with ignore_excpetion(Exception):
line["output"] = json_settings[
environ["analyzer_env"]]["malware_output_folder"]
parsed_args = vars(self._analyze_parser.parse_args(""))
parsed = Namespace({
**parsed_args,
**line
}, ["open", "print"], [
"db_dump_json", "db_dump_html", "disk_dump_html",
"disk_dump_json"
])
if not parsed.uuid:
return
if int(parsed.analyzer_timeout) > 0 and int(
parsed.analyzer_timeout) < 240:
json_settings[
environ["analyzer_env"]]["analyzer_timeout"] = int(
parsed.analyzer_timeout)
if int(parsed.function_timeout) > 0 and int(
parsed.function_timeout) < 240:
json_settings[
environ["analyzer_env"]]["function_timeout"] = int(
parsed.function_timeout)
good_exec = True
if good_exec:
log_string(
"Default timeout {}s for the task, and {}s for each logic".
format(
json_settings[environ["analyzer_env"]]["analyzer_timeout"],
json_settings[environ["analyzer_env"]]
["function_timeout"]), "Yellow")
else:
log_string("Parsing failed, something went wrong..", "Red")
return
log_string("Task {} (Started)".format(parsed.uuid), "Yellow")
if parsed.file:
with ignore_excpetion(Exception):
setup_task_logger(parsed.uuid)
self.analyze_file(parsed)
cancel_task_logger(parsed.uuid)
else:
log_string("File, Folder or Buffer is missing", "Red")
log_string("Task {} (Finished)".format(parsed.uuid), "Green")
def get_urls(self, data):
'''
check if root domain exists in the top 10000 or not
if yes appened it to list
'''
roots = []
temp_var = list(set(findall(self.links, self.wordsstripped)))
for _ in temp_var:
url = ""
if not _[0].startswith(("http://", "https://", "ftp://")):
url = "http://" + _[0]
if get_tld(url, fail_silently=True):
root = None
with ignore_excpetion(Exception):
root = get_fld(url, fix_protocol=True)
if root:
roots.append(root)
if roots:
for domain in self.topdomains:
dist = edit_distance(domain, root)
if dist <= 2:
data.append({
"Distance": dist,
"URL": root,
"Similar": domain
})
def get_average(temp_list):
'''
no verbose
'''
with ignore_excpetion(Exception):
return sum(temp_list) / len(temp_list)
return 0
def create_image(_buffer, temp_c, temp_s) -> str:
'''
no verbose
'''
with ignore_excpetion(Exception):
temp_list = list(_buffer)
_list = list(chunk_list(temp_list, temp_c))
out = list(chunk_list([int(get_average(l)) for l in _list], int(temp_s)))
temp__x = 10
temp_h = len(out) * temp__x
temp_w = int(temp_s) * temp__x
img = Image.new('RGB', (temp_w, temp_h), (255, 255, 255))
draw = ImageDraw.Draw(img)
temp__x1 = 0
temp_y1 = 0
temp__x2 = temp__x
temp_y2 = temp__x
for row in out:
for item in row:
value = 255 - item
if 0 <= value <= 255:
draw.rectangle([temp__x1, temp_y1, temp__x2, temp_y2], fill=(value, value, value), outline="#C8C8C8")
temp__x1 = temp__x1 + temp__x
temp__x2 = temp__x1 + temp__x
if temp__x1 >= temp_w:
temp_y1 = temp_y2
temp_y2 = temp_y2 + temp__x
temp__x1 = 0
temp__x2 = temp__x1 + temp__x
buffer = BytesIO()
img.save(buffer, format="JPEG", quality=10, optimize=True)
bimage = b64encode(buffer.getvalue())
output = "data:image/jpeg;base64, {}".format(bimage.decode("utf-8", errors="ignore"))
return output
return "0"
def check_mitre_similarity(self, data):
'''
check detections from parsediocs.json against wordsstripped, if yes bring attack info (added > 3 less FB )
'''
_list = []
file_buffer = loads(open(self.parsediocs).read())
for attack in file_buffer:
for ioc in file_buffer[attack]:
if ioc.lower() in self.wordsstripped and len(ioc.lower()) > 3:
_list.append(ioc.lower())
if len(_list) > 0:
temp_x = self.search_in_mitre_and_return(
self.mitre.fulldict, attack)
temp_dict = {
"Id": attack,
"Name": "None",
"Detected": ', '.join(_list),
"Description": "None"
}
if temp_x:
with ignore_excpetion(Exception):
temp_dict = {
"Id": attack,
"Name": temp_x["name"],
"Detected": ', '.join(_list),
"Description": temp_x["description"]
}
data["Attack"].append(temp_dict)
_list = []
def rcompile_and_find(self, data, filename):
'''
parse the detections and check them against wordsstripped
'''
with copen(filename, "r", encoding='utf8') as file:
for _ in loads(file.read()):
with ignore_excpetion(Exception):
if "Type" in _ and "QREGEX" in _["Type"]:
_list = []
tempmatches = 0
for item in _["Detection"]:
if _["Options"]["Word"] == "Normal":
temp_value = rsearch(
rcompile(r"{}".format(item),
_["Options"]["Flag"]),
self.wordsstripped)
elif _["Options"]["Word"] != "Normal":
temp_value = rsearch(
rcompile(r"\b{}\b".format(item),
_["Options"]["Flag"]),
self.wordsstripped)
if temp_value is not None:
_list.append(temp_value.group())
tempmatches += 1
if _list and tempmatches >= _["Options"]["Required"]:
data.append({
"Matched": tempmatches,
"Required": _["Options"]["Required"],
"Behavior": _["Name"],
"Detected": ', '.join(_list)
})
def unpack_file(data, _path):
'''
unpack files using 7z into temp folder
'''
data["Packed"] = {"Files": [],
"Detected": [],
"_Detected": ["Name", "Path"],
"_Files": ["Name", "Type", "Extension", "md5", "Path"]}
with ignore_excpetion(Exception):
process = Popen(["7z", "t", _path, "-pdummypassword2019!!"], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, error = process.communicate()
if b"ERROR: Wrong password" in error:
return
process = Popen(["7z", "x", _path, "-aoa", "-o" + data["Location"]["Folder"]], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, error = process.communicate()
for currentp_ath, folders, files in walk(data["Location"]["Folder"]):
for file in files:
temp_buffer = open(path.join(currentp_ath, file), "rb").read()
_md5 = md5(temp_buffer).hexdigest()
mime = from_file(path.join(currentp_ath, file), mime=True)
data["Packed"]["Files"].append({"Name": file,
"Type": mime,
"Extension": guess_type(path.join(currentp_ath, file))[0],
"Path": path.join(currentp_ath, file),
"md5": _md5})
data["FilesDumps"].update({path.join(currentp_ath, file): temp_buffer})
def finduses(self):
'''
find all relationship_type uses value and parse them into hardcoded list
'''
temp_l = self.search_in_mitre_and_return(self.fulldict, {'relationship_type':'uses'}, ['source_ref', 'target_ref', 'description', 'collection'])
temp_d = {}
for temp_i in temp_l:
temp_s = self.search_once(self.fulldict, {'id':temp_i['source_ref']})
temp_u = self.search_once(self.fulldict, {'id':temp_i['target_ref']})
temp_xx = None
with ignore_excpetion(Exception):
temp_xx = temp_u['external_references'][0]['external_id']
if temp_s and temp_u:
if temp_d.get(temp_s['type'.lower().rstrip()]):
if temp_d[temp_s['type']].get(temp_s['name']) == [] or temp_d[temp_s['type']].get(temp_s['name']):
temp_d[temp_s['type']][temp_s['name']].append({'id':temp_xx, 'name':temp_u['name'], 'type':temp_u['type'], 'description':temp_i['description'], 'collection':temp_i['collection']})
else:
temp_d[temp_s['type']].update({temp_s['name']:[{'id':temp_xx, 'name':temp_u['name'], 'type':temp_u['type'], 'description':temp_i['description'], 'collection':temp_i['collection']}]})
else:
temp_d.update({temp_s['type'].lower().rstrip():{temp_s['name']:[{'id':temp_xx, 'name':temp_u['name'], 'type':temp_u['type'], 'description':temp_i['description'], 'collection':temp_i['collection']}]}})
for temp_i in temp_d['intrusion-set']:
for temp_ii in temp_d['intrusion-set'][temp_i]:
if temp_ii['type'] == 'malware' or temp_ii['type'] == 'tool':
temp_ii['description'] = []
for temp_x in temp_d[temp_ii['type']][temp_ii['name']]:
temp_xx = self.search_once(self.fulldict, {'name':temp_x['name']})
temp_ii['description'].append({'id':temp_xx['external_references'][0]['external_id'], 'name':temp_x['name'], 'type':temp_x['type']})
return temp_d
def create_d3_artifacts(self, data) -> bool:
'''
get artifacts from data and generate d3
'''
data["REFS"] = {
"GRAPH": {
"nodes": [],
"links": []
},
"TEXT": [],
"_TEXT": ["From", "To"]
}
_node = []
_links = []
_list = []
_temp = []
with ignore_excpetion(Exception):
for item in data["Strings"]["IPS"]:
_list.append({"From": "File", "To": item["IP"]})
with ignore_excpetion(Exception):
for item in data["Strings"]["EMAILs"]:
_list.append({"From": "File", "To": item["EMAIL"]})
for item in _list:
if item["From"] not in _temp:
_temp.append(item["From"])
_node.append({"func": item["From"]})
if item["To"] not in _temp:
_temp.append(item["To"])
_node.append({"func": item["To"]})
for item in _list:
with ignore_excpetion(Exception):
temp_var_s = _temp.index(item["From"])
temp_var_t = _temp.index(item["To"])
if next((item
for item in _links if item["source"] == temp_var_s
and item["target"] == temp_var_t), False) is False:
_links.append({"source": temp_var_s, "target": temp_var_t})
if len(_node) > 0 and len(_links) > 0:
data["REFS"]["GRAPH"]["nodes"] = _node
data["REFS"]["GRAPH"]["links"] = _links
data["REFS"]["TEXT"] = _list
def search_once(self, temp_s, temp_d):
'''
search once
'''
with ignore_excpetion(Exception):
for temp_x in temp_s:
if all((temp_k in temp_x and temp_x[temp_k] == temp_var) for temp_k, temp_var in temp_d.items()):
return temp_x
return None
def intro(filename, link):
'''
this function is needed for the home page intro
'''
intromarkdown = ""
with ignore_excpetion(Exception):
ret_request = get(link, verify=False, timeout=2)
if ret_request.ok is True:
intromarkdown = search(rcompile(r"\#\# Features.*", DOTALL),
ret_request.text).group(0)
if intromarkdown == "":
with ignore_excpetion(Exception):
readmefolder = path.abspath(
path.join(path.dirname(__file__), filename))
with open(readmefolder, "rU", encoding="utf-8") as file:
intromarkdown = search(rcompile(r"\#\# Features.*", DOTALL),
file.read()).group(0)
return intromarkdown
def test_base64(self, _str) -> str:
'''
match decoding base64 then encoding means most likely base64
'''
with ignore_excpetion(Exception):
temp_str = b64decode(_str)
if b64encode(temp_str) == _str:
return temp_str
return ""
def get_functions_old(self, temp_f) -> list:
'''
get function names and constant strings by regex
'''
_list = []
print(temp_f)
strings = findall(rb"[\x24][\xd8]\\*\*\* ([\x20-\x7e]{4,})", temp_f)
for _ in strings:
if b"() " in _:
__ = _.split(b"() ")
with ignore_excpetion(Exception):
_list.append({"Type":"Function", "Name":__[0].decode("utf-8", errors="ignore")})
_list.append({"Type":"String", "Name":__[1].decode("utf-8", errors="ignore")})
strings = findall(b"[\x24][\xd8] ([\x20-\x7e]{4,})", temp_f) #<--- check this out
for _ in strings:
with ignore_excpetion(Exception):
_list.append({"Type":"String", "Name":_.decode("utf-8", errors="ignore")})
return _list
def extract_macros(self, path) -> list:
'''
Extract macros
'''
temp_list = []
with ignore_excpetion(Exception):
for (temp_f, temp_s, vbaname, vbacode) in VBA_Parser(path).extract_macros():
temp_list.append({"Name": vbaname, "VBA": vbacode})
return temp_list
def xref(self, r2p, line) -> list:
'''
get refes
'''
temp_string = ""
with ignore_excpetion(Exception):
add = line.split(" ")[0]
int(add, 0)
temp_string = r2p.cmd("pd 1 @ " + add + "~XREF")
return temp_string.split("\n")
def analyze(self, data):
'''
start ocr reading logic for packed files only
'''
self.words = []
data["OCR"] = deepcopy(self.datastruct)
with ignore_excpetion(Exception):
if len(data["Packed"]["Files"]) > 0:
self.mix_and_setup_file_ocr(data, data["Packed"]["Files"])
self.check_ocr_text(data["OCR"]["OCR"], self.words)
def get_headers(self, data, msg) -> list:
'''
get msg headers by buffer
'''
headers = []
for key, value in msg.header.items():
data.append({"Key": key, "Value": value, "descriptions": ""})
with ignore_excpetion(Exception):
headers.append(str.encode(value)) # convert to bytes...
return headers
def find_codes(self, codes, data):
'''
find codes
'''
keys = ["PCAP", "Patterns"]
for key in keys:
with ignore_excpetion(Exception):
if key in data and "IP4S" in data[key]:
for temp_value in data[key]["IP4S"]:
if temp_value["Code"] not in codes and temp_value["Code"]:
codes.append(temp_value["Code"])
def find_flags(self, flags, data):
'''
get ISO 3166-1 alpha-2 codes
'''
keys = ["PCAP", "Patterns"]
for key in keys:
with ignore_excpetion(Exception):
if key in data and "IP4S" in data[key]:
for temp_value in data[key]["IP4S"]:
if temp_value["Alpha2"] not in flags and len(temp_value["Alpha2"]) > 0:
flags.append(temp_value["Alpha2"].lower())
def push_to_elastic(uuid, json):
'''
Not implemented
'''
with ignore_excpetion(Exception):
res = ELASTIC_SEARCH.index(index='jsdoc',
ignore=400,
doc_type='doc',
id=uuid,
body=json)
return res
def session_key(filename):
'''
get the generated session key
'''
key = ""
with ignore_excpetion(Exception):
readmefolder = path.abspath(path.join(path.dirname(__file__),
filename))
with open(readmefolder, "rU", encoding="utf-8") as file:
key = file.read()
return key
def mix_and_setup_file_ocr(self, data, paths):
'''
loop paths, convert each image to RGBA, and read text from image
'''
for item in paths:
with ignore_excpetion(Exception):
image = Image.open(BytesIO(data["FilesDumps"][item["Path"]]))
image = image.convert("RGBA")
text = image_to_string(image, config='--psm 6')
words = findall("[\x20-\x7e]{4,}", text)
if len(words) > 0:
self.words.append([words, item["Path"]])
def do_cache_switches(self):
good_exec = False
with ignore_excpetion(Exception):
with StringIO() as buf, redirect_stdout(buf):
self._analyze_parser.print_help()
output = buf.getvalue()
#subbed = search(compile(r"Analysis switches\:.*", DOTALL), output).group(0)
put_cache("switches", output)
good_exec = True
if good_exec:
log_string("Dumped switches", "Green")
else:
log_string("Dumping switches failed", "Red")
def is_toggled(self):
'''
toggled route (this will keep track of toggled items)
'''
with ignore_excpetion(Exception):
if current_user.is_authenticated:
json_content = request.get_json(silent=True)
for key, value in json_content.items():
if value == "false":
session["navs"].remove(key)
else:
session["navs"].append(key)
return jsonify("Done")
def serialize_obj(obj):
'''
recursive str serialization obj
'''
with ignore_excpetion(Exception):
if isinstance(obj, dict):
for key, value in obj.items():
obj[key] = serialize_obj(value)
elif isinstance(obj, list):
for i, item in enumerate(obj):
obj[i] = serialize_obj(item)
else:
obj = str(obj)
return obj
def get_entropy_float_ret(data) -> float:
'''
get entropy of buffer
'''
with ignore_excpetion(Exception):
if not data:
return 0.0
entropy = 0
counter = Counter(data)
temp_len = len(data)
for count in counter.values():
temp_var = float(count) / temp_len
entropy += -temp_var * log2(temp_var)
return entropy
return 0.0
def get_entropy(data) -> str:
'''
get entropy of buffer
'''
with ignore_excpetion(Exception):
if not data:
return "0.0 (Minimum: 0.0, Max: 8.0)"
entropy = 0
counter = Counter(data)
temp_len = len(data)
for count in counter.values():
temp_var = float(count) / temp_len
entropy += -temp_var * log2(temp_var)
return "{} (Minimum: 0.0, Maximum: 8.0)".format(entropy)
return "None"
def check_packed_files(_path, files) -> bool:
'''
check if archive contains strings or not
'''
with ignore_excpetion(Exception):
detect = 0
process = Popen(["7z", "l", _path], stdin=PIPE, stdout=PIPE, stderr=PIPE)
output, error = process.communicate()
output = output.decode("utf-8", errors="ignore")
for _ in files:
if _.lower() in output.lower():
detect += 1
if detect == len(files):
return True
return False
def check_hex(self, _data):
'''
check if buffer contains tags <>
'''
temp_list = []
temp_var = findall(self.hex, self.wordsstripped)
if len(temp_var) > 0:
for _ in temp_var:
temp_list.append(_)
for temp_var in set(temp_list):
with ignore_excpetion(Exception):
parsed = unhexlify(temp_var)
_data.append({
"Count": temp_list.count(temp_var),
"HEX": temp_var,
"Parsed": parsed.decode('utf-8', errors="ignore")
})
