def test_heuristics_valid():
heuristic_list = [random_model_obj(Heuristic) for _ in range(4)]
heuristics = {x.heur_id: x for x in heuristic_list}
attack_ids = list(
set([
random.choice(list(attack_map.keys()))
for _ in range(random.randint(1, 3))
]))
signatures = {}
score_map = {}
for x in range(random.randint(2, 4)):
name = get_random_word()
if x >= 2:
score_map[name] = random.randint(10, 100)
signatures[name] = random.randint(1, 3)
service_heur = dict(heur_id=random.choice(list(heuristics.keys())),
score=0,
attack_ids=attack_ids,
signatures=signatures,
frequency=0,
score_map=score_map)
result_heur = service_heuristic_to_result_heuristic(
deepcopy(service_heur), heuristics)
assert result_heur is not None
assert service_heur['heur_id'] == result_heur['heur_id']
assert service_heur['score'] != result_heur['score']
for attack in result_heur['attack']:
assert attack['attack_id'] in attack_ids
for signature in result_heur['signature']:
assert signature['name'] in signatures
assert signature['frequency'] == signatures[signature['name']]
def test_heuristics_valid():
heuristic_list = [random_model_obj(Heuristic) for _ in range(4)]
heuristics = {x.heur_id: x for x in heuristic_list}
software_ids = list(set([random.choice(list(software_map.keys())) for _ in range(random.randint(1, 3))]))
attack_ids = list(set([random.choice(list(attack_map.keys())) for _ in range(random.randint(1, 3))]))
attack_ids_to_fetch_details_for = attack_ids[:]
for software_id in software_ids:
software_attack_ids = software_map[software_id]["attack_ids"]
for software_attack_id in software_attack_ids:
if software_attack_id in attack_map and software_attack_id not in attack_ids_to_fetch_details_for:
attack_ids_to_fetch_details_for.append(software_attack_id)
else:
print(f"Invalid related attack_id '{software_attack_id}' for software '{software_id}'. Ignoring it.")
attack_id_details = {attack_id: {"pattern": attack_map[attack_id]["name"], "categories": attack_map[attack_id]["categories"]} for attack_id in attack_ids_to_fetch_details_for}
attack_ids.extend(software_ids)
signatures = {}
score_map = {}
for x in range(random.randint(2, 4)):
name = get_random_word()
if x >= 2:
score_map[name] = random.randint(10, 100)
signatures[name] = random.randint(1, 3)
service_heur = dict(
heur_id=random.choice(list(heuristics.keys())),
score=0,
attack_ids=attack_ids,
signatures=signatures,
frequency=0,
score_map=score_map
)
result_heur = service_heuristic_to_result_heuristic(deepcopy(service_heur), heuristics)
assert result_heur is not None
assert service_heur['heur_id'] == result_heur['heur_id']
assert service_heur['score'] != result_heur['score']
for attack in result_heur['attack']:
attack_id = attack['attack_id']
assert attack_id in attack_ids_to_fetch_details_for
assert attack['pattern'] == attack_id_details[attack_id]['pattern']
assert attack['categories'] == attack_id_details[attack_id]['categories']
for signature in result_heur['signature']:
assert signature['name'] in signatures
assert signature['frequency'] == signatures[signature['name']]
def _create_results_for_file(ds, fs, f, possible_childs=None, log=None):
r_list = []
services_done = []
section_body_format = ["TEXT", "MEMORY_DUMP", "GRAPH_DATA", "URL", "JSON", "KEY_VALUE"]
section_depth_list = [[1, 1, 2, 3, 1], [1, 2, 1], [1, 2, 3, 1], [1, 2]]
section_depth = random.choice(section_depth_list)
for _ in range(random.randint(2, 5)):
r = random_model_obj(Result)
# Only one result per service per file
while r.response.service_name in services_done:
r.response.service_name = random.choice(list(SERVICES.keys()))
for depth_id, section in enumerate(r.result.sections):
section.depth = section_depth[depth_id % len(section_depth)]
section.body_format = random.choice(section_body_format)
section.heuristic.heur_id = random.choice([f"{r.response.service_name.upper()}.{x+1}" for x in range(5)])
if section.body_format == "GRAPH_DATA":
cmap_min = 0
cmap_max = random.choice([5, 10, 20])
color_map_data = {
'type': 'colormap',
'data': {
'domain': [cmap_min, cmap_max],
'values': [random.random() * cmap_max for _ in range(50)]
}
}
section.body = json.dumps(color_map_data)
elif section.body_format == "URL":
data = [{"url": get_random_uri()} for _ in range(random.randint(1, 4))]
section.body = json.dumps(data)
elif section.body_format in ["JSON", "KEY_VALUE"]:
data = {get_random_word(): get_random_id() for _ in range(random.randint(3, 9))}
section.body = json.dumps(data)
services_done.append(r.response.service_name)
# Set the sha256
r.sha256 = f
if random.randint(1, 10) > 8:
# Generate and empty result
r_key = f"{r.build_key()}.e"
ds.emptyresult.save(r_key, random_model_obj(EmptyResult))
else:
r_key = r.build_key()
# Set random extracted files that are not top level
if not possible_childs:
r.response.extracted = []
else:
for e in r.response.extracted:
e.sha256 = random.choice(possible_childs)
# Set random supplementary files that are not top level
if r.response.supplementary:
# Edit the first file to be an ontology file
s = r.response.supplementary[0]
# Create a random ontology
onto = random_minimal_obj(ResultOntology).as_primitives(strip_null=True)
onto['header']['sha256'] = f
onto['header']['service_name'] = r.response.service_name
onto['header']['service_version'] = r.response.service_version
onto['header']['service_tool_version'] = r.response.service_tool_version
# Create it's file record
supp_file = random_model_obj(File)
byte_str = json.dumps(onto).encode('utf-8')
sha256 = hashlib.sha256(byte_str).hexdigest()
supp_file.sha256 = sha256
ds.file.save(sha256, supp_file)
fs.put(sha256, byte_str)
# Add the random files
s.sha256 = sha256
s.name = "random.ontology"
s.description = f"Random Ontology file for: {f}"
r.response.supplementary = [s]
ds.result.save(r_key, r)
if log:
log.info(f"\t\t\t{r_key}")
r_list.append(r_key)
return r_list
def _create_results_for_file(ds, f, possible_childs=None, log=None):
r_list = []
services_done = []
section_depth_list = [[1, 1, 2, 3, 1], [1, 2, 1], [1, 2, 3, 1], [1, 2]]
section_depth = random.choice(section_depth_list)
for _ in range(random.randint(2, 5)):
r = random_model_obj(Result)
for depth_id, section in enumerate(r.result.sections):
section.depth = section_depth[depth_id % len(section_depth)]
if section.body_format == "GRAPH_DATA":
cmap_min = 0
cmap_max = random.choice([5, 10, 20])
color_map_data = {
'type': 'colormap',
'data': {
'domain': [cmap_min, cmap_max],
'values':
[random.random() * cmap_max for _ in range(50)]
}
}
section.body = json.dumps(color_map_data)
elif section.body_format == "URL":
data = [{
"url": get_random_uri()
} for _ in range(random.randint(1, 4))]
section.body = json.dumps(data)
elif section.body_format in ["JSON", "KEY_VALUE"]:
data = {
get_random_word(): get_random_id()
for _ in range(random.randint(3, 9))
}
section.body = json.dumps(data)
# Only one result per service per file
while r.response.service_name in services_done:
r.response.service_name = random.choice(list(SERVICES.keys()))
services_done.append(r.response.service_name)
# Set the sha256
r.sha256 = f
if random.randint(1, 10) > 8:
# Generate and empty result
r_key = f"{r.build_key()}.e"
ds.emptyresult.save(r_key, random_model_obj(EmptyResult))
else:
r_key = r.build_key()
# Set random extracted files that are not top level
if not possible_childs:
r.response.extracted = []
else:
for e in r.response.extracted:
e.sha256 = random.choice(possible_childs)
# Set random supplementary files that are not top level
if not possible_childs:
r.response.supplementary = []
else:
for s in r.response.supplementary:
s.sha256 = random.choice(possible_childs)
ds.result.save(r_key, r)
if log:
log.info(f"\t\t\t{r_key}")
r_list.append(r_key)
return r_list