def _find_search_optimizations(filters):
"""
Searches through all the filters, and creates white/blacklists of types and
IDs, which can be used to optimize the filesystem search.
:param filters: An iterable of filter objects representing a query
:return: A 2-tuple of AuthSet objects: the first is for object types, and
the second is for object IDs.
"""
# The basic approach to this is to determine what is allowed and
# prohibited, independently, and then combine them to create the final
# white/blacklists.
allowed_types = allowed_ids = None
prohibited_types = set()
prohibited_ids = set()
for filter_ in filters:
if filter_.property == "type":
if filter_.op in ("=", "in"):
allowed_types = _update_allow(allowed_types, filter_.value)
elif filter_.op == "!=":
prohibited_types.add(filter_.value)
elif filter_.property == "id":
if filter_.op == "=":
# An "allow" ID filter implies a type filter too, since IDs
# contain types within them.
allowed_ids = _update_allow(allowed_ids, filter_.value)
allowed_types = _update_allow(allowed_types,
get_type_from_id(filter_.value))
elif filter_.op == "!=":
prohibited_ids.add(filter_.value)
elif filter_.op == "in":
allowed_ids = _update_allow(allowed_ids, filter_.value)
allowed_types = _update_allow(allowed_types,
(get_type_from_id(id_)
for id_ in filter_.value))
opt_types = AuthSet(allowed_types, prohibited_types)
opt_ids = AuthSet(allowed_ids, prohibited_ids)
# If we have both type and ID whitelists, perform a type-based intersection
# on them, to further optimize. (Some of the cross-property constraints
# occur above; this is essentially a second pass which operates on the
# final whitelists, which among other things, incorporates any of the
# prohibitions found above.)
if opt_types.auth_type == AuthSet.WHITE and \
opt_ids.auth_type == AuthSet.WHITE:
opt_types.values.intersection_update(
get_type_from_id(id_) for id_ in opt_ids.values)
opt_ids.values.intersection_update(
id_ for id_ in opt_ids.values
if get_type_from_id(id_) in opt_types.values)
return opt_types, opt_ids
def get_techniques_used_by_all_groups(self, stix_format=True):
groups = self.get_groups()
groups = self.remove_revoked(groups)
techniques = self.get_techniques()
group_relationships = list()
group_techniques_ref = list()
groups_use_techniques = list()
filters = [
Filter("type", "=", "relationship"),
Filter('relationship_type', '=', 'uses')
]
relationships = self.COMPOSITE_DS.query(filters)
for rel in relationships:
if get_type_from_id(rel.source_ref) == 'intrusion-set'\
and get_type_from_id(rel.target_ref) == 'attack-pattern':
group_relationships.append(rel)
for g in groups:
for rel in group_relationships:
if g['id'] == rel['source_ref']:
gs = json.loads(g.serialize())
gs['technique_ref'] = rel['target_ref']
gs['relationship_description'] = rel['description']
gs['relationship_id'] = rel['id']
group_techniques_ref.append(gs)
for gt in group_techniques_ref:
for t in techniques:
if gt['technique_ref'] == t['id']:
if 'revoked' in t.keys():
gt['revoked'] = t['revoked']
tactic_list = list()
if 'kill_chain_phases' in t.keys():
tactic_list = t['kill_chain_phases']
gt['technique'] = t['name']
if 'description' in t.keys():
gt['technique_description'] = t['description']
gt['tactic'] = tactic_list
gt['technique_id'] = t['external_references'][0][
'external_id']
gt['matrix'] = t['external_references'][0]['source_name']
if 'x_mitre_platforms' in t.keys():
gt['platform'] = t['x_mitre_platforms']
if 'x_mitre_data_sources' in t.keys():
gt['data_sources'] = t['x_mitre_data_sources']
if 'x_mitre_permissions_required' in t.keys():
gt['permissions_required'] = t[
'x_mitre_permissions_required']
if 'x_mitre_effective_permissions' in t.keys():
gt['effective_permissions'] = t[
'x_mitre_effective_permissions']
groups_use_techniques.append(gt)
if not stix_format:
groups_use_techniques = self.translate_stix_objects(
groups_use_techniques)
return groups_use_techniques
def get_techniques_mitigated_by_mitigation(self,
stix_object,
stix_format=True):
relationships = self.get_relationships_by_object(stix_object)
mitigation_relationships = list()
for relation in relationships:
if get_type_from_id(relation.source_ref) == 'course-of-action':
mitigation_relationships.append(relation)
filter_objects = [
Filter('type', '=', 'attack-pattern'),
Filter('id', '=', [r.target_ref for r in mitigation_relationships])
]
try:
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(
filter_objects)
except:
enterprise_stix_objects = []
try:
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
except:
pre_stix_objects = []
try:
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
except:
mobile_stix_objects = []
all_techniques_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
if not stix_format:
all_techniques_list = self.translate_stix_objects(
all_techniques_list)
return all_techniques_list
def get_software_used_by_group(self, stix_object, stix_format=True):
relationships = self.get_relationships_by_object(stix_object)
software_relationships = list()
for relation in relationships:
if get_type_from_id(relation.target_ref) in ['malware', 'tool']:
software_relationships.append(relation)
filter_objects = [
Filter('type', 'in', ['malware', 'tool']),
Filter('id', '=', [r.target_ref for r in software_relationships])
]
try:
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(
filter_objects)
except:
enterprise_stix_objects = []
try:
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
except:
pre_stix_objects = []
try:
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
except:
mobile_stix_objects = []
all_software_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
if not stix_format:
all_software_list = self.translate_stix_objects(all_software_list)
return all_software_list
def load_baseline(cls):
print("Loading baseline from MITRE. This may take some time...")
for act in cls.__get__all_actors():
actor = Actor(ext_id=act.id, name=act.name)
if "aliases" in act:
actor.alias = json.dumps(act['aliases'])
if "description" in act:
actor.description = act.description
db.session.add(actor)
for tec in cls.__get_all_techniques():
behaviour = Behaviour(extId=tec.id, name=tec.name)
if "description" in tec:
behaviour.description = tec.description
if "x_mitre_platforms" in tec:
behaviour.platforms = json.dumps(tec['x_mitre_platforms'])
for rel in cls.__find_relationships(target_id=tec.id, rel_type='mitigates'):
mitigation = cls.__get_mitigation_by_id(rel.source_ref)
behaviour.mitigation_name = mitigation[0].name
behaviour.mitigation_desc = mitigation[0].description
db.session.add(behaviour)
for sw in cls.__get_all_software():
software = Software(extId=sw.id, name=sw.name, type=sw.type)
if "description" in sw:
software.description = sw.description
if "x_mitre_aliases" in sw:
software.alias = json.dumps(sw['x_mitre_aliases'])
if "x_mitre_platforms" in sw:
software.platforms = json.dumps(sw['x_mitre_platforms'])
db.session.add(software)
for actor in Actor.query.all():
#print("MITRE: Setting relationships for actor {}...".format(actor.name))
for rel in cls.__find_relationships(source_id=actor.ext_id, rel_type='uses'):
type = utils.get_type_from_id(rel.target_ref)
if type in ['attack-pattern']:
behaviour = Behaviour.query.filter_by(ext_id=rel.target_ref).first()
if behaviour:
actor.uses_behaviours.append(behaviour)
elif type in ['malware', 'tool']:
software = Software.query.filter_by(ext_id=rel.target_ref).first()
if software:
actor.uses_software.append(software)
db.session.add(actor)
db.session.commit()
print("Successfully loaded baseline from MITRE Att&ck knowledge base")
def get_techniques_used_by_group_software(self,
stix_object,
stix_format=True):
# Get all relationships available for group
relationships = self.get_relationships_by_object(stix_object)
software_relationships = list()
# Get all software relationships from group
for relation in relationships:
if get_type_from_id(relation.target_ref) in ['malware', 'tool']:
software_relationships.append(relation)
# Get all used by the software that is used by group
filter_objects = [
Filter('type', '=', 'relationship'),
Filter('relationship_type', '=', 'uses'),
Filter('source_ref', 'in',
[r.target_ref for r in software_relationships])
]
try:
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(
filter_objects)
except:
enterprise_stix_objects = []
try:
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
except:
pre_stix_objects = []
try:
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
except:
mobile_stix_objects = []
software_uses = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
# Get all techniques used by the software that is used by group
filter_techniques = [
Filter('type', '=', 'attack-pattern'),
Filter('id', 'in', [s.target_ref for s in software_uses])
]
try:
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(
filter_techniques)
except:
enterprise_stix_objects = []
try:
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_techniques)
except:
pre_stix_objects = []
try:
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(
filter_techniques)
except:
mobile_stix_objects = []
all_techniques_list = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
if not stix_format:
all_techniques_list = self.translate_stix_objects(
all_techniques_list)
return all_techniques_list
def get_techniques_by_group(self, stix_id):
group_uses = [
r
for r in self.src.relationships(stix_id, 'uses', source_only=True)
if get_type_from_id(r.target_ref) in ['malware', 'tool']
]
software_uses = self.src.query([
Filter('type', '=', 'relationship'),
Filter('relationship_type', '=', 'uses'),
Filter('source_ref', 'in', [r.source_ref for r in group_uses])
])
techniques_query = self.src.query([
Filter('type', '=', 'attack-pattern'),
Filter('id', 'in', [r.target_ref for r in software_uses])
])
dict_techniques = []
for current_technique in techniques_query:
string_report = current_technique.serialize()
dict_report = json.loads(string_report)
dict_techniques.append(dict_report)
techniques_df = pd.DataFrame(dict_techniques)
if techniques_df.empty:
return techniques_df
techniques_df = techniques_df.fillna("")
techniques_df["kill_chain_phases"] = techniques_df[
"kill_chain_phases"].apply(lambda x: [
y["phase_name"] for y in x if 'mitre' in y['kill_chain_name']
])
techniques_df = techniques_df.rename(
index=str,
columns={
"x_mitre_permissions_required": "permissions_required",
"x_mitre_platforms": "platforms",
"id": "mitre_id",
"kill_chain_phases": "tactics"
})
column_set = set(techniques_df.columns.values)
for i in self.columns_list["techniques"]:
column_set.add(i)
techniques_df = techniques_df.reindex(columns=list(column_set),
fill_value="")
return techniques_df[self.columns_list["techniques"]]
def get_techniques_by_group_software(src, group_stix_id):
# get the malware, tools that the group uses
group_uses = [
r for r in src.relationships(group_stix_id, 'uses', source_only=True)
if get_type_from_id(r.target_ref) in ['malware', 'tool']
]
# get the technique stix ids that the malware, tools use
software_uses = src.query([
Filter('type', '=', 'relationship'),
Filter('relationship_type', '=', 'uses'),
Filter('source_ref', 'in', [r.source_ref for r in group_uses])
])
#get the techniques themselves
return src.query([
Filter('type', '=', 'attack-pattern'),
Filter('id', 'in', [r.target_ref for r in software_uses])
])
def get_techniques_mitigated_by_all_mitigations(self, stix_format=True):
# Get all relationships available
relationships = self.get_relationships()
# Get all mitigation relationships
mitigation_relationships = list()
for relation in relationships:
if get_type_from_id(relation.source_ref) in ['course-of-action']:
mitigation_relationships.append(relation)
# Get all techniques
techniques = self.get_techniques()
all_techniques_list = list()
# loop through mitigation relationships to match technique
for mr in mitigation_relationships:
for t in techniques:
if t['id'] == mr['target_ref']:
all_techniques_list.append(t)
if not stix_format:
all_techniques_list = self.translate_stix_objects(all_techniques_list)
return all_techniques_list
def get_software_map(composite_ds):
print("Parsing softwares ...")
malware_filter = Filter('type', '=', 'malware')
tool_filter = Filter('type', '=', 'tool')
software_map = {}
revoke_map = {}
for cur_filter in [malware_filter, tool_filter]:
for item in composite_ds.query(cur_filter):
name = item['name']
software_id = None
for er in item['external_references']:
if er['source_name'] in [
"mitre-attack", "mobile-mitre-attack",
"mitre-mobile-attack"
]:
software_id = er['external_id']
break
if software_id:
if not item['revoked']:
if cur_filter == malware_filter:
soft_type = 'malware'
else:
soft_type = 'tool'
desc = item['description']
platforms = item.get('x_mitre_platforms') or []
related_attack_ids = []
attack_pattern_refs = [
r.target_ref for r in composite_ds.relationships(
item, 'uses', source_only=True)
if get_type_from_id(r.target_ref) == 'attack-pattern'
]
for attack_item in composite_ds.query([
Filter('type', '=', 'attack-pattern'),
Filter('id', 'in', attack_pattern_refs)
]):
for er in attack_item['external_references']:
if er['source_name'] in [
"mitre-attack", "mobile-mitre-attack",
"mitre-mobile-attack"
]:
related_attack_ids.append(er['external_id'])
break
software_map[software_id] = {
"name": name,
"description": desc,
"platforms": platforms,
"software_id": software_id,
"type": soft_type,
"attack_ids": related_attack_ids
}
print(f"\tAdding {name.upper()} as ID: {software_id}")
else:
revoke_id = get_revoked_by(item['id'], composite_ds)
if revoke_id is None:
print(
f"\t[WARN] {name.upper()} ({software_id}) has been revoked without being replaced."
)
else:
revoke_map[software_id] = revoke_id
print(
f"\tAdding revoked {name.upper()} to the revoked map: {software_id} => {revoke_id}"
)
else:
print(f"[ERR] Ignored {name.upper()}: No attack ID found.")
return software_map, revoke_map
