def get_collection_src():
"""
Get collection src from collections objects provided by TAXII_SERVER
:return: collection source object
"""
global CS
server = Server(TAXII_SERVER)
collections = [
c for c in server.api_roots[0].collections
if c._title not in EXCLUDE_COLLECTIONS
]
if CS.get_all_data_sources():
print("Reusing collections")
return CS
else:
print("Creating new collections")
for collection in collections:
print("Adding collection %s %s" %
(collection._title, collection.id))
CS.add_data_source(TAXIICollectionSource(collection))
return CS
def __init__(self, source='taxii', local=None):
"""
Initialization - Creates a matrix generator object
:param server: Source to utilize (taxii or local)
:param local: string path to local cache of stix data
"""
self.convert_data = {}
if source.lower() not in ['taxii', 'local']:
print(
'[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii" or "local"'
.format(source))
raise ValueError
if source.lower() == 'taxii':
self.server = Server('https://cti-taxii.mitre.org/taxii')
self.api_root = self.server.api_roots[0]
self.collections = dict()
for collection in self.api_root.collections:
if collection.title != "PRE-ATT&CK":
tc = Collection(
'https://cti-taxii.mitre.org/stix/collections/' +
collection.id)
self.collections[collection.title.split(' ')
[0].lower()] = TAXIICollectionSource(tc)
elif source.lower() == 'local':
if local is not None:
hd = MemoryStore()
if 'mobile' in local.lower():
self.collections['mobile'] = hd.load_from_file(local)
else:
self.collections['enterprise'] = hd.load_from_file(local)
else:
print(
'[MatrixGen] - "local" source specified, but path to local source not provided'
)
raise ValueError
self.matrix = {}
self._build_matrix()
def attnck_taxii():
# Initialize dictionary to hold Enterprise ATT&CK content
ent_attack = {}
pre_attack = {}
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
ENTERPRISE_ATTCK = "95ecc380-afe9-11e4-9b6c-751b66dd541e"
PRE_ATTCK = "062767bd-02d2-4b72-84ba-56caef0f8658"
MOBILE_ATTCK = "2f669986-b40b-4423-b720-4396ca6a462b"
enterprise_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
pre_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/062767bd-02d2-4b72-84ba-56caef0f8658/"
)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(enterprise_attack_collection)
tc_source2 = TAXIICollectionSource(pre_attack_collection)
# Create filters to retrieve content from Enterprise ATT&CK based on type
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
filter_objs_pre = {
"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"),
"relationships": Filter("type", "=", "relationship")
}
# Retrieve all Enterprise ATT&CK content
for key in filter_objs:
ent_attack[key] = tc_source.query(filter_objs[key])
for key in filter_objs_pre:
pre_attack[key] = tc_source2.query(filter_objs[key])
return ent_attack, pre_attack
def __init__(self, source='taxii', resource=None):
"""
Initialization - Creates a matrix generator object
:param source: Source to utilize (taxii, remote, or local)
:param resource: string path to local cache of stix data (local) or url of an ATT&CK Workbench (remote)
"""
self.convert_data = {}
self.collections = dict()
if source.lower() not in ['taxii', 'local', 'remote']:
print(
'[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii", "remote" or '
'"local"'.format(source))
raise ValueError
if source.lower() == 'taxii':
self.server = Server('https://cti-taxii.mitre.org/taxii')
self.api_root = self.server.api_roots[0]
for collection in self.api_root.collections:
if collection.title != "PRE-ATT&CK":
tc = Collection(
'https://cti-taxii.mitre.org/stix/collections/' +
collection.id)
self.collections[collection.title.split(' ')
[0].lower()] = TAXIICollectionSource(tc)
elif source.lower() == 'local':
if resource is not None:
hd = MemoryStore()
hd.load_from_file(resource)
if 'mobile' in resource.lower():
self.collections['mobile'] = hd
else:
self.collections['enterprise'] = hd
else:
print(
'[MatrixGen] - "local" source specified, but path to local source not provided'
)
raise ValueError
elif source.lower() == 'remote':
if resource is not None:
if ':' not in resource[6:]:
print(
'[MatrixGen] - "remote" source missing port; assuming ":3000"'
)
resource += ":3000"
if not resource.startswith('http'):
resource = 'http://' + resource
for dataset in ['enterprise', 'mobile']:
hd = MemoryStore()
response = requests.get(
f"{resource}/api/stix-bundles?domain={dataset}-"
f"attack&includeRevoked=true&includeDeprecated=true")
response.raise_for_status(
) # ensure we notice bad responses
_add(hd, json.loads(response.text), True, None)
self.collections[dataset] = hd
else:
print(
f'[MatrixGen] - WARNING: "remote" selected without providing a "resource" url. The use of '
f'"remote" requires the inclusion of a "resource" url to an ATT&CK Workbench instance. No matrix '
f'will be generated...')
self.matrix = {}
self._build_matrix()
def execute(self, quals, columns):
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
tc_source = TAXIICollectionSource(collection)
conn_string = _conn_string
query = "SELECT id,name FROM mitre_intrusion_set"
try:
conn = ag.connect(conn_string)
cur = conn.cursor()
cur.execute(query)
while True:
records = cur.fetchall()
if not records:
break
for i in range(0, len(records)):
line = dict()
intrusion_set_id = records[i][0]
intrusion_set_name = records[i][1]
result = self.get_attack_pattern(tc_source,
intrusion_set_id)
for j in range(0, len(result)):
tmp_attack_pattern = dict(result[j])
for column_name in self.columns:
if (column_name == 'external_references'):
ext_ref = tmp_attack_pattern[column_name]
ref_list = list()
for index in range(0, len(ext_ref)):
if ('url' in ext_ref[index]):
ref_list.append(ext_ref[index]['url'])
line[column_name] = ref_list
elif (column_name == 'kill_chain_phases'):
kc_phase = tmp_attack_pattern[column_name]
line[column_name] = kc_phase[0]['phase_name']
elif (column_name == 'platforms'):
if ('x_mitre_platforms' in tmp_attack_pattern):
line[column_name] = tmp_attack_pattern[
'x_mitre_platforms']
elif (column_name == 'id'):
if ('id' in tmp_attack_pattern):
line[column_name] = tmp_attack_pattern[
column_name]
elif (column_name == 'name'):
if ('name' in tmp_attack_pattern):
line[column_name] = tmp_attack_pattern[
column_name]
elif (column_name == 'gid'):
line[column_name] = intrusion_set_id
elif (column_name == 'gname'):
line[column_name] = intrusion_set_name
else:
line[column_name] = tmp_attack_pattern[
column_name]
yield line
except Exception, e:
log_to_postgres(e)
#######################
# Getting data
#######################
# import the stix2 library and the taxii2client libraries
from stix2 import TAXIICollectionSource, Filter
from taxii2client import Collection
# establish TAXII2 Collection instance
collection = Collection("https://limo.anomali.com/api/v1/taxii2/feeds/collections/107/", user="******", password="******")
# supply the TAXII2 collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# build your filter -- kinda seems broken now
# f1 = Filter("type","=", "indicator")
#retrieve the STIX objects
results = tc_source.query()
print("Retrieving...")
print("Got {} results".format(len(results)))
#######################
# Adding Data Back
#######################
# Note that the password below is incorrect. You can find the real password in the Google Doc
collection = Collection('https://ubertaxii.com/scratch/collections/4a26ee4a-924c-4d5e-8519-5d87efe7f6a8/', user='******', password='******')
status = collection.add_objects(str(Bundle(sighting)))
class attack_client(object):
TC_ENTERPRISE_SOURCE = None
TC_PRE_SOURCE = None
TC_MOBILE_SOURCE = None
COMPOSITE_DS = None
def __init__(self, local_path=None):
if local_path is not None and os.path.isdir(os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)):
self.TC_ENTERPRISE_SOURCE = FileSystemSource(
os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR))
self.TC_PRE_SOURCE = FileSystemSource(
os.path.join(local_path, PRE_ATTCK_LOCAL_DIR))
self.TC_MOBILE_SOURCE = FileSystemSource(
os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR))
else:
ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
ENTERPRISE_ATTCK + "/")
PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK +
"/")
MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
MOBILE_ATTCK + "/")
self.TC_ENTERPRISE_SOURCE = TAXIICollectionSource(
ENTERPRISE_COLLECTION)
self.TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION)
self.TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION)
self.COMPOSITE_DS = CompositeDataSource()
self.COMPOSITE_DS.add_data_sources([
self.TC_ENTERPRISE_SOURCE, self.TC_PRE_SOURCE,
self.TC_MOBILE_SOURCE
])
def translate_stix_objects(self, stix_objects):
technique_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"object_marking_refs": "object_marking_refs",
"name": "technique",
"description": "technique_description",
"kill_chain_phases": "tactic",
"x_mitre_detection": "technique_detection",
"x_mitre_platforms": "platform",
"x_mitre_data_sources": "data_sources",
"x_mitre_defense_bypassed": "defense_bypassed",
"x_mitre_permissions_required": "permissions_required",
"x_mitre_effective_permissions": "effective_permissions",
"x_mitre_system_requirements": "system_requirements",
"x_mitre_network_requirements": "network_requirements",
"x_mitre_remote_support": "remote_support",
"x_mitre_contributors": "contributors",
"x_mitre_detectable_by_common_defenses":
"detectable_by_common_defenses",
"x_mitre_detectable_by_common_defenses_explanation":
"detectable_explanation",
"x_mitre_difficulty_for_adversary": "difficulty_for_adversary",
"x_mitre_difficulty_for_adversary_explanation":
"difficulty_explanation",
"x_mitre_tactic_type": "tactic_type",
"x_mitre_impact_type": "impact_type",
"external_references": "external_references"
}
mitigation_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"name": "mitigation",
"description": "mitigation_description",
"external_references": "external_references",
"x_mitre_old_attack_id": "old_mitigation_id"
}
group_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"name": "group",
"description": "group_description",
"aliases": "group_aliases",
"external_references": "external_references",
"x_mitre_contributors": "contributors"
}
software_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"name": "software",
"description": "software_description",
"labels": "software_labels",
"x_mitre_aliases": "software_aliases",
"x_mitre_platforms": "software_platform",
"external_references": "external_references",
"x_mitre_contributors": "contributors",
"x_mitre_old_attack_id": "old_software_id"
}
relationship_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"relationship_type": "relationship",
"description": "relationship_description",
"source_ref": "source_object",
"target_ref": "target_object"
}
tactic_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"object_marking_refs": "object_marking_refs",
"name": "tactic",
"description": "tactic_description",
"x_mitre_shortname": "tactic_shortname",
"external_references": "external_references"
}
matrix_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"modified": "modified",
"object_marking_refs": "object_marking_refs",
"name": "matrix",
"description": "matrix_description",
"tactic_refs": "tactic_references",
"external_references": "external_references"
}
identity_stix_mapping = {
"type": "type",
"id": "id",
"created_by_ref": "created_by_ref",
"created": "created",
"definition_type": "marking_definition_type",
"definition": "marking_definition"
}
marking_stix_mapping = {
"type": "type",
"id": "id",
"created": "created",
"modified": "modified",
"object_marking_refs": "object_marking_refs",
"name": "identity",
"identity_class": "identity_class"
}
# ******** Helper Functions ********
def handle_list(list_object, object_type):
if object_type == "external_references":
obj_dict['url'] = list_object[0]['url']
obj_dict['matrix'] = list_object[0]['source_name']
if obj_dict['type'] == 'attack-pattern':
for ref in list_object:
if ref['source_name'] == 'capec':
obj_dict['capec_id'] = ref['external_id']
obj_dict['capec_url'] = ref['url']
obj_dict['technique_id'] = list_object[0]['external_id']
elif obj_dict['type'] == 'course-of-action':
obj_dict['mitigation_id'] = list_object[0]['external_id']
elif obj_dict['type'] == 'group':
obj_dict['group_id'] = list_object[0]['external_id']
elif obj_dict['type'] == 'software':
obj_dict['software_id'] = list_object[0]['external_id']
elif obj_dict['type'] == 'tactic':
obj_dict['tactic_id'] = list_object[0]['external_id']
elif obj_dict['type'] == 'matrix':
obj_dict['matrix_id'] = list_object[0]['external_id']
elif object_type == "kill_chain_phases":
tactic_list = list()
for phase in list_object:
tactic_list.append(phase['phase_name'])
obj_dict['tactic'] = tactic_list
stix_objects_list = list()
for obj in stix_objects:
if isinstance(obj, dict):
obj_dict = obj
else:
obj_dict = json.loads(
obj.serialize()) # From STIX to Python Dict
dict_keys = list(obj_dict.keys())
for key in dict_keys:
if obj['type'] == "attack-pattern":
stix_mapping = technique_stix_mapping
elif obj['type'] == "course-of-action":
stix_mapping = mitigation_stix_mapping
elif obj['type'] == "intrusion-set":
stix_mapping = group_stix_mapping
elif obj['type'] == "malware" or obj['type'] == "tool":
stix_mapping = software_stix_mapping
elif obj['type'] == "relationship":
stix_mapping = relationship_stix_mapping
elif obj['type'] == "x-mitre-tactic":
stix_mapping = tactic_stix_mapping
elif obj['type'] == "x-mitre-matrix":
stix_mapping = matrix_stix_mapping
elif obj['type'] == "identity":
stix_mapping = identity_stix_mapping
elif obj['type'] == "marking-definition":
stix_mapping = marking_stix_mapping
else:
exit
if key in stix_mapping.keys():
if key == "external_references" or key == "kill_chain_phases":
handle_list(obj_dict[key], key)
else:
new_key = stix_mapping[key]
obj_dict[new_key] = obj_dict.pop(key)
stix_objects_list.append(obj_dict)
return stix_objects_list
def remove_revoked(self, stix_objects, extract=False):
handle_revoked = list()
for obj in stix_objects:
if 'revoked' in obj.keys() and obj['revoked'] == True:
if extract:
handle_revoked.append(obj)
else:
continue
handle_revoked.append(obj)
return handle_revoked
# ******** Enterprise ATT&CK Technology Domain *******
def get_enterprise(self, stix_format=True):
enterprise_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship"),
"tactics": Filter("type", "=", "x-mitre-tactic"),
"matrix": Filter("type", "=", "x-mitre-matrix"),
"identity": Filter("type", "=", "identity"),
"marking-definition": Filter("type", "=", "marking-definition")
}
enterprise_stix_objects = {}
for key in enterprise_filter_objects:
enterprise_stix_objects[key] = (self.TC_ENTERPRISE_SOURCE.query(
enterprise_filter_objects[key]))
if not stix_format:
enterprise_stix_objects[key] = self.translate_stix_objects(
enterprise_stix_objects[key])
return enterprise_stix_objects
def get_enterprise_techniques(self, stix_format=True):
enterprise_techniques = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "attack-pattern"))
if not stix_format:
enterprise_techniques = self.translate_stix_objects(
enterprise_techniques)
return enterprise_techniques
def get_enterprise_mitigations(self, stix_format=True):
enterprise_mitigations = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "course-of-action"))
if not stix_format:
enterprise_mitigations = self.translate_stix_objects(
enterprise_mitigations)
return enterprise_mitigations
def get_enterprise_groups(self, stix_format=True):
enterprise_groups = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "intrusion-set"))
if not stix_format:
enterprise_groups = self.translate_stix_objects(enterprise_groups)
return enterprise_groups
def get_enterprise_malware(self, stix_format=True):
enterprise_malware = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "malware"))
if not stix_format:
enterprise_malware = self.translate_stix_objects(
enterprise_malware)
return enterprise_malware
def get_enterprise_tools(self, stix_format=True):
enterprise_tools = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "tool"))
if not stix_format:
enterprise_tools = self.translate_stix_objects(enterprise_tools)
return enterprise_tools
def get_enterprise_relationships(self, stix_format=True):
enterprise_relationships = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "relationship"))
if not stix_format:
enterprise_relationships = self.translate_stix_objects(
enterprise_relationships)
return enterprise_relationships
def get_enterprise_tactics(self, stix_format=True):
enterprise_tactics = self.TC_ENTERPRISE_SOURCE.query(
Filter("type", "=", "x-mitre-tactic"))
if not stix_format:
enterprise_tactics = self.translate_stix_objects(
enterprise_tactics)
return enterprise_tactics
# ******** Pre ATT&CK Domain *******
def get_pre(self, stix_format=True):
pre_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"),
"relationships": Filter("type", "=", "relationship"),
"tactics": Filter("type", "=", "x-mitre-tactic"),
"matrix": Filter("type", "=", "x-mitre-matrix"),
"identity": Filter("type", "=", "identity"),
"marking-definition": Filter("type", "=", "marking-definition")
}
pre_stix_objects = {}
for key in pre_filter_objects:
pre_stix_objects[key] = self.TC_PRE_SOURCE.query(
pre_filter_objects[key])
if not stix_format:
pre_stix_objects[key] = self.translate_stix_objects(
pre_stix_objects[key])
return pre_stix_objects
def get_pre_techniques(self, stix_format=True):
pre_techniques = self.TC_PRE_SOURCE.query(
Filter("type", "=", "attack-pattern"))
if not stix_format:
pre_techniques = self.translate_stix_objects(pre_techniques)
return pre_techniques
def get_pre_groups(self, stix_format=True):
pre_groups = self.TC_PRE_SOURCE.query(
Filter("type", "=", "intrusion-set"))
if not stix_format:
pre_groups = self.translate_stix_objects(pre_groups)
return pre_groups
def get_pre_relationships(self, stix_format=True):
pre_relationships = self.TC_PRE_SOURCE.query(
Filter("type", "=", "relationship"))
if not stix_format:
pre_relationships = self.translate_stix_objects(pre_relationships)
return pre_relationships
def get_pre_tactics(self, stix_format=True):
pre_tactics = self.TC_PRE_SOURCE.query(
Filter("type", "=", "x-mitre-tactic"))
if not stix_format:
pre_tactics = self.translate_stix_objects(pre_tactics)
return pre_tactics
# ******** Mobile ATT&CK Technology Domain *******
def get_mobile(self, stix_format=True):
mobile_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship"),
"tactics": Filter("type", "=", "x-mitre-tactic"),
"matrix": Filter("type", "=", "x-mitre-matrix"),
"identity": Filter("type", "=", "identity"),
"marking-definition": Filter("type", "=", "marking-definition")
}
mobile_stix_objects = {}
for key in mobile_filter_objects:
mobile_stix_objects[key] = self.TC_MOBILE_SOURCE.query(
mobile_filter_objects[key])
if not stix_format:
mobile_stix_objects[key] = self.translate_stix_objects(
mobile_stix_objects[key])
return mobile_stix_objects
def get_mobile_techniques(self, stix_format=True):
mobile_techniques = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "attack-pattern"))
if not stix_format:
mobile_techniques = self.translate_stix_objects(mobile_techniques)
return mobile_techniques
def get_mobile_mitigations(self, stix_format=True):
mobile_mitigations = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "course-of-action"))
if not stix_format:
mobile_mitigations = self.translate_stix_objects(
mobile_mitigations)
return mobile_mitigations
def get_mobile_groups(self, stix_format=True):
mobile_groups = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "intrusion-set"))
if not stix_format:
mobile_groups = self.translate_stix_objects(mobile_groups)
return mobile_groups
def get_mobile_malware(self, stix_format=True):
mobile_malware = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "malware"))
if not stix_format:
mobile_malware = self.translate_stix_objects(mobile_malware)
return mobile_malware
def get_mobile_tools(self, stix_format=True):
mobile_tools = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "tool"))
if not stix_format:
mobile_tools = self.translate_stix_objects(mobile_tools)
return mobile_tools
def get_mobile_relationships(self, stix_format=True):
mobile_relationships = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "relationship"))
if not stix_format:
mobile_relationships = self.translate_stix_objects(
mobile_relationships)
return mobile_relationships
def get_mobile_tactics(self, stix_format=True):
mobile_tactics = self.TC_MOBILE_SOURCE.query(
Filter("type", "=", "x-mitre-tactic"))
if not stix_format:
mobile_tactics = self.translate_stix_objects(mobile_tactics)
return mobile_tactics
# ******** Get All Functions ********
def get_stix_objects(self, stix_format=True):
enterprise_objects = self.get_enterprise()
pre_objects = self.get_pre()
mobile_objects = self.get_mobile()
for keypre in pre_objects.keys():
for preobj in pre_objects[keypre]:
if keypre in enterprise_objects.keys():
if preobj not in enterprise_objects[keypre]:
enterprise_objects[keypre].append(preobj)
for keymob in mobile_objects.keys():
for mobobj in mobile_objects[keymob]:
if keymob in enterprise_objects.keys():
if mobobj not in enterprise_objects[keymob]:
enterprise_objects[keymob].append(mobobj)
if not stix_format:
for enterkey in enterprise_objects.keys():
enterprise_objects[enterkey] = self.translate_stix_objects(
enterprise_objects[enterkey])
return enterprise_objects
def get_techniques(self, stix_format=True):
all_techniques = self.COMPOSITE_DS.query(
Filter("type", "=", "attack-pattern"))
if not stix_format:
all_techniques = self.translate_stix_objects(all_techniques)
return all_techniques
def get_groups(self, stix_format=True):
all_groups = self.COMPOSITE_DS.query(
Filter("type", "=", "intrusion-set"))
if not stix_format:
all_groups = self.translate_stix_objects(all_groups)
return all_groups
def get_mitigations(self, stix_format=True):
enterprise_mitigations = self.get_enterprise_mitigations()
mobile_mitigations = self.get_mobile_mitigations()
for mm in mobile_mitigations:
if mm not in enterprise_mitigations:
enterprise_mitigations.append(mm)
if not stix_format:
enterprise_mitigations = self.translate_stix_objects(
enterprise_mitigations)
return enterprise_mitigations
def get_software(self, stix_format=True):
enterprise_malware = self.get_enterprise_malware()
enterprise_tools = self.get_enterprise_tools()
mobile_malware = self.get_mobile_malware()
mobile_tools = self.get_mobile_tools()
for mt in mobile_tools:
if mt not in enterprise_tools:
enterprise_tools.append(mt)
for mmal in mobile_malware:
if mmal not in enterprise_malware:
enterprise_malware.append(mmal)
all_software = enterprise_tools + enterprise_malware
if not stix_format:
all_software = self.translate_stix_objects(all_software)
return all_software
def get_relationships(self, stix_format=True):
all_relationships = self.COMPOSITE_DS.query(
Filter("type", "=", "relationship"))
if not stix_format:
all_relationships = self.translate_stix_objects(all_relationships)
return all_relationships
def get_tactics(self, stix_format=True):
all_tactics = self.COMPOSITE_DS.query(
Filter("type", "=", "x-mitre-tactic"))
if not stix_format:
all_tactics = self.translate_stix_objects(all_tactics)
return all_tactics
# ******** Custom Functions ********
def get_technique_by_name(self, name, case=True, stix_format=True):
def test_ds_taxii(collection):
ds = TAXIICollectionSource(collection)
assert ds.collection is not None
def test_get_stix2_object(collection):
tc_sink = TAXIICollectionSource(collection)
objects = tc_sink.get("indicator--d81f86b9-975b-bc0b-775e-810c5ad45a4f")
assert objects
class attack_client(object):
ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + ENTERPRISE_ATTCK + "/")
TC_ENTERPRISE_SOURCE = TAXIICollectionSource(ENTERPRISE_COLLECTION)
PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK + "/")
TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION)
MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + MOBILE_ATTCK + "/")
TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION)
# ******** Helper Functions ********
def parse_stix_objects(self, stix_objects, stix_object_type):
stix_objects_list = list()
if stix_object_type == 'techniques':
for technique in stix_objects:
technique_dict = {
'type': technique['type'],
'id': technique['id'],
'created_by_ref': technique['created_by_ref'],
'created': str(technique['created']),
'modified': str(technique['modified']),
'object_marking_refs': technique['object_marking_refs'],
'url': technique['external_references'][0]['url'],
'matrix': technique['external_references'][0]['source_name'],
'technique': technique['name'],
'technique_description': technique['description'],
'tactic': self.handle_list(technique,'kill_chain_phases'),
'technique_id': technique['external_references'][0]['external_id'],
'platform': self.try_except(technique,'x_mitre_platforms'),
'data_sources': self.try_except(technique,'x_mitre_data_sources'),
'defense_bypassed': self.try_except(technique,'x_mitre_defense_bypassed'),
'permissions_required': self.try_except(technique,'x_mitre_permissions_required'),
'effective_permissions': self.try_except(technique,'x_mitre_effective_permissions'),
'system_requirements': self.try_except(technique,'x_mitre_system_requirements'),
'network_requirements': self.try_except(technique,'x_mitre_network_requirements'),
'remote_support': self.try_except(technique,'x_mitre_remote_support'),
'contributors': self.try_except(technique,'x_mitre_contributors'),
'technique_references': self.try_except(technique,'external_references'),
'detectable_by_common_defenses': self.try_except(technique,'x_mitre_detectable_by_common_defenses'),
'detectable_explanation': self.try_except(technique,'x_mitre_detectable_by_common_defenses_explanation'),
'difficulty_for_adversary': self.try_except(technique,'x_mitre_difficulty_for_adversary'),
'difficulty_explanation': self.try_except(technique,'x_mitre_difficulty_for_adversary_explanation'),
'tactic_type': self.try_except(technique,'x_mitre_tactic_type')
}
stix_objects_list.append(technique_dict)
elif stix_object_type == "mitigations":
for mitigation in stix_objects:
mitigation_dict = {
'type': mitigation['type'],
'id': mitigation['id'],
'created_by_ref': mitigation['created_by_ref'],
'created': str(mitigation['created']),
'modified': str(mitigation['modified']),
'matrix': mitigation['external_references'][0]['source_name'],
'url': mitigation['external_references'][0]['url'],
'mitigation': mitigation['name'],
'mitigation_description': mitigation['description'],
'mitigation_id': mitigation['external_references'][0]['external_id'],
'mitigation_references': self.handle_list(mitigation,'external_references')
}
stix_objects_list.append(mitigation_dict)
elif stix_object_type == "groups":
for group in stix_objects:
group_dict = {
'type': group['type'],
'id': group['id'],
'created_by_ref': self.try_except(group, 'created_by_ref'),
'matrix': group['external_references'][0]['source_name'],
'created': str(group['created']),
'modified': str(group['modified']),
'url': group['external_references'][0]['url'],
'group': group['name'],
'group_description': self.try_except(group, 'description'),
'group_aliases': self.try_except(group, 'aliases'),
'group_id': group['external_references'][0]['external_id'],
'group_references': self.try_except(group,'external_references')
}
stix_objects_list.append(group_dict)
elif stix_object_type == "tools" or stix_object_type == "malware" or stix_object_type == "software":
for software in stix_objects:
software_dict = {
'type': software['type'],
'id': software['id'],
'created_by_ref': software['created_by_ref'],
'created': str(software['created']),
'modified': str(software['modified']),
'matrix': software['external_references'][0]['source_name'],
'software': software['name'],
'software_description': software['description'],
'software_labels': self.try_except(software, 'labels'),
'software_id': software['external_references'][0]['external_id'],
'url': software['external_references'][0]['url'],
'software_aliases': self.try_except(software, 'x_mitre_aliases'),
'software_references': self.try_except(software,'external_references')
}
stix_objects_list.append(software_dict)
elif stix_object_type == "relationships":
for relationship in stix_objects:
relationship_dict = {
'type': relationship['type'],
'id': relationship['id'],
'created_by_ref': relationship['created_by_ref'],
'created': str(relationship['created']),
'modified': str(relationship['modified']),
'relationship': relationship['relationship_type'],
'relationship_description': self.try_except(relationship, 'description'),
'source_object': relationship['source_ref'],
'target_object': relationship['target_ref']
}
stix_objects_list.append(relationship_dict)
else:
exit
return stix_objects_list
def handle_list(self, stix_objects, object_type):
objects_list = list()
if object_type == 'kill_chain_phases':
for o in stix_objects[object_type]:
objects_list.append(o.phase_name)
return objects_list
elif object_type == 'external_references':
for o in stix_objects[object_type]:
if "url" in o:
objects_list.append(o.url)
else:
objects_list.append(o.source_name)
return objects_list
else:
for o in stix_objects[object_type]:
objects_list.append(o)
return objects_list
def try_except(self, stix_objects, object_type):
if object_type in stix_objects:
specific_stix_object = stix_objects[object_type]
if isinstance(specific_stix_object, list):
lists = self.handle_list(stix_objects, object_type)
return lists
else:
return stix_objects[object_type]
else:
return None
# ******** Get All Functions ********
def get_all_stix_objects(self):
techniques_pre_keys = {"techniques","groups","relationships"}
techniques_mobile_keys = {"techniques","mitigations","groups","malware","tools","relationships"}
enterprise_objects = self.get_all_enterprise()
pre_objects = self.get_all_pre()
mobile_objects = self.get_all_mobile()
for key in techniques_pre_keys:
for pre in pre_objects[key]:
if pre not in enterprise_objects[key]:
enterprise_objects[key].append(pre)
for key in techniques_mobile_keys:
for m in mobile_objects[key]:
if m not in enterprise_objects[key]:
enterprise_objects[key].append(m)
return enterprise_objects
def get_all_attack(self):
techniques = self.get_all_techniques_with_mitigations()
software = self.get_all_software()
software_use = self.get_techniques_used_by_software()
groups = self.get_all_groups()
groups_use = self.get_all_used_by_group()
for s in software:
del s['type'],s['id'],s['created_by_ref'],s['created'],s['modified']
for g in groups:
del g['type'],g['id'],g['created_by_ref'],g['created'],g['modified']
all_attack = techniques + software + software_use + groups + groups_use
return all_attack
def get_all_enterprise(self):
enterprise_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
enterprise_stix_objects = {}
for key in enterprise_filter_objects:
enterprise_stix_objects[key] = self.TC_ENTERPRISE_SOURCE.query(enterprise_filter_objects[key])
enterprise_stix_objects[key] = self.parse_stix_objects(enterprise_stix_objects[key], key)
return enterprise_stix_objects
def get_all_pre(self):
pre_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"),
"relationships": Filter("type", "=", "relationship")
}
pre_stix_objects = {}
for key in pre_filter_objects:
pre_stix_objects[key] = self.TC_PRE_SOURCE.query(pre_filter_objects[key])
pre_stix_objects[key] = self.parse_stix_objects(pre_stix_objects[key], key)
return pre_stix_objects
def get_all_mobile(self):
mobile_filter_objects = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
mobile_stix_objects = {}
for key in mobile_filter_objects:
mobile_stix_objects[key] = self.TC_MOBILE_SOURCE.query(mobile_filter_objects[key])
mobile_stix_objects[key] = self.parse_stix_objects(mobile_stix_objects[key], key)
return mobile_stix_objects
def get_all_techniques(self):
enterprise_techniques = self.get_all_enterprise_techniques()
pre_techniques = self.get_all_pre_techniques()
mobile_techniques = self.get_all_mobile_techniques()
all_techniques = enterprise_techniques + pre_techniques + mobile_techniques
return all_techniques
def get_all_groups(self):
enterprise_groups = self.get_all_enterprise_groups()
pre_groups = self.get_all_pre_groups()
mobile_groups = self.get_all_mobile_groups()
for pg in pre_groups:
if pg not in enterprise_groups:
enterprise_groups.append(pg)
for mg in mobile_groups:
if mg not in enterprise_groups:
enterprise_groups.append(mg)
return enterprise_groups
def get_all_mitigations(self):
enterprise_mitigations = self.get_all_enterprise_mitigations()
mobile_mitigations = self.get_all_mobile_mitigations()
for mm in mobile_mitigations:
if mm not in enterprise_mitigations:
enterprise_mitigations.append(mm)
return enterprise_mitigations
def get_all_software(self):
enterprise_malware = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "malware"))
enterprise_tools = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "tool"))
mobile_malware = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "malware"))
mobile_tools = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "tool"))
for mt in mobile_tools:
if mt not in enterprise_tools:
enterprise_tools.append(mt)
for mmal in mobile_malware:
if mmal not in enterprise_malware:
enterprise_malware.append(mmal)
all_software = enterprise_tools + enterprise_malware
all_software = self.parse_stix_objects(all_software, 'software')
return all_software
def get_all_relationships(self):
enterprise_relationships = self.get_all_enterprise_relationships()
pre_relationships = self.get_all_pre_relationships()
mobile_relationships = self.get_all_mobile_relationships()
for pr in pre_relationships:
if pr not in enterprise_relationships:
enterprise_relationships.append(pr)
for mr in mobile_relationships:
if mr not in enterprise_relationships:
enterprise_relationships.append(mr)
return enterprise_relationships
def get_all_techniques_with_mitigations(self):
all_mitigations_mitigate = []
technique_ids = []
all_mitigations_relationships = self.get_relationships_by_object('mitigations')
techniques = self.get_all_techniques()
for mr in all_mitigations_relationships:
for t in techniques:
if t['id'] == mr['target_object']:
all_mitigations_dict = {
'matrix': t['matrix'],
'mitigation': mr['mitigation'],
'mitigation_description': mr['mitigation_description'],
'mitigation_id' :mr['mitigation_id'],
'mitigation_references': mr['mitigation_references'],
'technique' : t['technique'],
'technique_description' : t['technique_description'],
'tactic' : t['tactic'],
'url' : t['url'],
'technique_id' : t['technique_id'],
'platform' : t['platform'],
'data_sources' : t['data_sources'],
'defense_bypassed' : t['defense_bypassed'],
'permissions_required' : t['permissions_required'],
'effective_permissions' : t['effective_permissions'],
'system_requirements' : t['system_requirements'],
'network_requirements' : t['network_requirements'],
'remote_support' : t['remote_support'],
'contributors' : t['contributors'],
'technique_references' : t['technique_references'],
'detectable_by_common_defenses' : t['detectable_by_common_defenses'],
'detectable_explanation' : t['detectable_explanation'],
'difficulty_for_adversary' : t['difficulty_for_adversary'],
'difficulty_explanation': t['difficulty_explanation'],
'tactic_type' : t['tactic_type']
}
all_mitigations_mitigate.append(all_mitigations_dict)
technique_ids.append(t['technique_id'])
for t in techniques:
if t['technique_id'] not in technique_ids:
all_techniques_dict = {
'matrix': t['matrix'],
'technique' : t['technique'],
'technique_description' : t['technique_description'],
'tactic' : t['tactic'],
'url' : t['url'],
'technique_id' : t['technique_id'],
'platform' : t['platform'],
'data_sources' : t['data_sources'],
'defense_bypassed' : t['defense_bypassed'],
'permissions_required' : t['permissions_required'],
'effective_permissions' : t['effective_permissions'],
'system_requirements' : t['system_requirements'],
'network_requirements' : t['network_requirements'],
'remote_support' : t['remote_support'],
'contributors' : t['contributors'],
'technique_references' : t['technique_references'],
'detectable_by_common_defenses' : t['detectable_by_common_defenses'],
'detectable_explanation' : t['detectable_explanation'],
'difficulty_for_adversary' : t['difficulty_for_adversary'],
'difficulty_explanation': t['difficulty_explanation'],
'tactic_type' : t['tactic_type']
}
all_mitigations_mitigate.append(all_techniques_dict)
return all_mitigations_mitigate
# ******** Enterprise Matrix Functions ********
def get_all_enterprise_techniques(self):
enterprise_techniques = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "attack-pattern"))
enterprise_techniques = self.parse_stix_objects(enterprise_techniques, 'techniques')
return enterprise_techniques
def get_all_enterprise_groups(self):
enterprise_groups = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "intrusion-set"))
enterprise_groups = self.parse_stix_objects(enterprise_groups, 'groups')
return enterprise_groups
def get_all_enterprise_mitigations(self):
enterprise_mitigations = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "course-of-action"))
enterprise_mitigations = self.parse_stix_objects(enterprise_mitigations, 'mitigations')
return enterprise_mitigations
def get_all_enterprise_relationships(self):
enterprise_relationships = self.TC_ENTERPRISE_SOURCE.query(Filter("type", "=", "relationship"))
enterprise_relationships = self.parse_stix_objects(enterprise_relationships, 'relationships')
return enterprise_relationships
# ******** Pre Matrix Functions ********
def get_all_pre_techniques(self):
pre_techniques = self.TC_PRE_SOURCE.query(Filter("type", "=", "attack-pattern"))
pre_techniques = self.parse_stix_objects(pre_techniques, 'techniques')
return pre_techniques
def get_all_pre_groups(self):
pre_groups = self.TC_PRE_SOURCE.query(Filter("type", "=", "intrusion-set"))
pre_groups = self.parse_stix_objects(pre_groups, 'groups')
return pre_groups
def get_all_pre_relationships(self):
pre_relationships = self.TC_PRE_SOURCE.query(Filter("type", "=", "relationship"))
pre_relationships = self.parse_stix_objects(pre_relationships, 'relationships')
return pre_relationships
# ******** Mobile Matrix Functions ********
def get_all_mobile_techniques(self):
mobile_techniques = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "attack-pattern"))
mobile_techniques = self.parse_stix_objects(mobile_techniques, 'techniques')
return mobile_techniques
def get_all_mobile_groups(self):
mobile_groups = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "intrusion-set"))
mobile_groups = self.parse_stix_objects(mobile_groups, 'groups')
return mobile_groups
def get_all_mobile_mitigations(self):
mobile_mitigations = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "course-of-action"))
mobile_mitigations = self.parse_stix_objects(mobile_mitigations, 'mitigations')
return mobile_mitigations
def get_all_mobile_relationships(self):
mobile_relationships = self.TC_MOBILE_SOURCE.query(Filter("type", "=", "relationship"))
mobile_relationships = self.parse_stix_objects(mobile_relationships, 'relationships')
return mobile_relationships
# ******** Custom Functions ********
def get_technique_by_name(self, name):
filter_objects = [
Filter('type', '=', 'attack-pattern'),
Filter('name', '=', name)
]
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects)
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
all_stix_objects = self.parse_stix_objects(all_stix_objects, "techniques")
for o in all_stix_objects:
return o
def get_object_by_attack_id(self, object_type, attack_id):
valid_objects = {'attack-pattern','course-of-action','intrusion-set','malware','tool'}
if object_type not in valid_objects:
raise ValueError("ERROR: Valid object must be one of %r" % valid_objects)
else:
dictionary = {
"attack-pattern": "techniques",
"course-of-action": "mitigations",
"intrusion-set": "groups",
"malware": "malware",
"tool": "tools"
}
filter_objects = [
Filter('type', '=', object_type),
Filter('external_references.external_id', '=', attack_id)
]
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects)
enterprise_stix_objects = self.parse_stix_objects(enterprise_stix_objects, dictionary[object_type])
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
pre_stix_objects = self.parse_stix_objects(pre_stix_objects, dictionary[object_type])
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
mobile_stix_objects = self.parse_stix_objects(mobile_stix_objects, dictionary[object_type])
all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
for o in all_stix_objects:
return o
def get_group_by_alias(self, group_alias):
filter_objects = [
Filter('type', '=', 'intrusion-set'),
Filter('aliases', '=', group_alias)
]
enterprise_stix_objects = self.TC_ENTERPRISE_SOURCE.query(filter_objects)
pre_stix_objects = self.TC_PRE_SOURCE.query(filter_objects)
mobile_stix_objects = self.TC_MOBILE_SOURCE.query(filter_objects)
all_stix_objects = enterprise_stix_objects + pre_stix_objects + mobile_stix_objects
all_stix_objects = self.parse_stix_objects(all_stix_objects, 'groups')
for o in all_stix_objects:
return o
def get_relationships_by_object(self, stix_object):
valid_objects = {'groups','software','mitigations'}
all_relationships = []
relationships = self.get_all_relationships()
if stix_object not in valid_objects:
raise ValueError("ERROR: Valid object must be one of %r" % valid_objects)
else:
if stix_object.lower() == 'groups':
groups = self.get_all_groups()
for g in groups:
for r in relationships:
if g['id'] == r['source_object'] and r['relationship'] == 'uses':
all_groups_relationships_dict = {
'target_object' : r['target_object'],
'relationship_description' : r['relationship_description'],
'matrix': g['matrix'],
'url': g['url'],
'group': g['group'],
'group_description': g['group_description'],
'group_aliases': g['group_aliases'],
'group_id': g['group_id'],
'group_references': g['group_references']
}
all_relationships.append(all_groups_relationships_dict)
elif stix_object.lower() == 'software':
software = self.get_all_software()
relationships = self.get_all_relationships()
for s in software:
for r in relationships:
if s['id'] == r['source_object'] and r['relationship'] == 'uses':
all_software_relationships_dict = {
'target_object' : r['target_object'],
'relationship_description' : r['relationship_description'],
'software_type': s['type'],
'matrix': s['matrix'],
'software': s['software'],
'software_description': s['software_description'],
'software_labels':s['software_labels'],
'software_id': s['software_id'],
'url': s['url'],
'software_aliases': s['software_aliases'],
'software_references': s['software_references']
}
all_relationships.append(all_software_relationships_dict)
else:
mitigations = self.get_all_mitigations()
relationships = self.get_all_relationships()
for m in mitigations:
for r in relationships:
if m['id'] == r['source_object'] and r['relationship'] == 'mitigates':
all_mitigations_relationships_dict = {
'target_object' : r['target_object'],
'matrix': m['matrix'],
'mitigation': m['mitigation'],
'mitigation_description': m['mitigation_description'],
'mitigation_id': m['mitigation_id'],
'mitigation_references': m['mitigation_references']
}
all_relationships.append(all_mitigations_relationships_dict)
return all_relationships
def get_all_data_sources(self):
techniques = self.get_all_techniques()
data_sources = []
for t in techniques:
for ds in t['data_sources'] or []:
data_sources.append(ds.lower())
return list(set(data_sources))
def get_techniques_used_by_software(self, software_name=None):
all_software_use = []
all_techniques_used = []
all_software_relationships = self.get_relationships_by_object('software')
techniques = self.get_all_techniques()
for sr in all_software_relationships:
for t in techniques:
if t['id'] == sr['target_object']:
all_software_use_dict = {
'matrix': t['matrix'],
'relationship_description': sr['relationship_description'],
'software': sr['software'],
'software_description': sr['software_description'],
'software_labels':sr['software_labels'],
'software_id': sr['software_id'],
'software_aliases': sr['software_aliases'],
'software_references': sr['software_references'],
'technique' : t['technique'],
'technique_description' : t['technique_description'],
'tactic' : t['tactic'],
'technique_id' : t['technique_id'],
'url' : t['url'],
#'platform' : t['platform'],
#'data_sources' : t['data_sources'],
#'defense_bypassed' : t['defense_bypassed'],
#'permissions_required' : t['permissions_required'],
#'effective_permissions' : t['effective_permissions'],
#'system_requirements' : t['system_requirements'],
#'network_requirements' : t['network_requirements'],
#'remote_support' : t['remote_support'],
'contributors' : t['contributors'],
#'technique_references' : t['technique_references'],
#'detectable_by_common_defenses' : t['detectable_by_common_defenses'],
#'detectable_explanation' : t['detectable_explanation'],
#'difficulty_for_adversary' : t['difficulty_for_adversary'],
#'difficulty_explanation': t['difficulty_explanation'],
#'tactic_type' : t['tactic_type']
}
all_software_use.append(all_software_use_dict)
if software_name is None:
return all_software_use
else:
for sn in all_software_use:
if software_name.lower() in sn['software'].lower():
all_techniques_used.append(sn)
return all_techniques_used
def get_techniques_used_by_group(self, group_name=None):
all_groups_use = []
all_techniques_used = []
all_groups_relationships = self.get_relationships_by_object('groups')
techniques = self.get_all_techniques()
for gr in all_groups_relationships:
for t in techniques:
if t['id'] == gr['target_object']:
all_groups_use_dict = {
'matrix': t['matrix'],
'relationship_description': gr['relationship_description'],
'group': gr['group'],
'group_description': gr['group_description'],
'group_aliases': gr['group_aliases'],
'group_id': gr['group_id'],
'group_references': gr['group_references'],
'technique' : t['technique'],
'technique_description' : t['technique_description'],
'tactic' : t['tactic'],
'technique_id' : t['technique_id'],
'url' : t['url'],
#'platform' : t['platform'],
#'data_sources' : t['data_sources'],
#'defense_bypassed' : t['defense_bypassed'],
#'permissions_required' : t['permissions_required'],
#'effective_permissions' : t['effective_permissions'],
#'system_requirements' : t['system_requirements'],
#'network_requirements' : t['network_requirements'],
#'remote_support' : t['remote_support'],
'contributors' : t['contributors'],
#'technique_references' : t['technique_references'],
#'detectable_by_common_defenses' : t['detectable_by_common_defenses'],
#'detectable_explanation' : t['detectable_explanation'],
#'difficulty_for_adversary' : t['difficulty_for_adversary'],
#'difficulty_explanation': t['difficulty_explanation'],
#'tactic_type' : t['tactic_type']
}
all_groups_use.append(all_groups_use_dict)
if group_name is None:
return all_groups_use
else:
for gn in all_groups_use:
if group_name.lower() in gn['group'].lower():
all_techniques_used.append(gn)
return all_techniques_used
def get_software_used_by_group(self, group_name=None):
all_groups_software_use =[]
all_groups_use = []
all_software_used = []
all_groups_relationships = self.get_relationships_by_object('groups')
software_techniques = self.get_techniques_used_by_software()
software = self.get_all_software()
for gr in all_groups_relationships:
for s in software:
if s['id'] == gr['target_object']:
all_groups_software = {
'matrix': s['matrix'],
'relationship_description': gr['relationship_description'],
'group': gr['group'],
'group_description': gr['group_description'],
'group_aliases': gr['group_aliases'],
'group_id': gr['group_id'],
'group_references': gr['group_references'],
'software_url': s['url'],
'software': s['software'],
'software_description': s['software_description'],
'software_labels':s['software_labels'],
'software_id': s['software_id'],
'software_aliases': s['software_aliases'],
'software_references': s['software_references']
}
all_groups_software_use.append(all_groups_software)
for st in software_techniques:
for gs in all_groups_software_use:
if gs['software_id'] == st['software_id']:
all_software_technique = {
'matrix': gs['matrix'],
'relationship_description': gs['relationship_description'],
'group': gs['group'],
'group_description': gs['group_description'],
'group_aliases': gs['group_aliases'],
'group_id': gs['group_id'],
'group_references': gs['group_references'],
'software url': gs['software_url'],
'software': gs['software'],
'software_description': gs['software_description'],
'software_labels':gs['software_labels'],
'software_id': gs['software_id'],
'software_aliases': gs['software_aliases'],
'software_references': gs['software_references'],
'technique' : st['technique'],
'technique_description' : st['technique_description'],
'tactic' : st['tactic'],
'technique_id' : st['technique_id']
}
all_groups_use.append(all_software_technique)
if group_name is None:
return all_groups_use
else:
for gn in all_groups_use:
if group_name.lower() in gn['group'].lower():
all_software_used.append(gn)
return all_software_used
def get_all_used_by_group(self, group_name=None):
all_used = []
if group_name is None:
software = self.get_software_used_by_group()
techniques = self.get_techniques_used_by_group()
all_used = software + techniques
else:
software = self.get_software_used_by_group(group_name)
techniques = self.get_techniques_used_by_group(group_name)
all_used = software + techniques
return all_used
from stix2 import Filter, TAXIICollectionSource
from taxii2client import Collection
import argparse
import csv
#Connect to TAXII server
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
fs = TAXIICollectionSource(collection)
techniques_data_sources = {}
#Get a group by its name
def get_group_by_alias(src, alias):
return src.query(
[Filter('type', '=', 'intrusion-set'),
Filter('aliases', '=', alias)])
#Return group's techniques
def get_technique_by_group(src, stix_id):
relations = src.relationships(stix_id, 'uses', source_only=True)
return src.query([
Filter('type', '=', 'attack-pattern'),
Filter('id', 'in', [r.target_ref for r in relations])
])
#Return all known threat actors
def get_collections(api_roots):
for api_root in api_roots:
collections = api_root.collections
for c in collections:
print("\tCOLLECTION:", c.title)
c_info = collection_info(c)
tc_source = TAXIICollectionSource(c)
add_data_source(tc_source)
stix_objects = check_stix_object_type()
print("\t\tDATA =>", list(stix_objects.keys()), end='\n')
# SRO
_objects = c.get_objects()
objects = _objects['objects']
for o in objects:
if o['type'] == 'relationship':
sql = """
INSERT IGNORE INTO sro_relationship(source_ref, target_ref, relationship_type)
values (%s, %s, %s)
"""
cur.execute(sql, (o['source_ref'], o['target_ref'],
o['relationship_type']))
db.commit()
# SDO
for stix_object_type in stix_objects:
print("\t\t\t", stix_object_type,
len(stix_objects[stix_object_type]))
for o in stix_objects[stix_object_type]:
TYPE = str(type(o))
#
cols = []
table = "sdo_"
if "AttackPattern" in TYPE:
cols = db_format['attack_pattern']
table += 'attack_pattern'
elif "Campaign" in TYPE:
cols = db_format['campaign']
table += 'campaign'
elif "CourseOfAction" in TYPE:
cols = db_format['course_of_action']
table += 'course_of_action'
elif "Identity" in TYPE:
cols = db_format['identity']
table += 'identity'
elif "Indicator" in TYPE:
cols = db_format['indicator']
table += 'indicator'
elif "IntrusionSet" in TYPE:
cols = db_format['intrusion_set']
table += 'intrusion_set'
elif "Malware" in TYPE:
cols = db_format['malware']
table += 'malware'
elif "ObservedData" in TYPE:
cols = db_format['observed_data']
table += 'observed_data'
elif "Report" in TYPE:
cols = db_format['report']
table += 'report'
elif "ThreatActor" in TYPE:
cols = db_format['threat_actor']
table += 'threat_actor'
elif "Tool" in TYPE:
cols = db_format['tool']
table += 'tool'
elif "Vulnerability" in TYPE:
cols = db_format['vulnerability']
table += 'vulnerability'
else:
continue
sql = "INSERT IGNORE INTO %s (%s) VALUES(%s)" % (
table, ", ".join(cols + db_format['common']),
", ".join([
"\"" +
pymysql.escape_string(str(o.get(c, "None"))) + "\""
for c in cols + db_format['common']
]))
cur.execute(sql)
db.commit()
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--matrix",
type=str,
action="store",
default="enterprise",
help="Matrix to query (enterprise, mobile, pre)")
# Techniques
parser.add_argument(
"--dump-all-techniques",
action="store_true",
help="Dump a CSV file with technique,subtechnique,name")
# Data Sources
parser.add_argument(
"--dump-data-sources",
action="store_true",
help="Dump a list of unique data sources to data_sources.txt")
parser.add_argument(
"--dump-metadata",
action="store_true",
help=
"Dump a CSV file technique-metadata.csv, containing unique technique-metadata pairings."
)
parser.add_argument(
"--dump-matching-techniques",
action="store_true",
help=
"Dump techniques that map to match-data-sources to matching-techniques.txt"
)
parser.add_argument(
"--match-data-sources",
type=str,
action="store",
help=
"A file containing a list of data sources that to match against techniques."
)
args = parser.parse_args()
match_data_sources = None
if args.match_data_sources:
match_data_sources = parse_data_source_list(args.match_data_sources)
args.matrix = args.matrix.lower()
if args.matrix == 'pre':
matrix = "062767bd-02d2-4b72-84ba-56caef0f8658"
elif args.matrix == 'mobile':
matrix = "2f669986-b40b-4423-b720-4396ca6a462b"
elif args.matrix == 'enterprise':
matrix = "95ecc380-afe9-11e4-9b6c-751b66dd541e"
# Initialize dictionary to hold Enterprise ATT&CK content
attack = {}
# Establish TAXII2 Collection instance for Enterprise ATT&CK
collection = Collection("https://cti-taxii.mitre.org/stix/collections/{0}/"\
.format(matrix))
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Create filters to retrieve content from Enterprise ATT&CK
filter_objs = {"techniques": Filter("type", "=", "attack-pattern")}
# Retrieve all Enterprise ATT&CK content
for key in filter_objs:
attack[key] = tc_source.query(filter_objs[key])
all_techniques = attack["techniques"]
all_techniques = remove_revoked_deprecated(all_techniques)
technique_count = 0
techniques_without_data_source = 0
techniques_observable = 0
techniques_with_data_sources = []
data_sources = set()
matching_techniques = set()
for technique in all_techniques:
technique_count += 1
technique_id = technique['external_references'][0]['external_id']
if 'x_mitre_data_sources' in technique.keys():
if match_data_sources is not None:
if data_source_match(technique['x_mitre_data_sources'],
match_list=match_data_sources) == True:
techniques_observable += 1
if args.dump_matching_techniques == True:
matching_techniques.add(technique_id)
if args.dump_data_sources == True:
[
data_sources.add(data_source)
for data_source in technique['x_mitre_data_sources']
]
if args.dump_metadata == True:
[
techniques_with_data_sources.append(
(technique_id, data_source))
for data_source in technique['x_mitre_data_sources']
]
else:
techniques_without_data_source += 1
# Output files based on input arguments.
if match_data_sources is not None:
print('Techniques: {0}'.format(technique_count))
print('Techniques Observable: {0} ({1}%)'\
.format(techniques_observable,
round((techniques_observable / technique_count) * 100)))
if args.dump_data_sources == True:
with open('data_sources.txt', 'w') as fh_data_sources:
data_sources = list(data_sources)
data_sources.sort()
for data_source in data_sources:
fh_data_sources.write('{0}\n'.format(data_source))
if args.dump_matching_techniques == True:
with open('matching_techniques.txt', 'w') as fh_matching_techniques:
matching_techniques = list(matching_techniques)
matching_techniques.sort()
for data_source in matching_techniques:
fh_matching_techniques.write('{0}\n'.format(data_source))
if args.dump_metadata == True:
with open('technique_metadata.csv', 'w') as fh_techniques:
csvwriter = csv.writer(fh_techniques, quoting=csv.QUOTE_ALL)
csvwriter.writerow(['id', 'data_source'])
for technique in techniques_with_data_sources:
csvwriter.writerow(technique)
if args.dump_all_techniques == True:
with open('all_techniques.csv', 'w') as fh_all_techniques:
csvwriter = csv.writer(fh_all_techniques, quoting=csv.QUOTE_ALL)
csvwriter.writerow([
'technique_id', 'technique_name', 'technique_url',
'technique_description'
])
for technique in all_techniques:
# Handle techniques that do not have a description (these probably
# should not exist and should be contributed).
try:
description = technique.description
except AttributeError:
description = 'NONE'
csvwriter.writerow([
technique.external_references[0].external_id,
technique.name, technique.external_references[0].url,
description
])
def taxii_import(server_url, collection_url):
if not (server_url or collection_url):
print('Please specify one of --server_url or --collection_url')
exit(-1)
if server_url:
collection_url = _get_collection_url(server_url)
if collection_url:
print(
'Importing data from collection at: {0:s}'.format(collection_url))
collection = Collection(collection_url)
tc_source = TAXIICollectionSource(collection)
object_classes = {
'attack-pattern': attack_pattern.AttackPattern,
'campaign': campaign.Campaign,
'course-of-action': course_of_action.CourseOfAction,
'identity': identity.Identity,
'intrusion-set': intrusion_set.IntrusionSet,
'malware': malware.Malware,
'threat-actor': threat_actor.ThreatActor,
'tool': tool.Tool,
'vulnerability': vulnerability.Vulnerability,
}
all_objects = {}
for name, yeti_class in object_classes.items():
print('Fetching', name)
stats = {
'updated': 0,
'new': 0,
'skipped': 0,
}
try:
for item in tc_source.query(Filter('type', '=', name)):
item_json = json.loads(item.serialize())
obj = yeti_class.get(item.id)
if not obj:
obj = yeti_class(**item).save()
stats['new'] += 1
all_objects[item['id']] = obj
if obj.modified >= item.modified or obj.revoked or obj.equals(
item_json):
stats['skipped'] += 1
elif obj.modified < item.modified:
obj.update(item_json)
stats['updated'] += 1
except requests.exceptions.HTTPError as error:
print(f'HTTPError: {error}')
except datastore.DataSourceError as error:
print(f'DataSourceError: {error}')
print(
f"[{name}] New: {stats['new']}, Updated: {stats['updated']}, "
f"Skipped: {stats['skipped']}")
print('Getting relationships')
stats = 0
taxii_filter = Filter('type', '=', 'relationship')
for relationship in tc_source.query(taxii_filter):
stats += 1
source = all_objects[relationship.source_ref]
target = all_objects[relationship.target_ref]
source.link_to(target,
stix_rel=json.loads(relationship.serialize()))
print('Added {0:d} relationships'.format(stats))
def build_iterator(self, limit: int = -1) -> List:
"""Retrieves all entries from the feed.
Returns:
A list of objects, containing the indicators.
"""
indicators: List[Dict] = list()
mitre_id_list: Set[str] = set()
indicator_values_list: Set[str] = set()
external_refs: Set[str] = set()
counter = 0
# For each collection
for collection in self.collections:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
# Establish TAXII2 Collection instance
collection_url = urljoin(self.base_url,
f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url,
verify=self.verify,
proxies=self.proxies)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection_data)
# Create filters to retrieve content
filter_objs = {
"Technique": {
"name": "attack-pattern",
"filter": Filter("type", "=", "attack-pattern")
},
"Mitigation": {
"name": "course-of-action",
"filter": Filter("type", "=", "course-of-action")
},
"Group": {
"name": "intrusion-set",
"filter": Filter("type", "=", "intrusion-set")
},
"Malware": {
"name": "malware",
"filter": Filter("type", "=", "malware")
},
"Tool": {
"name": "tool",
"filter": Filter("type", "=", "tool")
},
}
# Retrieve content
for concept in filter_objs:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
input_filter = filter_objs[concept]['filter']
try:
mitre_data = tc_source.query(input_filter)
except Exception:
continue
# For each item in the MITRE list, add an indicator to the indicators list
for mitreItem in mitre_data:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
mitre_item_json = json.loads(str(mitreItem))
value = None
# Try and map a friendly name to the value before the real ID
try:
externals = [
x['external_id'] for x in mitre_item_json.get(
'external_references', [])
if x['source_name'] == 'mitre-attack'
and x['external_id']
]
value = externals[0]
except Exception:
value = None
if not value:
value = mitre_item_json.get('x_mitre_old_attack_id',
None)
if not value:
value = mitre_item_json.get('id')
if mitre_item_json.get('id') not in mitre_id_list:
# If the indicator already exists, then append the new data
# to the existing indicator.
if value in indicator_values_list:
# Append data to the original item
original_item = [
x for x in indicators
if x.get('value') == value
][0]
if original_item['rawJSON'].get('id', None):
try:
original_item['rawJSON'][
'id'] += f"\n{mitre_item_json.get('id', '')}"
except Exception:
pass
if original_item['rawJSON'].get('created', None):
try:
original_item['rawJSON'][
'created'] += f"\n{mitre_item_json.get('created', '')}"
except Exception:
pass
if original_item['rawJSON'].get('modified', None):
try:
original_item['rawJSON'][
'modified'] += f"\n{mitre_item_json.get('modified', '')}"
except Exception:
pass
if original_item['rawJSON'].get(
'description', None):
try:
if not original_item['rawJSON'].get(
'description').startswith("###"):
original_item['rawJSON']['description'] = \
f"### {original_item['rawJSON'].get('type')}\n" \
f"{original_item['rawJSON']['description']}"
original_item['rawJSON']['description'] += \
f"\n\n_____\n\n### {mitre_item_json.get('type')}\n" \
f"{mitre_item_json.get('description', '')}"
except Exception:
pass
if original_item['rawJSON'].get(
'external_references', None):
try:
original_item['rawJSON'][
'external_references'].extend(
mitre_item_json.get(
'external_references', []))
except Exception:
pass
if original_item['rawJSON'].get(
'kill_chain_phases', None):
try:
original_item['rawJSON'][
'kill_chain_phases'].extend(
mitre_item_json.get(
'kill_chain_phases', []))
except Exception:
pass
if original_item['rawJSON'].get('aliases', None):
try:
original_item['rawJSON']['aliases'].extend(
mitre_item_json.get('aliases', []))
except Exception:
pass
else:
indicators.append({
"value": value,
"score": self.reputation,
"type": "MITRE ATT&CK",
"rawJSON": mitre_item_json,
})
indicator_values_list.add(value)
counter += 1
mitre_id_list.add(mitre_item_json.get('id'))
# Create a duplicate indicator using the "external_id" from the
# original indicator, if the user has selected "includeAPT" as True
if self.include_apt:
ext_refs = [
x.get('external_id')
for x in mitre_item_json.get(
'external_references')
if x.get('external_id')
and x.get('source_name') != "mitre-attack"
]
for x in ext_refs:
if x not in external_refs:
indicators.append({
"value":
x,
"score":
self.reputation,
"type":
"MITRE ATT&CK",
"rawJSON":
mitre_item_json,
})
external_refs.add(x)
# Finally, map all the fields from the indicator
# rawjson to the fields in the indicator
for indicator in indicators:
indicator['fields'] = dict()
for field, value in mitre_field_mapping.items():
try:
# Try and map the field
value_type = value['type']
value_name = value['name']
if value_type == "list":
indicator['fields'][field] = "\n".join(
indicator['rawJSON'][value_name])
else:
indicator['fields'][field] = indicator['rawJSON'][
value_name]
except KeyError:
# If the field does not exist in the indicator
# then move on
pass
except Exception as err:
demisto.error(f"Error when mapping Mitre Fields - {err}")
return indicators
from stix2 import TAXIICollectionSource, Filter
from taxii2client import Server, Collection
import json
from openpyxl import Workbook
from openpyxl.styles import Font
#enterprise attack source
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
# supply the TAXII2 collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
count = 1
def parse_mitre_item(filter_type):
global count
wb = Workbook()
for item in filter_type:
name = item[2]
print name
category = tc_source.query(item)
#generate list of deduped list of keys
key_list = []
deduped_keys = []
for lst in category:
for k, v in lst.items():
key_list.append(k)
def load_taxii(new=False):
collection = Collection("https://cti-taxii.mitre.org/stix/collections/" + domainToTaxiiCollectionId[domain])
data_store = TAXIICollectionSource(collection)
parse_subtechniques(data_store, new)
return load_datastore(data_store)
# Import needed libraries
from stix2 import TAXIICollectionSource, Filter
from taxii2client.v20 import Server, Collection
import json
import re
from openpyxl import Workbook
from openpyxl.styles import Font
from openpyxl.styles import Alignment
#__________________________________________________________________________________________________________________________
# Enterprise attack source
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
# Supply the TAXII2 collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Filter to only techniques
filt = Filter('type', '=', 'attack-pattern')
techniques = tc_source.query([filt])
#__________________________________________________________________________________________________________________________
# Generate list of all kill chain phases and put them in proper order
kc_list = []
for technique in techniques:
for k, v in technique.items():
if k == "kill_chain_phases":
for i in v:
for k, v in i.items():
kc_list.append(v)
async def insert_attack_stix_data(self):
"""
Function to pull stix/taxii information and insert in to the local db
:return: status code
"""
logging.info('Downloading ATT&CK data from STIX/TAXII...')
attack = {}
collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/")
tc_source = TAXIICollectionSource(collection)
filter_objs = {"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship")}
for key in filter_objs:
attack[key] = tc_source.query(filter_objs[key])
references = {}
# add all of the patterns and dictionary keys/values for each technique and software
for i in attack["techniques"]:
references[i["id"]] = {"name": i["name"], "id": i["external_references"][0]["external_id"],
"example_uses": [],
"description": i['description'].replace('<code>', '').replace('</code>', '').replace(
'\n', '').encode('ascii', 'ignore').decode('ascii'),
"similar_words": [i["name"]]}
for i in attack["relationships"]:
if i["relationship_type"] == 'uses':
if 'attack-pattern' in i["target_ref"]:
use = i["description"]
# remove unnecessary strings, fix unicode errors
use = use.replace('<code>', '').replace('</code>', '').replace('"', "").replace(',', '').replace(
'\t', '').replace(' ', ' ').replace('\n', '').encode('ascii', 'ignore').decode('ascii')
find_pattern = re.compile('\[.*?\]\(.*?\)') # get rid of att&ck reference (name)[link to site]
m = find_pattern.findall(use)
if len(m) > 0:
for j in m:
use = use.replace(j, '')
if use[0:2] == '\'s':
use = use[3:]
elif use[0] == ' ':
use = use[1:]
# combine all the examples to one list
references[i["target_ref"]]["example_uses"].append(use)
for i in attack["malware"]:
if 'description' not in i: # some software do not have description, example: darkmoon https://attack.mitre.org/software/S0209
continue
else:
references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"],
"examples": [], "example_uses": [], "similar_words": [i["name"]]}
for i in attack["tools"]:
references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"], "examples": [],
"example_uses": [], "similar_words": [i["name"]]}
attack_data = references
logging.info("Finished...now creating the database.")
cur_uids = await self.dao.get('attack_uids') if await self.dao.get('attack_uids') else []
cur_items = [i['uid'] for i in cur_uids]
for k, v in attack_data.items():
if k not in cur_items:
await self.dao.insert('attack_uids', dict(uid=k, description=defang_text(v['description']), tid=v['id'],
name=v['name']))
if 'regex_patterns' in v:
[await self.dao.insert('regex_patterns', dict(uid=k, regex_pattern=defang_text(x))) for x in
v['regex_patterns']]
if 'similar_words' in v:
[await self.dao.insert('similar_words', dict(uid=k, similar_word=defang_text(x))) for x in
v['similar_words']]
if 'false_negatives' in v:
[await self.dao.insert('false_negatives', dict(uid=k, false_negative=defang_text(x))) for x in
v['false_negatives']]
if 'false_positives' in v:
[await self.dao.insert('false_positives', dict(uid=k, false_positive=defang_text(x))) for x in
v['false_positives']]
if 'true_positives' in v:
[await self.dao.insert('true_positives', dict(uid=k, true_positive=defang_text(x))) for x in
v['true_positives']]
if 'example_uses' in v:
[await self.dao.insert('true_positives', dict(uid=k, true_positive=defang_text(x))) for x in
v['example_uses']]
logging.info('[!] DB Item Count: {}'.format(len(await self.dao.get('attack_uids'))))
from taxii2client import Server
from stix2 import CustomObject, properties, TAXIICollectionSource
from taxii2client import Collection
from stix2 import Filter
import sys
discovery_url = sys.argv[1]
poll_url = sys.argv[2]
#descovery
server = Server(discovery_url, 'user1', 'Password1')
print(server.title)
#poll
collection = Collection(poll_url, 'user1', 'Password1')
tc_source = TAXIICollectionSource(collection)
f1 = Filter("type", "=", "indicator")
indicators = tc_source.query([f1])
for indicator in indicators:
print(indicator)
from stix2 import TAXIICollectionSource, Filter
from taxii2client.v20 import Collection
import argparse
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
# Supply the collection to TAXIICollection
tc_src = TAXIICollectionSource(collection)
def data_sources():
"""returns all data sources in Enterprise ATT&CK"""
all_data_srcs = []
# Get all techniques in Enterprise ATT&CK
techniques = tc_src.query([Filter("type", "=", "attack-pattern")])
# Get all data sources in Enterprise ATT&CK
for tech in techniques:
if 'x_mitre_data_sources' in tech:
all_data_srcs += [
data_src for data_src in tech.x_mitre_data_sources
if data_src not in all_data_srcs
]
return all_data_srcs
def test_get_stix2_object(collection):
tc_sink = TAXIICollectionSource(collection)
objects = tc_sink.get("indicator--00000000-0000-4000-8000-000000000001")
assert objects
def get_tactic_techniques(self, tactic_name=None, tactic_id=None):
"""
Get all the techniques for a give tactic
Reference:
https://github.com/mitre/cti/blob/master/USAGE.md
:param tactic_name: tactic name
:param tactic_id: tactic ID
:return: techs
"""
if self.attack_server is None:
self.connect_server()
#
# STIX type for technique is attack-pattern
#
tech_filter = Filter("type", "=", "attack-pattern")
#
# Find name if id is given
#
if tactic_name is None:
if tactic_id is not None:
tactic_name = MitreAttackTactic.get_name(tactic_id)
#
# Not found in MITRE document. We actually need to use lower case for the tactic
# name, and also replace space with -.
#
t_name = tactic_name.replace(' ', '-')
t_name = t_name.lower()
tactic_filter = Filter("kill_chain_phases.phase_name", "=", t_name)
#
# Only look for Enterprise ATT&CK at this point
#
collection_title = "Enterprise ATT&CK"
collection = self.collection_dict[collection_title]
tc_source = TAXIICollectionSource(collection)
mitre_techs = tc_source.query([tech_filter, tactic_filter])
#
# The returned AttackPattern is not serializable. Pick the fields we want
#
techs = []
for mitre_tech in mitre_techs:
refs = []
mitre_tech_id = ""
for r in mitre_tech["external_references"]:
ref = {"url": r.get("url", "")}
if r.get("source_name", None) == "mitre-attack":
mitre_tech_id = r.get("external_id", "")
refs.append(ref)
tech = {
"name": mitre_tech.get("name", ""),
"description": mitre_tech.get("description", ""),
"external_references": refs,
"x_mitre_detection": mitre_tech.get("x_mitre_detection", ""),
"mitre_tech_id": mitre_tech_id
}
techs.append(tech)
return techs
def build_iterator(self, create_relationships=False, is_up_to_6_2=True, limit: int = -1):
"""Retrieves all entries from the feed.
Returns:
A list of objects, containing the indicators.
"""
indicators: List[Dict] = list()
mitre_id_list: Set[str] = set()
mitre_relationships_list = []
id_to_name: Dict = {}
mitre_id_to_mitre_name: Dict = {}
counter = 0
# For each collection
for collection in self.collections:
# fetch only enterprise objects
if collection.id != ENTERPRISE_COLLECTION_ID:
continue
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
# Establish TAXII2 Collection instance
collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection_data)
for concept in FILTER_OBJS:
if 0 < limit <= counter:
break
input_filter = FILTER_OBJS[concept]['filter']
try:
mitre_data = tc_source.query(input_filter)
except Exception:
continue
for mitre_item in mitre_data:
if 0 < limit <= counter:
break
mitre_item_json = json.loads(str(mitre_item))
if mitre_item_json.get('id') not in mitre_id_list:
value = mitre_item_json.get('name')
item_type = get_item_type(mitre_item_json.get('type'), is_up_to_6_2)
if item_type == 'Relationship' and create_relationships:
if mitre_item_json.get('relationship_type') == 'revoked-by':
continue
mitre_relationships_list.append(mitre_item_json)
else:
if is_indicator_deprecated_or_revoked(mitre_item_json):
continue
id_to_name[mitre_item_json.get('id')] = value
indicator_obj = self.create_indicator(item_type, value, mitre_item_json)
add_obj_to_mitre_id_to_mitre_name(mitre_id_to_mitre_name, mitre_item_json)
indicators.append(indicator_obj)
counter += 1
mitre_id_list.add(mitre_item_json.get('id'))
return indicators, mitre_relationships_list, id_to_name, mitre_id_to_mitre_name
from stix2 import TAXIICollectionSource, Filter
from taxii2client import Server
server = Server("https://cti-taxii.mitre.org/taxii/")
api_root = server.api_roots[0]
#
# Three collections: Enterprise ATT&CK, PRE-ATT&CK, MOBILE
#
for collection in api_root.collections:
print(collection.title + ": " + collection.id)
#collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/")
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Create filters to retrieve content from Enterprise ATT&CK
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
# filter for tactics
#fltr = Filter("type", "=", "tactic")
#tactics = tc_source.query(fltr)
def cti_create(collection, client):
cti = client['cti']
cve = cti['cve']
cwe = cti['cwe']
capec = cti['capec']
attack_techniques = cti['attack_techniques']
attack_relationships = cti['attack_relationships']
attack_groups = cti['attack_groups']
attack_software = cti['attack_software']
attack_mitigations = cti['attack_mitigations']
misp_threat_actor = cti['misp_threat_actor']
if collection == "CVE":
files = [
f for f in os.listdir(NVD_PATH)
if os.path.isfile(os.path.join(NVD_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(NVD_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
nvd_dict = json.loads(jsonfile.read())
for element in nvd_dict['CVE_Items']:
cve.insert_one(element)
jsonfile.close()
return
elif collection == "CWE":
files = [
f for f in os.listdir(CWE_PATH)
if os.path.isfile(os.path.join(CWE_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(CWE_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
cwe_dict = xmltodict.parse(jsonfile.read())
cwe_list = cwe_dict['Weakness_Catalog']['Weaknesses']['Weakness']
for element in cwe_list:
cwe.insert_one(json.loads(
json.dumps(element).replace("@", "")))
jsonfile.close()
return
elif collection == "CAPEC":
files = [
f for f in os.listdir(CAPEC_PATH)
if os.path.isfile(os.path.join(CAPEC_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(CAPEC_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
capec_dict = xmltodict.parse(jsonfile.read())
capec_list = capec_dict['Attack_Pattern_Catalog'][
'Attack_Patterns']['Attack_Pattern']
for element in capec_list:
capec.insert_one(
json.loads(json.dumps(element).replace("@", "")))
jsonfile.close()
return
elif collection == "ATTACK":
technique_helper = []
software_helper = []
attack_id = {}
for collection in API_ROOT.collections:
attack_id[collection.title] = collection.id
attack_collection = {}
enterprise_attack = Collection(MITRE_STIX +
attack_id["Enterprise ATT&CK"])
pre_attack = Collection(MITRE_STIX + attack_id["PRE-ATT&CK"])
enterprise_attack_source = TAXIICollectionSource(enterprise_attack)
pre_attack_source = TAXIICollectionSource(pre_attack)
teacher = dict((requests.get(ATTACK_TEACHER).json()))
teacher_dict = {}
for technique in teacher['techniques']:
if technique["color"] == RED:
teacher_dict[technique["techniqueID"]] = '5'
elif technique["color"] == ORANGE:
teacher_dict[technique["techniqueID"]] = '4'
elif technique["color"] == YELLOW:
teacher_dict[technique["techniqueID"]] = '3'
elif technique["color"] == GREEN:
teacher_dict[technique["techniqueID"]] = '2'
elif technique["color"] == BLUE:
teacher_dict[technique["techniqueID"]] = '1'
else:
teacher_dict[technique["techniqueID"]] = 'Unknown'
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
for key in filter_objs:
attack_collection[key] = []
try:
attack_collection[key] += enterprise_attack_source.query(
filter_objs[key])
except:
pass
try:
attack_collection[key] += pre_attack_source.query(
filter_objs[key])
except:
pass
for entity in attack_collection["relationships"]:
attack_relationships.insert_one(dict(entity))
for entity in attack_collection["techniques"]:
dict_entity = dict(entity)
for references in dict_entity['external_references']:
if 'attack' in references['source_name']:
attack_id = references['external_id']
if attack_id in teacher_dict.keys():
dict_entity['level'] = teacher_dict[attack_id]
else:
dict_entity['level'] = 'Unknown'
technique_helper.append(dict_entity)
attack_techniques.insert_one(dict_entity)
for entity in attack_collection["malware"]:
dict_entity = dict(entity)
sw_id = dict_entity['id']
techniques = []
measured_techniques = 0
ttp_score_sum = 0
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'attack-pattern' in dict_rel['target_ref']:
techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'attack-pattern' in dict_rel['source_ref']:
techniques.append(dict_rel['source_ref'])
for ttp_id in techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += int(technique['level'])
if measured_techniques == 0:
dict_entity['score'] = 'Unknown'
else:
dict_entity['score'] = round(
ttp_score_sum / measured_techniques, 2)
software_helper.append(dict_entity)
attack_software.insert_one(dict_entity)
for entity in attack_collection["tools"]:
dict_entity = dict(entity)
sw_id = dict_entity['id']
techniques = []
measured_techniques = 0
ttp_score_sum = 0
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'attack-pattern' in dict_rel['target_ref']:
techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'attack-pattern' in dict_rel['source_ref']:
techniques.append(dict_rel['source_ref'])
for ttp_id in techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += int(technique['level'])
if measured_techniques == 0:
dict_entity['score'] = 'Unknown'
else:
dict_entity['score'] = round(
ttp_score_sum / measured_techniques, 2)
software_helper.append(dict_entity)
attack_software.insert_one(dict_entity)
for entity in attack_collection["groups"]:
dict_entity = dict(entity)
gr_id = dict_entity['id']
measured_techniques = 0
ttp_score_sum = 0
used_techniques = []
measured_software = 0
sw_score_sum = 0
used_software = []
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == gr_id:
if 'attack-pattern' in dict_rel['target_ref']:
used_techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == gr_id:
if 'attack-pattern' in dict_rel['source_ref']:
used_techniques.append(dict_rel['source_ref'])
for ttp_id in used_techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += float(technique['level'])
if measured_techniques == 0:
dict_entity['skill_level'] = 'Unknown'
else:
dict_entity['skill_level'] = float(ttp_score_sum /
measured_techniques)
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'malware' in dict_rel[
'target_ref'] or 'tool' in dict_rel['target_ref']:
used_software.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'malware' in dict_rel[
'target_ref'] or 'tool' in dict_rel['target_ref']:
used_software.append(dict_rel['source_ref'])
for sw_id in used_software:
for software in software_helper:
if software[
'id'] == sw_id and software['score'] != "Unknown":
measured_software += 1
sw_score_sum += float(software['score'])
if measured_software == 0:
dict_entity['opportunity'] = 'Unknown'
else:
dict_entity['opportunity'] = float(sw_score_sum /
measured_software)
if (dict_entity['skill_level'] == "Unknown") and (
dict_entity['opportunity'] != 'Unknown'):
dict_entity['skill_level'] = dict_entity['opportunity']
if (dict_entity['opportunity'] == "Unknown") and (
dict_entity['skill_level'] != 'Unknown'):
dict_entity['opportunity'] = dict_entity['skill_level']
if dict_entity['opportunity'] != 'Unknown' and dict_entity[
'skill_level'] != 'Unknown':
dict_entity['skill_level'] = int(
modf(OWASP * dict_entity['skill_level'])[1])
dict_entity['opportunity'] = int(
modf(OWASP * dict_entity['opportunity'])[1])
stix_group_id = dict_entity['id'].split("--")[1]
attack_group_id = None
for external_reference in dict_entity['external_references']:
if 'attack' in external_reference['source_name']:
attack_group_id = external_reference['external_id']
break
threat_actors = []
try:
misp_list = misp_threat_actor.distinct(
'uuid', {"meta.refs": GROUP_URL(attack_group_id)
}) + misp_threat_actor.distinct(
'uuid', {"related.dest-uuid": stix_group_id})
for threat_actor in misp_list:
if threat_actor not in threat_actors:
threat_actors.append(threat_actor)
except:
pass
motive_list = []
size_list = []
for threat_actor in threat_actors:
threat_actor_list = misp_threat_actor.find(
{'uuid': threat_actor})
for threat_actor_dict in threat_actor_list:
motive_list.append(int(threat_actor_dict['motive']))
size_list.append(int(threat_actor_dict['size']))
if len(motive_list) > 0:
dict_entity['motive'] = str(
sum(motive_list) / len(motive_list))
else:
dict_entity['motive'] = '0'
if dict_entity['motive'] == '0':
dict_entity['motive'] = 'Unknown'
if len(size_list) > 0:
dict_entity['size'] = str(sum(size_list) / len(size_list))
else:
dict_entity['size'] = '0'
if dict_entity['size'] == '0':
dict_entity['size'] = 'Unknown'
attack_groups.insert_one(dict_entity)
for entity in attack_collection["mitigations"]:
attack_mitigations.insert_one(dict(entity))
return
elif collection == "MISP":
with open(os.path.join(MISP_PATH, MISP_THREAT_ACTOR),
'r',
encoding="UTF-8") as misp_file:
threat_actors = json.loads(misp_file.read())
for threat_actor in threat_actors['values']:
dict_threat_actor = threat_actor_category(dict(threat_actor))
misp_threat_actor.insert_one(dict_threat_actor)
misp_file.close()
else:
print("Collection type not set")
return
attack_dict = {}
attack_stix_root_url = "https://cti-taxii.mitre.org/stix/collections/"
attack_enterprise_collection_url = attack_stix_root_url + api_root.collections[
0].id + '/'
# Print ATT&CK Enterprise Collection URL
print("ATT&CK Enterprise collection URL: " + attack_enterprise_collection_url)
print(
"------------------------------------------------------------------------")
#Establish TAXII2 Collection instance for Enterprise ATT&CK collection
Enterprise_collection = Collection(attack_enterprise_collection_url)
#Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(Enterprise_collection)
print("TAXII Collection Source :" + str(type(tc_source)))
print(
"------------------------------------------------------------------------")
# Create filters to retrieve content from Enterprise ATT&CK based on type
# filter_objs = {"techniques": Filter("type", "=", "attack-pattern"), # Technique
# "mitigations": Filter("type", "=", "course-of-action"), # Mitigation
# "groups": Filter("type", "=", "intrusion-set"), # Group
# "malware": Filter("type", "=", "malware"), # Software
# "tools": Filter("type", "=", "tool"), # Software
# "relationships": Filter("type", "=", "relationship")
# }
#