def __init__(self, local_path=None):
if local_path is not None and os.path.isdir(os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)):
self.TC_ENTERPRISE_SOURCE = FileSystemSource(
os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR))
self.TC_PRE_SOURCE = FileSystemSource(
os.path.join(local_path, PRE_ATTCK_LOCAL_DIR))
self.TC_MOBILE_SOURCE = FileSystemSource(
os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR))
else:
ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
ENTERPRISE_ATTCK + "/")
PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK +
"/")
MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
MOBILE_ATTCK + "/")
self.TC_ENTERPRISE_SOURCE = TAXIICollectionSource(
ENTERPRISE_COLLECTION)
self.TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION)
self.TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION)
self.COMPOSITE_DS = CompositeDataSource()
self.COMPOSITE_DS.add_data_sources([
self.TC_ENTERPRISE_SOURCE, self.TC_PRE_SOURCE,
self.TC_MOBILE_SOURCE
])
def establish_connection(collection: str):
"""establish a connection with the TAXII server.
arguments:
collection: string, the url of the collection with which to connect.
returns:
connection to the taxii collection
"""
try:
with open('proxy.json', "r") as json_data:
proxies = json.load(json_data)
link = t2c._HTTPConnection()
link.session.proxies.update(proxies)
collection = t2c.Collection(collection, conn=link)
tc_src = TAXIICollectionSource(collection)
except:
if verbose:
print(
"unable to find proxy file, falling back to standard callout... ",
end="",
flush=True)
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection = t2c.Collection(collection)
# Supply the collection to TAXIICollection
tc_src = TAXIICollectionSource(collection)
return tc_src
def execute(self, quals, columns):
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
tc_source = TAXIICollectionSource(collection)
group = self.get_intrusion_set(tc_source)
for index in range(0, len(group)):
intrusion_set = dict(group[index])
line = {}
for column_name in self.columns:
if (column_name == 'external_references'):
ext_ref = intrusion_set[column_name]
ref_list = list()
for index in range(0, len(ext_ref)):
if ('url' in ext_ref[index]):
ref_list.append(ext_ref[index]['url'])
line[column_name] = ref_list
elif (column_name == 'aliases'):
if ('aliases' in intrusion_set):
line[column_name] = intrusion_set[column_name]
elif (column_name == 'description'):
if ('description' in intrusion_set):
line[column_name] = intrusion_set[column_name].replace(
'\n', '').strip()
else:
line[column_name] = intrusion_set[column_name]
yield line
def load_taxii():
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/" +
domainToTaxiiCollectionId[domain])
data_store = TAXIICollectionSource(collection)
return load_datastore(data_store)
def load_taxii(new=False):
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/" +
domainToTaxiiCollectionId[domain])
data_store = TAXIICollectionSource(collection)
parse_subtechniques(data_store, new)
return load_datastore(data_store)
def test_can_read_error(collection_no_rw_access):
"""create a TAXIICOllectionSource with a taxii2client.Collection
instance that does not have read access, check ValueError exception is raised"""
with pytest.raises(DataSourceError) as excinfo:
TAXIICollectionSource(collection_no_rw_access)
assert "Collection object provided does not have read access" in str(excinfo.value)
def retrieve_attack_as_list():
server = Server("https://cti-taxii.mitre.org/taxii/")
api_root = server.api_roots[0]
for collection in api_root.collections:
logging.info(collection.title + ":" + collection.id)
attack = {}
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
tc_source = TAXIICollectionSource(collection)
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
techniques = tc_source.query(filter_objs['techniques'])
all_keys = gather_keys(techniques)
parsed_techniques = []
for technique in techniques:
parsed_technique = flatten_technique(technique, all_keys)
parsed_techniques = parsed_techniques + parsed_technique
return parsed_techniques
def build_iterator(self, create_relationships=False, is_up_to_6_2=True, limit: int = -1):
"""Retrieves all entries from the feed.
Returns:
A list of objects, containing the indicators.
"""
indicators: List[Dict] = list()
mitre_id_list: Set[str] = set()
mitre_relationships_list = []
id_to_name: Dict = {}
counter = 0
# For each collection
for collection in self.collections:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
# Establish TAXII2 Collection instance
collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection_data)
for concept in FILTER_OBJS:
if 0 < limit <= counter:
break
input_filter = FILTER_OBJS[concept]['filter']
try:
mitre_data = tc_source.query(input_filter)
except Exception:
continue
for mitre_item in mitre_data:
if 0 < limit <= counter:
break
mitre_item_json = json.loads(str(mitre_item))
if mitre_item_json.get('id') not in mitre_id_list:
value = mitre_item_json.get('name')
item_type = get_item_type(mitre_item_json.get('type'), is_up_to_6_2)
if item_type == 'Relationship' and create_relationships:
if mitre_item_json.get('relationship_type') == 'revoked-by':
continue
mitre_relationships_list.append(mitre_item_json)
else:
if is_indicator_deprecated_or_revoked(mitre_item_json):
continue
id_to_name[mitre_item_json.get('id')] = value
indicator_obj = self.create_indicator(item_type, value, mitre_item_json)
indicators.append(indicator_obj)
counter += 1
mitre_id_list.add(mitre_item_json.get('id'))
return indicators, mitre_relationships_list, id_to_name
def lookup_item(self,
item_name,
collection_title="Enterprise ATT&CK",
type_name="attack-pattern"):
"""
Look up an item using item name
:param item_name:
:param collection_title:
:param type_name:
:return:
"""
ret_item = None
if self.attack_server is None:
self.connect_server()
try:
collection = self.collection_dict[collection_title]
tc_source = TAXIICollectionSource(collection)
query_filter = Filter("type", "=", type_name)
attack = tc_source.query(query_filter)
for item in attack:
if item["name"] == item_name:
ret_item = item
break
except:
ret_item = None
return ret_item
def test_add_get_remove_filter(collection):
ds = TAXIICollectionSource(collection)
# First 3 filters are valid, remaining properties are erroneous in some way
valid_filters = [
Filter('type', '=', 'malware'),
Filter('id', '!=', 'stix object id'),
Filter('labels', 'in', ["heartbleed", "malicious-activity"]),
]
assert len(ds.filters) == 0
ds.filters.add(valid_filters[0])
assert len(ds.filters) == 1
# Addin the same filter again will have no effect since `filters` acts
# like a set
ds.filters.add(valid_filters[0])
assert len(ds.filters) == 1
ds.filters.add(valid_filters[1])
assert len(ds.filters) == 2
ds.filters.add(valid_filters[2])
assert len(ds.filters) == 3
assert valid_filters == [f for f in ds.filters]
# remove
ds.filters.remove(valid_filters[0])
assert len(ds.filters) == 2
ds.filters.add(valid_filters)
def get_tactics_by_technique_id(self,technique_ids):
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Create filters to retrieve content from Enterprise ATT&CK
filter_objs = {
"techniques": [
Filter('type', '=', "attack-pattern"),
Filter('external_references.external_id', 'in', technique_ids)]
}
for key in filter_objs:
attack[key] = tc_source.query(filter_objs[key])
# For visual purposes, print the first technique received
tactics=[]
response_data={}
description=""
response_data['tactics']={}
if len( attack["techniques"])>0:
for technique in attack["techniques"]:
for tactic in technique["kill_chain_phases"]:
tactics.append(tactic['phase_name'])
description=description+"\n"+technique['description']
response_data['tactics']=tactics
response_data['description']=description
return response_data
def taxii_import(server_url, collection_url):
if not (server_url or collection_url):
print('Please specify one of --server_url or --collection_url')
exit(-1)
if server_url:
collection_url = _get_collection_url(server_url)
if collection_url:
print('Importing data from collection at: {0:s}'.format(collection_url))
collection = Collection(collection_url)
tc_source = TAXIICollectionSource(collection)
all_objects = {}
for name, yeti_class in OBJECT_CLASSES.items():
print('Fetching', name)
stats = {
'updated': 0,
'new': 0,
'skipped': 0,
}
try:
for item in tc_source.query(Filter('type', '=', name)):
item_json = json.loads(item.serialize())
obj = yeti_class.get(item.id)
if not obj:
obj = yeti_class(**item).save()
stats['new'] += 1
elif obj.modified >= item.modified or obj.revoked or obj.equals(item_json):
stats['skipped'] += 1
elif obj.modified < item.modified:
obj.update(item_json)
stats['updated'] += 1
all_objects[item['id']] = obj
except requests.exceptions.HTTPError as error:
print(f'HTTPError: {error}')
except datastore.DataSourceError as error:
print(f'DataSourceError: {error}')
print(f"[{name}] New: {stats['new']}, Updated: {stats['updated']}, "
f"Skipped: {stats['skipped']}")
print('Getting relationships')
stats = 0
taxii_filter = Filter('type', '=', 'relationship')
for relationship in tc_source.query(taxii_filter):
stats += 1
source = _lazy_get_object(all_objects, relationship.source_ref)
target = _lazy_get_object(all_objects, relationship.target_ref)
source.link_to(target, stix_rel=json.loads(
relationship.serialize()))
print('Added {0:d} relationships'.format(stats))
def attnck_taxii():
# Initialize dictionary to hold Enterprise ATT&CK content
ent_attack = {}
pre_attack = {}
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
ENTERPRISE_ATTCK = "95ecc380-afe9-11e4-9b6c-751b66dd541e"
PRE_ATTCK = "062767bd-02d2-4b72-84ba-56caef0f8658"
MOBILE_ATTCK = "2f669986-b40b-4423-b720-4396ca6a462b"
enterprise_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
pre_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/062767bd-02d2-4b72-84ba-56caef0f8658/"
)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(enterprise_attack_collection)
tc_source2 = TAXIICollectionSource(pre_attack_collection)
# Create filters to retrieve content from Enterprise ATT&CK based on type
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
filter_objs_pre = {
"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"),
"relationships": Filter("type", "=", "relationship")
}
# Retrieve all Enterprise ATT&CK content
for key in filter_objs:
ent_attack[key] = tc_source.query(filter_objs[key])
for key in filter_objs_pre:
pre_attack[key] = tc_source2.query(filter_objs[key])
return ent_attack, pre_attack
def ma_get_definitions():
# Load MITRE's Att&ck Enterprise definitions
collection = Collection(MA_ENTERPRISE_TAXII_URL)
tc_source = TAXIICollectionSource(collection)
stix_filter = Filter("type", "=", "attack-pattern")
attack = tc_source.query(stix_filter)
return ma_parse(attack)
def get_mitre_data_by_filter(client, mitre_filter):
for collection in client.collections:
collection_url = urljoin(client.base_url, f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url, verify=client.verify, proxies=client.proxies)
tc_source = TAXIICollectionSource(collection_data)
if tc_source.query(mitre_filter):
mitre_data = tc_source.query(mitre_filter)[0]
return mitre_data
return {}
def get_data(self):
collections = {
"enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e",
"pre_attack": "062767bd-02d2-4b72-84ba-56caef0f8658",
"mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
}
collection = Collection(
f"https://cti-taxii.mitre.org/stix/collections/{collections['enterprise_attack']}/"
)
src = TAXIICollectionSource(collection)
return src
def polling(poll_url):
# poll
collection = Collection(poll_url, 'user1', 'Password1')
tc_source = TAXIICollectionSource(collection)
f1 = Filter("type", "=", "indicator")
indicators = tc_source.query([f1])
n = 0
for indicator in indicators:
print(indicator)
filewrite(indicator, n)
n += 1
def mitre_tactics_import():
collection = Collection(MITRE_TACTICS_URL)
tc_source = TAXIICollectionSource(collection)
kc = KillChains.get_or_create(name='mitre-attack',
human_name='MITRE ATT&CK')
kc.description = 'The MITRE ATT&CK tactics are represented as kill-chains in STIX2'
kc.save()
for item in tc_source.query(Filter('type', '=', 'x-mitre-tactic')):
print(f'Adding {item["x_mitre_shortname"]}')
kc.add_phase_to_killchain({
'name': item['x_mitre_shortname'],
'description': item['description']
})
def build_taxii_source(collection_name):
"""Downloads latest Enterprise or Mobile ATT&CK content from MITRE TAXII Server."""
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection_map = {
"enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e",
"mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
}
collection_url = "https://cti-taxii.mitre.org/stix/collections/" + collection_map[
collection_name] + "/"
collection = Collection(collection_url)
taxii_ds = TAXIICollectionSource(collection)
# Create an in-memory source (to prevent multiple web requests)
return MemorySource(stix_data=taxii_ds.query())
def establish_connection(collection: str):
"""establish a connection with the TAXII server.
arguments:
collection: string, the url of the collection with which to connect.
returns:
connection to the taxii collection
"""
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection = Collection(collection)
# Supply the collection to TAXIICollection
tc_src = TAXIICollectionSource(collection)
return tc_src
def connect_server(self, url=None):
"""
Allow user to specify what url to use
:param url:
:return:
"""
server_url = MITRE_TAXII_URL if url is None else url
self.attack_server = Server(server_url, proxies=self.proxies)
api_root = self.attack_server.api_roots[0]
# CompositeSource to query all the collections at once
c_sources = [
TAXIICollectionSource(collection)
for collection in api_root.collections
]
self.composite_ds = CompositeDataSource()
self.composite_ds.add_data_sources(c_sources)
def get_items(self, filters, collection_title="Enterprise ATT&CK"):
"""
Get items using filters
Reference:
https://github.com/mitre/cti/blob/master/USAGE.md
:param filters: list of filter
:return:
"""
if self.attack_server is None:
self.connect_server()
collection = self.collection_dict[collection_title]
tc_source = TAXIICollectionSource(collection)
items = tc_source.query(filters)
return items
def get_taxii_collection_source(cls):
# あらかじめ ATT&CK の TAXIICOllectionSourceを取得する
try:
proxies = System.get_request_proxies()
attck_txs = Server("%s/taxii/" % (cls.ATT_CK_TAXII_SERVER),
proxies=proxies)
api_root = attck_txs.api_roots[0]
for collection in api_root.collections:
if collection.title == cls.COLLCETION_TITLE:
collection = Collection(
"%s/stix/collections/%s/" %
(cls.ATT_CK_TAXII_SERVER, collection.id),
proxies=proxies)
return TAXIICollectionSource(collection)
return None
except Exception:
import traceback
traceback.print_exc()
return None
def get_mitre_value_from_id(client, args):
attack_ids = argToList(args.get('attack_ids', []))
attack_values = []
for attack_id in attack_ids:
collection_id = f"stix/collections/{ENTERPRISE_COLLECTION_ID}/"
collection_url = urljoin(client.base_url, collection_id)
collection_data = Collection(collection_url,
verify=client.verify,
proxies=client.proxies)
tc_source = TAXIICollectionSource(collection_data)
attack_pattern_obj = tc_source.query([
Filter("external_references.external_id", "=", attack_id),
Filter("type", "=", "attack-pattern")
])
attack_pattern_name = attack_pattern_obj[0][
'name'] if attack_pattern_obj else None
if attack_pattern_name and len(attack_id) > 5: # sub-technique
parent_name = tc_source.query([
Filter("external_references.external_id", "=", attack_id[:5]),
Filter("type", "=", "attack-pattern")
])[0]['name']
attack_pattern_name = f'{parent_name}: {attack_pattern_name}'
if attack_pattern_name:
attack_values.append({
'id': attack_id,
'value': attack_pattern_name
})
if attack_values:
return CommandResults(outputs=attack_values,
outputs_key_field='id',
outputs_prefix='MITREATTACK',
readable_output=tableToMarkdown(
'MITRE ATTACK Attack Patterns values:',
attack_values))
return CommandResults(
readable_output=f'MITRE ATTACK Attack Patterns values: '
f'No Attack Patterns found for {attack_ids}.')
def get_collection_src():
"""
Get collection src from collections objects provided by TAXII_SERVER
:return: collection source object
"""
global CS
server = Server(TAXII_SERVER)
collections = [
c for c in server.api_roots[0].collections
if c._title not in EXCLUDE_COLLECTIONS
]
if CS.get_all_data_sources():
print("Reusing collections")
return CS
else:
print("Creating new collections")
for collection in collections:
print("Adding collection %s %s" %
(collection._title, collection.id))
CS.add_data_source(TAXIICollectionSource(collection))
return CS
def test_parse_taxii_filters(collection):
query = [
Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
Filter("id", "=", "taxii stix object ID"),
Filter("type", "=", "taxii stix object ID"),
Filter("version", "=", "first"),
Filter("created_by_ref", "=", "Bane"),
]
taxii_filters_expected = [
Filter("added_after", "=", "2016-02-01T00:00:01.000Z"),
Filter("id", "=", "taxii stix object ID"),
Filter("type", "=", "taxii stix object ID"),
Filter("version", "=", "first")
]
ds = TAXIICollectionSource(collection)
taxii_filters = ds._parse_taxii_filters(query)
assert taxii_filters == taxii_filters_expected
def __init__(self, source='taxii', local=None):
"""
Initialization - Creates a matrix generator object
:param server: Source to utilize (taxii or local)
:param local: string path to local cache of stix data
"""
self.convert_data = {}
if source.lower() not in ['taxii', 'local']:
print(
'[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii" or "local"'
.format(source))
raise ValueError
if source.lower() == 'taxii':
self.server = Server('https://cti-taxii.mitre.org/taxii')
self.api_root = self.server.api_roots[0]
self.collections = dict()
for collection in self.api_root.collections:
if collection.title != "PRE-ATT&CK":
tc = Collection(
'https://cti-taxii.mitre.org/stix/collections/' +
collection.id)
self.collections[collection.title.split(' ')
[0].lower()] = TAXIICollectionSource(tc)
elif source.lower() == 'local':
if local is not None:
hd = MemoryStore()
if 'mobile' in local.lower():
self.collections['mobile'] = hd.load_from_file(local)
else:
self.collections['enterprise'] = hd.load_from_file(local)
else:
print(
'[MatrixGen] - "local" source specified, but path to local source not provided'
)
raise ValueError
self.matrix = {}
self._build_matrix()
def test_get_404():
"""a TAXIICollectionSource.get() call that receives an HTTP 404 response
code from the taxii2client should be be returned as None.
TAXII spec states that a TAXII server can return a 404 for nonexistent
resources or lack of access. Decided that None is acceptable reponse
to imply that state of the TAXII endpoint.
"""
class TAXIICollection404():
can_read = True
def get_object(self, id, version=None):
resp = Response()
resp.status_code = 404
resp.raise_for_status()
ds = TAXIICollectionSource(TAXIICollection404())
# this will raise 404 from mock TAXII Client but TAXIICollectionStore
# should handle gracefully and return None
stix_obj = ds.get("indicator--1")
assert stix_obj is None
def get_tech_mitigation(self, tech_id=None, tech_name=None):
"""
Get mitigation for a given tech
Reference:
https://github.com/mitre/cti/blob/master/USAGE.md
:param tech_id: STIX id for mitre tech
:return:
"""
# Connect first if not already
if self.attack_server is None:
self.connect_server()
collection_title = "Enterprise ATT&CK"
collection = self.collection_dict[collection_title]
tc_source = TAXIICollectionSource(collection)
# Need to get the obj first and then the STIX id
tech_filter = None
if tech_id is not None:
tech_filter = Filter("external_references.external_id", '=',
tech_id)
elif tech_name is not None:
tech_filter = Filter("name", '=', tech_name)
filt = [Filter("type", '=', "attack-pattern"), tech_filter]
tech = tc_source.query(filt)
relations = tc_source.relationships(tech[0].id,
"mitigates",
target_only=True)
filters = [
Filter("type", '=', "course-of-action"),
Filter("id", "in", [r.source_ref for r in relations])
]
ret = tc_source.query(filters)
return ret[0].get("description", "")
def cti_create(collection, client):
cti = client['cti']
cve = cti['cve']
cwe = cti['cwe']
capec = cti['capec']
attack_techniques = cti['attack_techniques']
attack_relationships = cti['attack_relationships']
attack_groups = cti['attack_groups']
attack_software = cti['attack_software']
attack_mitigations = cti['attack_mitigations']
misp_threat_actor = cti['misp_threat_actor']
if collection == "CVE":
files = [
f for f in os.listdir(NVD_PATH)
if os.path.isfile(os.path.join(NVD_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(NVD_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
nvd_dict = json.loads(jsonfile.read())
for element in nvd_dict['CVE_Items']:
cve.insert_one(element)
jsonfile.close()
return
elif collection == "CWE":
files = [
f for f in os.listdir(CWE_PATH)
if os.path.isfile(os.path.join(CWE_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(CWE_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
cwe_dict = xmltodict.parse(jsonfile.read())
cwe_list = cwe_dict['Weakness_Catalog']['Weaknesses']['Weakness']
for element in cwe_list:
cwe.insert_one(json.loads(
json.dumps(element).replace("@", "")))
jsonfile.close()
return
elif collection == "CAPEC":
files = [
f for f in os.listdir(CAPEC_PATH)
if os.path.isfile(os.path.join(CAPEC_PATH, f))
]
files.sort()
for file in files:
archive = zipfile.ZipFile(os.path.join(CAPEC_PATH, file), 'r')
jsonfile = archive.open(archive.namelist()[0])
capec_dict = xmltodict.parse(jsonfile.read())
capec_list = capec_dict['Attack_Pattern_Catalog'][
'Attack_Patterns']['Attack_Pattern']
for element in capec_list:
capec.insert_one(
json.loads(json.dumps(element).replace("@", "")))
jsonfile.close()
return
elif collection == "ATTACK":
technique_helper = []
software_helper = []
attack_id = {}
for collection in API_ROOT.collections:
attack_id[collection.title] = collection.id
attack_collection = {}
enterprise_attack = Collection(MITRE_STIX +
attack_id["Enterprise ATT&CK"])
pre_attack = Collection(MITRE_STIX + attack_id["PRE-ATT&CK"])
enterprise_attack_source = TAXIICollectionSource(enterprise_attack)
pre_attack_source = TAXIICollectionSource(pre_attack)
teacher = dict((requests.get(ATTACK_TEACHER).json()))
teacher_dict = {}
for technique in teacher['techniques']:
if technique["color"] == RED:
teacher_dict[technique["techniqueID"]] = '5'
elif technique["color"] == ORANGE:
teacher_dict[technique["techniqueID"]] = '4'
elif technique["color"] == YELLOW:
teacher_dict[technique["techniqueID"]] = '3'
elif technique["color"] == GREEN:
teacher_dict[technique["techniqueID"]] = '2'
elif technique["color"] == BLUE:
teacher_dict[technique["techniqueID"]] = '1'
else:
teacher_dict[technique["techniqueID"]] = 'Unknown'
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
for key in filter_objs:
attack_collection[key] = []
try:
attack_collection[key] += enterprise_attack_source.query(
filter_objs[key])
except:
pass
try:
attack_collection[key] += pre_attack_source.query(
filter_objs[key])
except:
pass
for entity in attack_collection["relationships"]:
attack_relationships.insert_one(dict(entity))
for entity in attack_collection["techniques"]:
dict_entity = dict(entity)
for references in dict_entity['external_references']:
if 'attack' in references['source_name']:
attack_id = references['external_id']
if attack_id in teacher_dict.keys():
dict_entity['level'] = teacher_dict[attack_id]
else:
dict_entity['level'] = 'Unknown'
technique_helper.append(dict_entity)
attack_techniques.insert_one(dict_entity)
for entity in attack_collection["malware"]:
dict_entity = dict(entity)
sw_id = dict_entity['id']
techniques = []
measured_techniques = 0
ttp_score_sum = 0
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'attack-pattern' in dict_rel['target_ref']:
techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'attack-pattern' in dict_rel['source_ref']:
techniques.append(dict_rel['source_ref'])
for ttp_id in techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += int(technique['level'])
if measured_techniques == 0:
dict_entity['score'] = 'Unknown'
else:
dict_entity['score'] = round(
ttp_score_sum / measured_techniques, 2)
software_helper.append(dict_entity)
attack_software.insert_one(dict_entity)
for entity in attack_collection["tools"]:
dict_entity = dict(entity)
sw_id = dict_entity['id']
techniques = []
measured_techniques = 0
ttp_score_sum = 0
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'attack-pattern' in dict_rel['target_ref']:
techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'attack-pattern' in dict_rel['source_ref']:
techniques.append(dict_rel['source_ref'])
for ttp_id in techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += int(technique['level'])
if measured_techniques == 0:
dict_entity['score'] = 'Unknown'
else:
dict_entity['score'] = round(
ttp_score_sum / measured_techniques, 2)
software_helper.append(dict_entity)
attack_software.insert_one(dict_entity)
for entity in attack_collection["groups"]:
dict_entity = dict(entity)
gr_id = dict_entity['id']
measured_techniques = 0
ttp_score_sum = 0
used_techniques = []
measured_software = 0
sw_score_sum = 0
used_software = []
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == gr_id:
if 'attack-pattern' in dict_rel['target_ref']:
used_techniques.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == gr_id:
if 'attack-pattern' in dict_rel['source_ref']:
used_techniques.append(dict_rel['source_ref'])
for ttp_id in used_techniques:
for technique in technique_helper:
if technique['id'] == ttp_id and technique[
'level'] != "Unknown":
measured_techniques += 1
ttp_score_sum += float(technique['level'])
if measured_techniques == 0:
dict_entity['skill_level'] = 'Unknown'
else:
dict_entity['skill_level'] = float(ttp_score_sum /
measured_techniques)
for relationship in attack_collection["relationships"]:
dict_rel = dict(relationship)
if dict_rel['source_ref'] == sw_id:
if 'malware' in dict_rel[
'target_ref'] or 'tool' in dict_rel['target_ref']:
used_software.append(dict_rel['target_ref'])
elif dict_rel['target_ref'] == sw_id:
if 'malware' in dict_rel[
'target_ref'] or 'tool' in dict_rel['target_ref']:
used_software.append(dict_rel['source_ref'])
for sw_id in used_software:
for software in software_helper:
if software[
'id'] == sw_id and software['score'] != "Unknown":
measured_software += 1
sw_score_sum += float(software['score'])
if measured_software == 0:
dict_entity['opportunity'] = 'Unknown'
else:
dict_entity['opportunity'] = float(sw_score_sum /
measured_software)
if (dict_entity['skill_level'] == "Unknown") and (
dict_entity['opportunity'] != 'Unknown'):
dict_entity['skill_level'] = dict_entity['opportunity']
if (dict_entity['opportunity'] == "Unknown") and (
dict_entity['skill_level'] != 'Unknown'):
dict_entity['opportunity'] = dict_entity['skill_level']
if dict_entity['opportunity'] != 'Unknown' and dict_entity[
'skill_level'] != 'Unknown':
dict_entity['skill_level'] = int(
modf(OWASP * dict_entity['skill_level'])[1])
dict_entity['opportunity'] = int(
modf(OWASP * dict_entity['opportunity'])[1])
stix_group_id = dict_entity['id'].split("--")[1]
attack_group_id = None
for external_reference in dict_entity['external_references']:
if 'attack' in external_reference['source_name']:
attack_group_id = external_reference['external_id']
break
threat_actors = []
try:
misp_list = misp_threat_actor.distinct(
'uuid', {"meta.refs": GROUP_URL(attack_group_id)
}) + misp_threat_actor.distinct(
'uuid', {"related.dest-uuid": stix_group_id})
for threat_actor in misp_list:
if threat_actor not in threat_actors:
threat_actors.append(threat_actor)
except:
pass
motive_list = []
size_list = []
for threat_actor in threat_actors:
threat_actor_list = misp_threat_actor.find(
{'uuid': threat_actor})
for threat_actor_dict in threat_actor_list:
motive_list.append(int(threat_actor_dict['motive']))
size_list.append(int(threat_actor_dict['size']))
if len(motive_list) > 0:
dict_entity['motive'] = str(
sum(motive_list) / len(motive_list))
else:
dict_entity['motive'] = '0'
if dict_entity['motive'] == '0':
dict_entity['motive'] = 'Unknown'
if len(size_list) > 0:
dict_entity['size'] = str(sum(size_list) / len(size_list))
else:
dict_entity['size'] = '0'
if dict_entity['size'] == '0':
dict_entity['size'] = 'Unknown'
attack_groups.insert_one(dict_entity)
for entity in attack_collection["mitigations"]:
attack_mitigations.insert_one(dict(entity))
return
elif collection == "MISP":
with open(os.path.join(MISP_PATH, MISP_THREAT_ACTOR),
'r',
encoding="UTF-8") as misp_file:
threat_actors = json.loads(misp_file.read())
for threat_actor in threat_actors['values']:
dict_threat_actor = threat_actor_category(dict(threat_actor))
misp_threat_actor.insert_one(dict_threat_actor)
misp_file.close()
else:
print("Collection type not set")
return