def __init__(self, local_path=None):
if local_path is not None and os.path.isdir(os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, PRE_ATTCK_LOCAL_DIR)) \
and os.path.isdir(os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR)):
self.TC_ENTERPRISE_SOURCE = FileSystemSource(
os.path.join(local_path, ENTERPRISE_ATTCK_LOCAL_DIR))
self.TC_PRE_SOURCE = FileSystemSource(
os.path.join(local_path, PRE_ATTCK_LOCAL_DIR))
self.TC_MOBILE_SOURCE = FileSystemSource(
os.path.join(local_path, MOBILE_ATTCK_LOCAL_DIR))
else:
ENTERPRISE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
ENTERPRISE_ATTCK + "/")
PRE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS + PRE_ATTCK +
"/")
MOBILE_COLLECTION = Collection(ATTCK_STIX_COLLECTIONS +
MOBILE_ATTCK + "/")
self.TC_ENTERPRISE_SOURCE = TAXIICollectionSource(
ENTERPRISE_COLLECTION)
self.TC_PRE_SOURCE = TAXIICollectionSource(PRE_COLLECTION)
self.TC_MOBILE_SOURCE = TAXIICollectionSource(MOBILE_COLLECTION)
self.COMPOSITE_DS = CompositeDataSource()
self.COMPOSITE_DS.add_data_sources([
self.TC_ENTERPRISE_SOURCE, self.TC_PRE_SOURCE,
self.TC_MOBILE_SOURCE
])
def load_taxii(new=False):
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/" +
domainToTaxiiCollectionId[domain])
data_store = TAXIICollectionSource(collection)
parse_subtechniques(data_store, new)
return load_datastore(data_store)
def build_iterator(self, create_relationships=False, is_up_to_6_2=True, limit: int = -1):
"""Retrieves all entries from the feed.
Returns:
A list of objects, containing the indicators.
"""
indicators: List[Dict] = list()
mitre_id_list: Set[str] = set()
mitre_relationships_list = []
id_to_name: Dict = {}
counter = 0
# For each collection
for collection in self.collections:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
# Establish TAXII2 Collection instance
collection_url = urljoin(self.base_url, f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url, verify=self.verify, proxies=self.proxies)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection_data)
for concept in FILTER_OBJS:
if 0 < limit <= counter:
break
input_filter = FILTER_OBJS[concept]['filter']
try:
mitre_data = tc_source.query(input_filter)
except Exception:
continue
for mitre_item in mitre_data:
if 0 < limit <= counter:
break
mitre_item_json = json.loads(str(mitre_item))
if mitre_item_json.get('id') not in mitre_id_list:
value = mitre_item_json.get('name')
item_type = get_item_type(mitre_item_json.get('type'), is_up_to_6_2)
if item_type == 'Relationship' and create_relationships:
if mitre_item_json.get('relationship_type') == 'revoked-by':
continue
mitre_relationships_list.append(mitre_item_json)
else:
if is_indicator_deprecated_or_revoked(mitre_item_json):
continue
id_to_name[mitre_item_json.get('id')] = value
indicator_obj = self.create_indicator(item_type, value, mitre_item_json)
indicators.append(indicator_obj)
counter += 1
mitre_id_list.add(mitre_item_json.get('id'))
return indicators, mitre_relationships_list, id_to_name
def test_collection_missing_can_write_property(collection_dict):
with pytest.raises(ValidationError) as excinfo:
collection_dict.pop("can_write")
Collection("https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/",
user="******", password="******", verify=False,
collection_info=collection_dict)
assert "No 'can_write' in Collection for request 'https://example.com/api1/collections/91a7b528-80eb-42ed-a74d-c6fbd5a26116/'" == str(excinfo.value)
def attnck_taxii():
# Initialize dictionary to hold Enterprise ATT&CK content
ent_attack = {}
pre_attack = {}
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
ENTERPRISE_ATTCK = "95ecc380-afe9-11e4-9b6c-751b66dd541e"
PRE_ATTCK = "062767bd-02d2-4b72-84ba-56caef0f8658"
MOBILE_ATTCK = "2f669986-b40b-4423-b720-4396ca6a462b"
enterprise_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
pre_attack_collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/062767bd-02d2-4b72-84ba-56caef0f8658/"
)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(enterprise_attack_collection)
tc_source2 = TAXIICollectionSource(pre_attack_collection)
# Create filters to retrieve content from Enterprise ATT&CK based on type
filter_objs = {
"techniques": Filter("type", "=", "attack-pattern"),
"mitigations": Filter("type", "=", "course-of-action"),
"groups": Filter("type", "=", "intrusion-set"),
"malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"),
"relationships": Filter("type", "=", "relationship")
}
filter_objs_pre = {
"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"),
"relationships": Filter("type", "=", "relationship")
}
# Retrieve all Enterprise ATT&CK content
for key in filter_objs:
ent_attack[key] = tc_source.query(filter_objs[key])
for key in filter_objs_pre:
pre_attack[key] = tc_source2.query(filter_objs[key])
return ent_attack, pre_attack
def test_collection_missing_trailing_slash():
set_collection_response()
collection = Collection(COLLECTION_URL[:-1])
responses.add(responses.GET, GET_OBJECT_URL, GET_OBJECT_RESPONSE,
status=200, content_type="%s; charset=utf-8" % MEDIA_TYPE_STIX_V20)
response = collection.get_object("indicator--252c7c11-daf2-42bd-843b-be65edca9f61")
indicator = response["objects"][0]
assert indicator["id"] == "indicator--252c7c11-daf2-42bd-843b-be65edca9f61"
def view_new_ioc():
collection = Collection(
'http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01',
user='******',
password='******')
new_objects = collection.get_objects()
print(new_objects)
return prepare_response({'data': new_objects})
def get_mitre_data_by_filter(client, mitre_filter):
for collection in client.collections:
collection_url = urljoin(client.base_url, f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url, verify=client.verify, proxies=client.proxies)
tc_source = TAXIICollectionSource(collection_data)
if tc_source.query(mitre_filter):
mitre_data = tc_source.query(mitre_filter)[0]
return mitre_data
return {}
def get_data(self):
collections = {
"enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e",
"pre_attack": "062767bd-02d2-4b72-84ba-56caef0f8658",
"mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
}
collection = Collection(
f"https://cti-taxii.mitre.org/stix/collections/{collections['enterprise_attack']}/"
)
src = TAXIICollectionSource(collection)
return src
def build_taxii_source(collection_name):
"""Downloads latest Enterprise or Mobile ATT&CK content from MITRE TAXII Server."""
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection_map = {
"enterprise_attack": "95ecc380-afe9-11e4-9b6c-751b66dd541e",
"mobile_attack": "2f669986-b40b-4423-b720-4396ca6a462b"
}
collection_url = "https://cti-taxii.mitre.org/stix/collections/" + collection_map[
collection_name] + "/"
collection = Collection(collection_url)
taxii_ds = TAXIICollectionSource(collection)
# Create an in-memory source (to prevent multiple web requests)
return MemorySource(stix_data=taxii_ds.query())
def establish_connection(collection: str):
"""establish a connection with the TAXII server.
arguments:
collection: string, the url of the collection with which to connect.
returns:
connection to the taxii collection
"""
# Establish TAXII2 Collection instance for Enterprise ATT&CK collection
collection = Collection(collection)
# Supply the collection to TAXIICollection
tc_src = TAXIICollectionSource(collection)
return tc_src
def create_ioc():
collection = Collection(
'http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01',
user='******',
password='******')
name = request.form.get('name')
rtype = request.form.get('rtype')
pattern = request.form.get('pattern')
labels = request.form.get('labels')
# return prepare_response({'status': 'error', 'message': 'Something went wrong while trying to save IoC'})
if rtype == "indicator":
indicator = Indicator(name=name, pattern=pattern, labels=labels)
bundle = Bundle([indicator]).serialize()
collection.add_objects(bundle)
elif rtype == "malware":
malware = Malware(name=name, labels=labels)
bundle = Bundle([malware]).serialize()
collection.add_objects(bundle)
elif rtype == "attack-pattern":
attack_pattern = AttackPattern(name=name,
pattern=pattern,
labels=labels)
bundle = Bundle([attack_pattern]).serialize()
collection.add_objects(bundle)
elif rtype == "identity":
identity = Identity(name=name, labels=labels)
bundle = Bundle([identity]).serialize()
collection.add_objects(bundle)
elif rtype == "threat-actor":
threat_actor = ThreatActor(name=name, pattern=pattern, labels=labels)
bundle = Bundle([threat_actor]).serialize()
collection.add_objects(bundle)
elif rtype == "tool":
tool = Tool(name=name, pattern=pattern, labels=labels)
bundle = Bundle([tool]).serialize()
collection.add_objects(bundle)
elif rtype == "vulnerability":
vuln = Vulnerability(name=name, pattern=pattern, labels=labels)
bundle = Bundle([vuln]).serialize()
collection.add_objects(bundle)
return prepare_response({
'status': 'success',
'message': 'IoC was saved successfully'
})
def get_mitre_value_from_id(client, args):
attack_ids = argToList(args.get('attack_ids', []))
attack_values = []
for attack_id in attack_ids:
collection_id = f"stix/collections/{ENTERPRISE_COLLECTION_ID}/"
collection_url = urljoin(client.base_url, collection_id)
collection_data = Collection(collection_url,
verify=client.verify,
proxies=client.proxies)
tc_source = TAXIICollectionSource(collection_data)
attack_pattern_obj = tc_source.query([
Filter("external_references.external_id", "=", attack_id),
Filter("type", "=", "attack-pattern")
])
attack_pattern_name = attack_pattern_obj[0][
'name'] if attack_pattern_obj else None
if attack_pattern_name and len(attack_id) > 5: # sub-technique
parent_name = tc_source.query([
Filter("external_references.external_id", "=", attack_id[:5]),
Filter("type", "=", "attack-pattern")
])[0]['name']
attack_pattern_name = f'{parent_name}: {attack_pattern_name}'
if attack_pattern_name:
attack_values.append({
'id': attack_id,
'value': attack_pattern_name
})
if attack_values:
return CommandResults(outputs=attack_values,
outputs_key_field='id',
outputs_prefix='MITREATTACK',
readable_output=tableToMarkdown(
'MITRE ATTACK Attack Patterns values:',
attack_values))
return CommandResults(
readable_output=f'MITRE ATTACK Attack Patterns values: '
f'No Attack Patterns found for {attack_ids}.')
def get_taxii_collection_source(cls):
# あらかじめ ATT&CK の TAXIICOllectionSourceを取得する
try:
proxies = System.get_request_proxies()
attck_txs = Server("%s/taxii/" % (cls.ATT_CK_TAXII_SERVER),
proxies=proxies)
print('>>> attck_txs: ' + str(attck_txs))
api_root = attck_txs.api_roots[0]
for collection in api_root.collections:
if collection.title == cls.COLLCETION_TITLE:
collection = Collection(
"%s/stix/collections/%s/" %
(cls.ATT_CK_TAXII_SERVER, collection.id),
proxies=proxies)
return TAXIICollectionSource(collection)
return None
except Exception:
import traceback
traceback.print_exc()
return None
def __init__(self, source='taxii', local=None):
"""
Initialization - Creates a matrix generator object
:param server: Source to utilize (taxii or local)
:param local: string path to local cache of stix data
"""
self.convert_data = {}
if source.lower() not in ['taxii', 'local']:
print(
'[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii" or "local"'
.format(source))
raise ValueError
if source.lower() == 'taxii':
self.server = Server('https://cti-taxii.mitre.org/taxii')
self.api_root = self.server.api_roots[0]
self.collections = dict()
for collection in self.api_root.collections:
if collection.title != "PRE-ATT&CK":
tc = Collection(
'https://cti-taxii.mitre.org/stix/collections/' +
collection.id)
self.collections[collection.title.split(' ')
[0].lower()] = TAXIICollectionSource(tc)
elif source.lower() == 'local':
if local is not None:
hd = MemoryStore()
if 'mobile' in local.lower():
self.collections['mobile'] = hd.load_from_file(local)
else:
self.collections['enterprise'] = hd.load_from_file(local)
else:
print(
'[MatrixGen] - "local" source specified, but path to local source not provided'
)
raise ValueError
self.matrix = {}
self._build_matrix()
def collection():
"""Default Collection object"""
# The collection response is needed to get information about the collection
set_collection_response()
return Collection(COLLECTION_URL)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--matrix",
type=str,
action="store",
default="enterprise",
help="Matrix to query (enterprise, mobile, pre)")
# Techniques
parser.add_argument(
"--dump-all-techniques",
action="store_true",
help="Dump a CSV file with technique,subtechnique,name")
# Data Sources
parser.add_argument(
"--dump-data-sources",
action="store_true",
help="Dump a list of unique data sources to data_sources.txt")
parser.add_argument(
"--dump-metadata",
action="store_true",
help=
"Dump a CSV file technique-metadata.csv, containing unique technique-metadata pairings."
)
parser.add_argument(
"--dump-matching-techniques",
action="store_true",
help=
"Dump techniques that map to match-data-sources to matching-techniques.txt"
)
parser.add_argument(
"--match-data-sources",
type=str,
action="store",
help=
"A file containing a list of data sources that to match against techniques."
)
args = parser.parse_args()
match_data_sources = None
if args.match_data_sources:
match_data_sources = parse_data_source_list(args.match_data_sources)
args.matrix = args.matrix.lower()
if args.matrix == 'pre':
matrix = "062767bd-02d2-4b72-84ba-56caef0f8658"
elif args.matrix == 'mobile':
matrix = "2f669986-b40b-4423-b720-4396ca6a462b"
elif args.matrix == 'enterprise':
matrix = "95ecc380-afe9-11e4-9b6c-751b66dd541e"
# Initialize dictionary to hold Enterprise ATT&CK content
attack = {}
# Establish TAXII2 Collection instance for Enterprise ATT&CK
collection = Collection("https://cti-taxii.mitre.org/stix/collections/{0}/"\
.format(matrix))
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Create filters to retrieve content from Enterprise ATT&CK
filter_objs = {"techniques": Filter("type", "=", "attack-pattern")}
# Retrieve all Enterprise ATT&CK content
for key in filter_objs:
attack[key] = tc_source.query(filter_objs[key])
all_techniques = attack["techniques"]
all_techniques = remove_revoked_deprecated(all_techniques)
technique_count = 0
techniques_without_data_source = 0
techniques_observable = 0
techniques_with_data_sources = []
data_sources = set()
matching_techniques = set()
for technique in all_techniques:
technique_count += 1
technique_id = technique['external_references'][0]['external_id']
if 'x_mitre_data_sources' in technique.keys():
if match_data_sources is not None:
if data_source_match(technique['x_mitre_data_sources'],
match_list=match_data_sources) == True:
techniques_observable += 1
if args.dump_matching_techniques == True:
matching_techniques.add(technique_id)
if args.dump_data_sources == True:
[
data_sources.add(data_source)
for data_source in technique['x_mitre_data_sources']
]
if args.dump_metadata == True:
[
techniques_with_data_sources.append(
(technique_id, data_source))
for data_source in technique['x_mitre_data_sources']
]
else:
techniques_without_data_source += 1
# Output files based on input arguments.
if match_data_sources is not None:
print('Techniques: {0}'.format(technique_count))
print('Techniques Observable: {0} ({1}%)'\
.format(techniques_observable,
round((techniques_observable / technique_count) * 100)))
if args.dump_data_sources == True:
with open('data_sources.txt', 'w') as fh_data_sources:
data_sources = list(data_sources)
data_sources.sort()
for data_source in data_sources:
fh_data_sources.write('{0}\n'.format(data_source))
if args.dump_matching_techniques == True:
with open('matching_techniques.txt', 'w') as fh_matching_techniques:
matching_techniques = list(matching_techniques)
matching_techniques.sort()
for data_source in matching_techniques:
fh_matching_techniques.write('{0}\n'.format(data_source))
if args.dump_metadata == True:
with open('technique_metadata.csv', 'w') as fh_techniques:
csvwriter = csv.writer(fh_techniques, quoting=csv.QUOTE_ALL)
csvwriter.writerow(['id', 'data_source'])
for technique in techniques_with_data_sources:
csvwriter.writerow(technique)
if args.dump_all_techniques == True:
with open('all_techniques.csv', 'w') as fh_all_techniques:
csvwriter = csv.writer(fh_all_techniques, quoting=csv.QUOTE_ALL)
csvwriter.writerow([
'technique_id', 'technique_name', 'technique_url',
'technique_description'
])
for technique in all_techniques:
# Handle techniques that do not have a description (these probably
# should not exist and should be contributed).
try:
description = technique.description
except AttributeError:
description = 'NONE'
csvwriter.writerow([
technique.external_references[0].external_id,
technique.name, technique.external_references[0].url,
description
])
def build_iterator(self, limit: int = -1) -> List:
"""Retrieves all entries from the feed.
Returns:
A list of objects, containing the indicators.
"""
indicators: List[Dict] = list()
mitre_id_list: Set[str] = set()
indicator_values_list: Set[str] = set()
external_refs: Set[str] = set()
counter = 0
# For each collection
for collection in self.collections:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
# Establish TAXII2 Collection instance
collection_url = urljoin(self.base_url,
f'stix/collections/{collection.id}/')
collection_data = Collection(collection_url,
verify=self.verify,
proxies=self.proxies)
# Supply the collection to TAXIICollection
tc_source = TAXIICollectionSource(collection_data)
# Create filters to retrieve content
filter_objs = {
"Technique": {
"name": "attack-pattern",
"filter": Filter("type", "=", "attack-pattern")
},
"Mitigation": {
"name": "course-of-action",
"filter": Filter("type", "=", "course-of-action")
},
"Group": {
"name": "intrusion-set",
"filter": Filter("type", "=", "intrusion-set")
},
"Malware": {
"name": "malware",
"filter": Filter("type", "=", "malware")
},
"Tool": {
"name": "tool",
"filter": Filter("type", "=", "tool")
},
}
# Retrieve content
for concept in filter_objs:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
input_filter = filter_objs[concept]['filter']
try:
mitre_data = tc_source.query(input_filter)
except Exception:
continue
# For each item in the MITRE list, add an indicator to the indicators list
for mitreItem in mitre_data:
# Stop when we have reached the limit defined
if 0 < limit <= counter:
break
mitre_item_json = json.loads(str(mitreItem))
value = None
# Try and map a friendly name to the value before the real ID
try:
externals = [
x['external_id'] for x in mitre_item_json.get(
'external_references', [])
if x['source_name'] == 'mitre-attack'
and x['external_id']
]
value = externals[0]
except Exception:
value = None
if not value:
value = mitre_item_json.get('x_mitre_old_attack_id',
None)
if not value:
value = mitre_item_json.get('id')
if mitre_item_json.get('id') not in mitre_id_list:
# If the indicator already exists, then append the new data
# to the existing indicator.
if value in indicator_values_list:
# Append data to the original item
original_item = [
x for x in indicators
if x.get('value') == value
][0]
if original_item['rawJSON'].get('id', None):
try:
original_item['rawJSON'][
'id'] += f"\n{mitre_item_json.get('id', '')}"
except Exception:
pass
if original_item['rawJSON'].get('created', None):
try:
original_item['rawJSON'][
'created'] += f"\n{mitre_item_json.get('created', '')}"
except Exception:
pass
if original_item['rawJSON'].get('modified', None):
try:
original_item['rawJSON'][
'modified'] += f"\n{mitre_item_json.get('modified', '')}"
except Exception:
pass
if original_item['rawJSON'].get(
'description', None):
try:
if not original_item['rawJSON'].get(
'description').startswith("###"):
original_item['rawJSON']['description'] = \
f"### {original_item['rawJSON'].get('type')}\n" \
f"{original_item['rawJSON']['description']}"
original_item['rawJSON']['description'] += \
f"\n\n_____\n\n### {mitre_item_json.get('type')}\n" \
f"{mitre_item_json.get('description', '')}"
except Exception:
pass
if original_item['rawJSON'].get(
'external_references', None):
try:
original_item['rawJSON'][
'external_references'].extend(
mitre_item_json.get(
'external_references', []))
except Exception:
pass
if original_item['rawJSON'].get(
'kill_chain_phases', None):
try:
original_item['rawJSON'][
'kill_chain_phases'].extend(
mitre_item_json.get(
'kill_chain_phases', []))
except Exception:
pass
if original_item['rawJSON'].get('aliases', None):
try:
original_item['rawJSON']['aliases'].extend(
mitre_item_json.get('aliases', []))
except Exception:
pass
else:
indicator_obj = {
"value": value,
"score": self.reputation,
"type": "MITRE ATT&CK",
"rawJSON": mitre_item_json,
"fields": {
"tags": self.tags,
}
}
if self.tlp_color:
indicator_obj['fields'][
'trafficlightprotocol'] = self.tlp_color
indicators.append(indicator_obj)
indicator_values_list.add(value)
counter += 1
mitre_id_list.add(mitre_item_json.get('id'))
# Create a duplicate indicator using the "external_id" from the
# original indicator, if the user has selected "includeAPT" as True
if self.include_apt:
ext_refs = [
x.get('external_id')
for x in mitre_item_json.get(
'external_references')
if x.get('external_id')
and x.get('source_name') != "mitre-attack"
]
for x in ext_refs:
if x not in external_refs:
indicator_obj = {
"value": x,
"score": self.reputation,
"type": "MITRE ATT&CK",
"rawJSON": mitre_item_json,
"fields": {
"tags": self.tags,
}
}
if self.tlp_color:
indicator_obj['fields'][
'trafficlightprotocol'] = self.tlp_color
indicators.append(indicator_obj)
external_refs.add(x)
# Finally, map all the fields from the indicator
# rawjson to the fields in the indicator
for indicator in indicators:
indicator['fields'] = dict()
for field, value in mitre_field_mapping.items():
try:
# Try and map the field
value_type = value['type']
value_name = value['name']
if value_type == "list":
indicator['fields'][field] = "\n".join(
indicator['rawJSON'][value_name])
else:
if value_name in ['created', 'modified']:
indicator['fields'][
field] = handle_multiple_dates_in_one_field(
value_name,
indicator['rawJSON'][value_name])
else:
indicator['fields'][field] = indicator['rawJSON'][
value_name]
except KeyError:
# If the field does not exist in the indicator
# then move on
pass
except Exception as err:
demisto.error(f"Error when mapping Mitre Fields - {err}")
return indicators
# Import needed libraries
from stix2 import TAXIICollectionSource, Filter
from taxii2client.v20 import Server, Collection
import json
import re
from openpyxl import Workbook
from openpyxl.styles import Font
from openpyxl.styles import Alignment
#__________________________________________________________________________________________________________________________
# Enterprise attack source
collection = Collection(
"https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/"
)
# Supply the TAXII2 collection to TAXIICollection
tc_source = TAXIICollectionSource(collection)
# Filter to only techniques
filt = Filter('type', '=', 'attack-pattern')
techniques = tc_source.query([filt])
#__________________________________________________________________________________________________________________________
# Generate list of all kill chain phases and put them in proper order
kc_list = []
for technique in techniques:
for k, v in technique.items():
if k == "kill_chain_phases":
for i in v:
for k, v in i.items():
kc_list.append(v)
from taxii2client.v20 import Collection
from stix2 import *
coll = Collection('http://127.0.0.1:5000/trustgroup1/collections/365fed99-08fa-fdcd-a1b3-fb247eb41d01',
user='******',
password='******')
indicator = Indicator(name="File hash for malware variant",
labels=["malicious-activity"],
pattern="[file:hashes.md5 = 'd41d8cd98f00b204e9800998ecf8427e']")
#print(indicator)
bundle = Bundle([indicator]).serialize()
coll.add_objects(bundle)
obj = coll.get_objects()
print(obj)
def __init__(self, source='taxii', resource=None):
"""
Initialization - Creates a matrix generator object
:param source: Source to utilize (taxii, remote, or local)
:param resource: string path to local cache of stix data (local) or url of an ATT&CK Workbench (remote)
"""
self.convert_data = {}
self.collections = dict()
if source.lower() not in ['taxii', 'local', 'remote']:
print(
'[MatrixGen] - Unable to generate matrix, source {} is not one of "taxii", "remote" or '
'"local"'.format(source))
raise ValueError
if source.lower() == 'taxii':
self.server = Server('https://cti-taxii.mitre.org/taxii')
self.api_root = self.server.api_roots[0]
for collection in self.api_root.collections:
if collection.title != "PRE-ATT&CK":
tc = Collection(
'https://cti-taxii.mitre.org/stix/collections/' +
collection.id)
self.collections[collection.title.split(' ')
[0].lower()] = TAXIICollectionSource(tc)
elif source.lower() == 'local':
if resource is not None:
hd = MemoryStore()
hd.load_from_file(resource)
if 'mobile' in resource.lower():
self.collections['mobile'] = hd
else:
self.collections['enterprise'] = hd
else:
print(
'[MatrixGen] - "local" source specified, but path to local source not provided'
)
raise ValueError
elif source.lower() == 'remote':
if resource is not None:
if ':' not in resource[6:]:
print(
'[MatrixGen] - "remote" source missing port; assuming ":3000"'
)
resource += ":3000"
if not resource.startswith('http'):
resource = 'http://' + resource
for dataset in ['enterprise', 'mobile']:
hd = MemoryStore()
response = requests.get(
f"{resource}/api/stix-bundles?domain={dataset}-"
f"attack&includeRevoked=true&includeDeprecated=true")
response.raise_for_status(
) # ensure we notice bad responses
_add(hd, json.loads(response.text), True, None)
self.collections[dataset] = hd
else:
print(
f'[MatrixGen] - WARNING: "remote" selected without providing a "resource" url. The use of '
f'"remote" requires the inclusion of a "resource" url to an ATT&CK Workbench instance. No matrix '
f'will be generated...')
self.matrix = {}
self._build_matrix()
def test_collection_unexpected_kwarg():
with pytest.raises(TypeError):
Collection(url="", conn=None, foo="bar")
def bad_writable_collection():
"""Collection with 'can_write=true', but the COLLECTION_URL is different
from the one in the response"""
set_collection_response(response=WRITABLE_COLLECTION)
return Collection(COLLECTION_URL)
def writable_collection():
"""Collection with 'can_write' set to 'true'."""
set_collection_response(WRITABLE_COLLECTION_URL, WRITABLE_COLLECTION)
return Collection(WRITABLE_COLLECTION_URL)
async def insert_attack_stix_data(self):
"""
Function to pull stix/taxii information and insert in to the local db
:return: status code
"""
logging.info('Downloading ATT&CK data from STIX/TAXII...')
attack = {}
collection = Collection("https://cti-taxii.mitre.org/stix/collections/95ecc380-afe9-11e4-9b6c-751b66dd541e/")
tc_source = TAXIICollectionSource(collection)
filter_objs = {"techniques": Filter("type", "=", "attack-pattern"),
"groups": Filter("type", "=", "intrusion-set"), "malware": Filter("type", "=", "malware"),
"tools": Filter("type", "=", "tool"), "relationships": Filter("type", "=", "relationship")}
for key in filter_objs:
attack[key] = tc_source.query(filter_objs[key])
references = {}
# add all of the patterns and dictionary keys/values for each technique and software
for i in attack["techniques"]:
references[i["id"]] = {"name": i["name"], "id": i["external_references"][0]["external_id"],
"example_uses": [],
"description": i.get('description', NO_DESC).replace('<code>', '').replace(
'</code>', '').replace('\n', '').encode('ascii', 'ignore').decode('ascii'),
"similar_words": [i["name"]]}
for i in attack["relationships"]:
if i["relationship_type"] == 'uses':
if 'attack-pattern' in i["target_ref"]:
use = i.get('description', NO_DESC)
# remove unnecessary strings, fix unicode errors
use = use.replace('<code>', '').replace('</code>', '').replace('"', "").replace(',', '').replace(
'\t', '').replace(' ', ' ').replace('\n', '').encode('ascii', 'ignore').decode('ascii')
find_pattern = re.compile('\[.*?\]\(.*?\)') # get rid of att&ck reference (name)[link to site]
m = find_pattern.findall(use)
if len(m) > 0:
for j in m:
use = use.replace(j, '')
if use[0:2] == '\'s':
use = use[3:]
elif use[0] == ' ':
use = use[1:]
# combine all the examples to one list
references[i["target_ref"]]["example_uses"].append(use)
for i in attack["malware"]:
if 'description' not in i: # some software do not have description, example: darkmoon https://attack.mitre.org/software/S0209
continue
else:
references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i["description"],
"examples": [], "example_uses": [], "similar_words": [i["name"]]}
for i in attack["tools"]:
references[i["id"]] = {"id": i['id'], "name": i["name"], "description": i.get('description', NO_DESC),
"examples": [], "example_uses": [], "similar_words": [i["name"]]}
attack_data = references
logging.info("Finished...now creating the database.")
cur_uids = await self.get_technique_uids()
for k, v in attack_data.items():
if k not in cur_uids:
await self.dao.insert('attack_uids', dict(uid=k, description=defang_text(v.get('description', NO_DESC)),
tid=v['id'], name=v['name']))
if 'regex_patterns' in v:
[await self.dao.insert_generate_uid('regex_patterns',
dict(attack_uid=k, regex_pattern=defang_text(x)))
for x in v['regex_patterns']]
if 'similar_words' in v:
[await self.dao.insert_generate_uid('similar_words',
dict(attack_uid=k, similar_word=defang_text(x)))
for x in v['similar_words']]
if 'false_negatives' in v:
[await self.dao.insert_generate_uid('false_negatives',
dict(attack_uid=k, false_negative=defang_text(x)))
for x in v['false_negatives']]
if 'false_positives' in v:
[await self.dao.insert_generate_uid('false_positives',
dict(attack_uid=k, false_positive=defang_text(x)))
for x in v['false_positives']]
if 'true_positives' in v:
[await self.dao.insert_generate_uid('true_positives',
dict(attack_uid=k, true_positive=defang_text(x)))
for x in v['true_positives']]
if 'example_uses' in v:
[await self.dao.insert_generate_uid('true_positives',
dict(attack_uid=k, true_positive=defang_text(x)))
for x in v['example_uses']]
logging.info('[!] DB Item Count: {}'.format(len(await self.dao.get('attack_uids'))))
def test_collection_with_custom_properties(collection_dict):
collection_dict["type"] = "domain"
col_obj = Collection(url=WRITABLE_COLLECTION_URL,
collection_info=collection_dict)
assert len(col_obj.custom_properties) == 1
assert col_obj.custom_properties["type"] == "domain"