def test_file_results_to_stix(self):
file_name = '.X0-lock'
data = {
"computer_identity": "12369754-bigdata4545.canlab.ibm.com",
"subQueryID": 1,
"type": "file",
"file_name": ".X0-lock",
"sha256hash":
"7236f966f07259a1de3ee0d48a3ef0ee47c4a551af7f0d76dcabbbb9d6e00940",
"sha1hash": "8b5e953be1db90172af66631132f6f27dda402d2",
"md5hash": "e5307d27f0eb9a27af8597a1ddc51e89",
"file_path": "/tmp/.X0-lock",
"modified_time": "1541424894"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_obj = TestBigFixResultsToStix.get_first_of_type(
objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {
'type', 'name', 'hashes', 'parent_directory_ref'
})
assert (file_obj['name'] == file_name)
def test_network_cim_to_stix(self):
count = 2
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
dest_ip = "127.0.0.1"
dest_port = "8090"
src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
src_port = "8080"
transport = "http"
data = {
"event_count": count,
"_time": time,
"user": user,
"dest_ip": dest_ip,
"dest_port": dest_port,
"src_ip": src_ip,
"src_port": src_port,
"protocol": transport
}
print(data)
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_obj is not None), 'network-traffic object type not found'
assert (nt_obj.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_obj['src_port'] == 8080)
assert (nt_obj['dst_port'] == 8090)
assert (nt_obj['protocols'] == ['http'])
ip_ref = nt_obj['dst_ref']
assert (ip_ref
in objects), f"dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == dest_ip)
ip_ref = nt_obj['src_ref']
assert (ip_ref
in objects), f"src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == src_ip)
def test_common_prop(self):
data = {"starttime": 1531169112, "eventcount": 5}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
assert (result_bundle_identity['id'] == data_source['id'])
assert (result_bundle_identity['name'] == data_source['name'])
assert (result_bundle_identity['identity_class'] ==
data_source['identity_class'])
observed_data = result_bundle_objects[1]
assert (observed_data['id'] is not None)
assert (observed_data['type'] == "observed-data")
assert (
observed_data['created_by_ref'] == result_bundle_identity['id'])
assert (observed_data['number_observed'] == 5)
assert (observed_data['created'] is not None)
assert (observed_data['modified'] is not None)
assert (observed_data['first_observed'] is not None)
assert (observed_data['last_observed'] is not None)
def translate_results(self, data_source, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX.
Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
self.mapping = options['mapping'] if 'mapping' in options else {}
json_data = json.loads(data)
data_source = json.loads(data_source)
if (not self.mapping):
map_file = open(self.default_mapping_file_path).read()
map_data = json.loads(map_file)
else:
map_data = self.mapping
results = json_to_stix_translator.convert_to_stix(
data_source, map_data, json_data,
transformers.get_all_transformers(), options, self.callback)
return json.dumps(results, indent=4, sort_keys=False)
def test_process_results_to_stix(self):
process_name = 'systemd'
data = {
"computer_identity": "12369754-bigdata4545.canlab.ibm.com",
"subQueryID": 1,
"start_time": "1541424881",
"type": "process",
"process_name": "systemd",
"process_id": "1",
"sha256hash":
"9c74c625b2aba7a2e8d8a42e2e94715c355367f7cbfa9bd5404ba52b726792a6",
"sha1hash": "916933045c5c91ebcaa325e7f8302f3a732a0a3d",
"md5hash": "28a9beb86c4d4c31ba572805bea8494f",
"file_path": "/usr/lib/systemd/systemd"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
print(json.dumps(result_bundle, indent=2))
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
process_obj = TestBigFixResultsToStix.get_first_of_type(
objects.values(), 'process')
assert (process_obj is not None), 'process object type not found'
assert (process_obj.keys() == {'type', 'name', 'pid', 'binary_ref'})
assert (process_obj['name'] == process_name)
def translate_results(self, data_source, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX.
Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
self.mapping = options['mapping'] if 'mapping' in options else {}
json_data = json.loads(data)
data_source = json.loads(data_source)
if (not self.mapping):
map_file = open(self.mapping_filepath).read()
map_data = json.loads(map_file)
else:
map_data = self.mapping
results = json_to_stix_translator.convert_to_stix(
data_source, map_data, json_data,
transformers.get_all_transformers(), options)
if len(results['objects']) - 1 == len(json_data):
for i in range(1, len(results['objects'])):
results['objects'][i]['number_observed'] = 1
else:
raise RuntimeError(
"Incorrect number of result objects after translation. Found: {}, expected: {}."
.format(len(results['objects']) - 1, len(json_data)))
return json.dumps(results, indent=4, sort_keys=False)
def test_email_cim_to_stix(self):
tag = "email"
count = 3
time = "2018-08-21T15:11:55.000+00:00"
src_user = "******"
subject = "Test Subject"
multi = "False"
data = {
"tag": tag,
"event_count": count,
"_time": time,
"src_user": src_user,
"subject": subject,
"is_multipart": multi
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
msg_obj = TestTransform.get_first_of_type(objects.values(),
'email-message')
assert (msg_obj is not None), 'email-message object type not found'
assert (msg_obj.keys() == {
'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'
})
assert (msg_obj['subject'] == "Test Subject")
assert (msg_obj['is_multipart'] == False)
sender_ref = msg_obj['sender_ref']
assert (sender_ref in objects
), f"sender_ref with key {msg_obj['sender_ref']} not found"
addr_obj = objects[sender_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
from_ref = msg_obj['from_ref']
assert (
sender_ref
in objects), f"from_ref with key {msg_obj['from_ref']} not found"
addr_obj = objects[from_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
def test_certificate_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
serial = "1234"
version = "1"
sig_algorithm = "md5WithRSAEncryption"
key_algorithm = "rsaEncryption"
issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new"
subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
data = {
"event_count": count,
"_time": time,
"ssl_serial": serial,
"ssl_version": version,
"ssl_signature_algorithm": sig_algorithm,
"ssl_issuer": issuer,
"ssl_subject": subject,
"ssl_hash": ssl_hash,
"ssl_publickey_algorithm": key_algorithm
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
cert_obj = TestTransform.get_first_of_type(objects.values(),
'x509-certificate')
assert (cert_obj is not None), 'x509-certificate object type not found'
assert (cert_obj.keys() == {
'type', 'serial_number', 'version', "signature_algorithm",
"subject_public_key_algorithm", "issuer", "subject", "hashes"
})
assert (cert_obj['serial_number'] == "1234")
assert (cert_obj['version'] == "1")
assert (cert_obj['signature_algorithm'] == "md5WithRSAEncryption")
assert (cert_obj['issuer'] ==
"C=US, ST=California, O=www.example.com, OU=new, CN=new")
assert (
cert_obj['subject'] ==
"C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
)
assert (cert_obj['subject_public_key_algorithm'] == "rsaEncryption")
assert (
cert_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
assert (objects.keys() == set(map(str, range(0, 1))))
def test_unmapped_attribute_alone(self):
data = {"unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
assert(objects == {})
def test_unmapped_attribute_with_mapped_attribute(self):
message = "\"GET /blog HTTP/1.1\" 200 2571"
data = {"message": message, "unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
assert (objects == {})
curr_obj = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'message')
assert (curr_obj is None), 'url object type not found'
def test_unmapped_attribute_with_mapped_attribute(self):
url = "https://example.com"
data = {"url": url, "unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
assert(objects != {})
curr_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert(curr_obj is not None), 'url object type not found'
assert(curr_obj.keys() == {'type', 'value'})
assert(curr_obj['value'] == url)
def test_custom_props(self):
data = {"logsourceid": 126, "qid": 55500004,
"identityip": "0.0.0.0", "magnitude": 4, "logsourcename": "someLogSourceName"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
observed_data = result_bundle['objects'][1]
assert('x_com_ibm_ariel' in observed_data)
custom_props = observed_data['x_com_ibm_ariel']
assert(custom_props['identity_ip'] == data['identityip'])
assert(custom_props['log_source_id'] == data['logsourceid'])
assert(custom_props['qid'] == data['qid'])
assert(custom_props['magnitude'] == data['magnitude'])
assert(custom_props['log_source_name'] == data['logsourcename'])
def test_artifact_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
artifact_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'artifact')
assert (artifact_object is not None), 'artifact object type not found'
assert (artifact_object.keys() == {'type', 'payload_bin'})
assert (artifact_object['type'] == 'artifact')
def test_payload_results(self):
data = {
"src_ip": "169.250.0.1",
"src_port": "1220",
"src_mac": "aa:bb:cc:dd:11:22",
"dest_ip": "127.0.0.1",
"dest_port": "1120",
"dest_mac": "ee:dd:bb:aa:cc:11",
"file_hash":
"741ad92448fd12a089a13c6de49fb204e4693e1d3e9f7715471c292adf8c6bef",
"user": "******",
"url":
"https://wally.fireeye.com/malware_analysis/analyses?maid=1",
"protocol": "tcp",
"_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06",
"_cd": "44:12606114",
"_indextime": "1546960685",
"_raw":
"Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1 spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111 fileHash=41a26255d16d121dc525a6445144b895 proto=tcp request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan cn1=0 externalId=1 cs4Label=link cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames \n",
"_serial": "0",
"_si": ["splunk3-01.internal.resilientsystems.com", "main"],
"_sourcetype": "fe_cef_syslog",
"_time": "2019-01-08T15:18:04.000+00:00",
"event_count": 1
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source,
map_data, [data],
transformers.get_all_transformers(),
options,
callback=hash_type_lookup)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
# somehow breaking the stix validation
# validated_result = validate_instance(observed_data)
# assert(validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
utf8 = "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1 spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111 fileHash=41a26255d16d121dc525a6445144b895 proto=tcp request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan cn1=0 externalId=1 cs4Label=link cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames \n"
assert (observed_data['x_com_splunk_spl']['utf8_payload'] == utf8)
def test_file_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'file')
assert (file_object is not None), 'file object type not found'
assert (file_object.keys() == {'type', 'name'})
assert (file_object['type'] == 'file')
assert (file_object['name'] == 'SubmitDiagInfo')
def test_common_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
assert (result_bundle_identity['id'] == data_source['id'])
assert (result_bundle_identity['name'] == data_source['name'])
assert (result_bundle_identity['identity_class'] ==
data_source['identity_class'])
observed_data = result_bundle_objects[1]
assert (observed_data['id'] is not None)
assert (observed_data['type'] == "observed-data")
assert (
observed_data['created_by_ref'] == result_bundle_identity['id'])
def test_process_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
proc_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'process')
assert (proc_object is not None), 'process object type not found'
assert (proc_object.keys() == {
'type', 'pid', 'command_line', 'created', 'image_ref',
'creator_user_ref'
})
assert (proc_object['type'] == 'process')
assert (proc_object['pid'] == 609)
assert (proc_object['command_line'] ==
'/System/Library/CoreServices/SubmitDiagInfo')
assert (proc_object['created'] == '2019-04-10T11:33:57.571Z')
image_ref = proc_object['image_ref']
assert (image_ref in objects
), f"dst_ref with key {proc_object['image_ref']} not found"
image_obj = objects[image_ref]
assert (image_obj.keys() == {'type', 'name'})
assert (image_obj['type'] == 'file')
assert (image_obj['name'] == 'SubmitDiagInfo')
creator_user_ref = proc_object['creator_user_ref']
assert (
creator_user_ref in objects
), f"dst_ref with key {proc_object['creator_user_ref']} not found"
creator_user_ref_obj = objects[creator_user_ref]
assert (creator_user_ref_obj.keys() == {'type', 'user_id'})
assert (creator_user_ref_obj['type'] == 'user')
assert (creator_user_ref_obj['user_id'] == '-')
def test_process_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
artifact_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'artifact')
assert (artifact_object is not None), 'artifact object type not found'
assert (artifact_object.keys() == {'type', 'payload_bin'})
assert (artifact_object['type'] == 'artifact')
assert (
artifact_object['payload_bin'] ==
'MTAuNDIuNDIuNDIgLSAtIFswNy9EZWMvMjAxODoxMTowNTowNyArMDEwMF0gIkdFVCAvYmxvZyBIVFRQLzEuMSIgMjAwIDI1NzEgIi0iICJNb3ppbGxhLzUuMCAoTWFjaW50b3NoOyBJbnRlbCBNYWMgT1MgWCAxMF8xNF8wKSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNzAuMC4zNTM4LjEwMiBTYWZhcmkvNTM3LjM2Ig=='
)
def test_network_traffic_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'network-traffic')
assert (nt_object is not None), 'network-traffic object type not found'
assert (nt_object.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_object['type'] == 'network-traffic')
assert (nt_object['src_port'] == 49745)
assert (nt_object['dst_port'] == 443)
assert (nt_object['protocols'] == ['ipv4', 'tcp'])
ip_ref = nt_object['dst_ref']
assert (
ip_ref
in objects), f"dst_ref with key {nt_object['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == '100.101.0.69')
ip_ref = nt_object['src_ref']
assert (
ip_ref
in objects), f"src_ref with key {nt_object['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == '107.0.0.48')
def test_common_prop(self):
data = {
"computer_identity": "12369754-bigdata4545.canlab.ibm.com",
"subQueryID": 1,
"start_time": "1541424881",
"type": "process",
"process_name": "systemd",
"process_id": "1",
"sha256hash":
"9c74c625b2aba7a2e8d8a42e2e94715c355367f7cbfa9bd5404ba52b726792a6",
"sha1hash": "916933045c5c91ebcaa325e7f8302f3a732a0a3d",
"md5hash": "28a9beb86c4d4c31ba572805bea8494f",
"file_path": "/usr/lib/systemd/systemd"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
print(json.dumps(result_bundle, indent=2))
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
assert (result_bundle_identity['id'] == data_source['id'])
assert (result_bundle_identity['name'] == data_source['name'])
assert (result_bundle_identity['identity_class'] ==
data_source['identity_class'])
observed_data = result_bundle_objects[1]
print(observed_data)
assert (observed_data['id'] is not None)
assert (observed_data['type'] == "observed-data")
assert (
observed_data['created_by_ref'] == result_bundle_identity['id'])
assert (observed_data['created'] is not None)
assert (observed_data['first_observed'] is not None)
assert (observed_data['last_observed'] is not None)
def test_change_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
file_bytes = "300"
user = "******"
objPath = "hkey_local_machine\\system\\bar\\foo"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "41a26255d16d121dc525a6445144b895"
file_name = "sample.dll"
file_size = 25536
data = {
"event_count": count, "_time": time, "user": user,
"bytes": file_bytes, "object_path": objPath, "file_path": filePath,
"file_create_time": create_time, "file_modify_time": modify_time,
"file_hash": file_hash, "file_size": file_size, "file_name": file_name
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert(validated_result.is_valid == True)
assert('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
wrk_obj = TestTransform.get_first_of_type(objects.values(), 'windows-registry-key')
assert(wrk_obj is not None)
assert(wrk_obj.keys() == {'type', 'key'})
assert(wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo")
user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account')
assert(user_obj is not None), 'user-account object type not found'
assert(user_obj.keys() == {'type', 'account_login', 'user_id'})
assert(user_obj['account_login'] == "ibm_user")
assert(user_obj['user_id'] == "ibm_user")
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert(file_obj is not None), 'file object type not found'
assert(file_obj.keys() == {'type', 'parent_directory_ref', 'created', 'modified', 'hashes', 'name', 'size'})
assert(file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert(file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert(file_obj['name'] == "sample.dll")
assert(file_obj['size'] == 25536)
assert (file_obj['hashes']['MD5'] == "41a26255d16d121dc525a6445144b895")
dir_ref = file_obj['parent_directory_ref']
assert(dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert(dir_obj is not None), 'directory object type not found'
assert(dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert(dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert(dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
print(objects.keys())
print(result_bundle_objects)
assert(objects.keys() == set(map(str, range(0, 5))))
def test_at_props(self):
transformer = None
actio = "actioniam-identity.serviceid-apikey.login"
isoti = "2018-07-16T15:00+0000"
alcht = "alchtenida:560ea90982962784957b94135af14810"
tarid = "taridcrn:v1:bluemix:public:iam-identity::a/560ea90982962784957b94135af14810::apikey:ApiKey-b1ebb918c"
tarna = "auto-generated-apikey-4df4db90-06ef-4efa-9d3f-bc7ccf247326"
event = "d9b5a3a9-8fb8-4d6d-90cd-0a50f4842c6a"
messa = "message is a very long string"
activ = "activity"
sampl = "sample-response"
alcha = "alchaccid560ea90982962784957b"
initi = "iam-ServiceId-f48385a1"
initn = "ServiceId-f48385a1"
initc = "apikey"
data = {
# "initiator_name": initn,
# "initiator_id": initi,
"ALCH_ACCOUNT_ID": alcha,
"responseData": sampl,
"eventType": activ,
"message": messa,
"type": "ActivityTracker",
"event_uuid": event,
"tags": [],
# "target_name": tarna,
# "target_id": tarid,
"ALCH_TENANT_ID": alcht,
"logmet_cluster": "topic3-elasticsearch_3",
"@timestamp": "2018-07-16T15:00:03.062Z",
"typeURI": "http://schemas.dmtf.org/cloud/audit/1.0/event",
"@version": "1",
"eventTime": isoti,
"action": actio,
"requestData": "requestdata",
"outcome": "success"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
print (result_bundle)
observed_data = result_bundle['objects'][1]
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
# Test that each data element is properly mapped and input into the STIX JSON
for key, value in objects.items():
assert(int(key) in list(range(0, len(objects))))
if(value['type'] == 'initiator_credential_type'):
assert(value['value'] == initc), "Wrong value returned " + key + ":" + str(value)
assert(True)
elif(value['type'] == 'initiator_name'):
assert(value['value'] == initn), "Wrong value returned " + \
key + ":" + str(value)
elif(value['type'] == 'initiator_id'):
assert(
value['value'] == initi), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'ALCH_ACCOUNT_ID'):
assert(
value['value'] == alcha), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'message'):
assert(
value['value'] == messa), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'event_uuid'):
assert(
value['value'] == event), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'ALCH_TENANT_ID'):
assert(
value['value'] == alcht), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'action'):
assert(
value['value'] == actio), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'eventTime'):
assert(
value['value'] == isoti), "Wrong value returned " + key + ":" + str(value)
else:
assert(False), "Returned a non-mapped value " + \
key + ":" + str(value)
def test_cybox_observables(self):
# transformer = None
payload = "SomeBase64Payload"
user_id = "someuserid2018"
url = "https://example.com"
domain = "example.com"
source_ip = "127.0.0.1"
destination_ip = "255.255.255.1"
data = {"sourceip": source_ip, "destinationip": destination_ip, "url": url,
"domain": domain, "payload": payload, "username": user_id, "protocol": 'TCP', "sourceport": 3000, "destinationport": 2000}
# data = {"Network": {"A" : source_ip}, "destinationip": destination_ip, "url": url,
# "domain": domain, "payload": payload, "username": user_id, "protocol": 'TCP', "sourceport": 3000, "destinationport": 2000}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
# Test that each data element is properly mapped and input into the STIX JSON
for key, value in objects.items():
assert(int(key) in list(range(0, len(objects))))
# Todo: handle case where there is both a source and destination ip, there will be more than one ipv4-addr
if(value['type'] == 'ipv4-addr'):
# assert(
# value['value'] == source_ip), "Wrong value returned " + key + ":" + str(value)
assert(True)
elif(value['type'] == 'url'):
assert(value['value'] == url), "Wrong value returned " + \
key + ":" + str(value)
elif(value['type'] == 'domain-name'):
assert(
value['value'] == domain), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'artifact'):
assert(
value['payload_bin'] == payload), "Wrong value returned " + key + ":" + str(value)
elif(value['type'] == 'user-account'):
assert(
value['user_id'] == user_id), "Wrong value returned " + key + ":" + str(value)
# Todo: should not be returned since the address passed in isn't ipv6, still needs to be fixed in logic
elif(value['type'] == 'ipv6-addr'):
# assert(
# value['value'] == source_ip), "Wrong value returned " + key + ":" + str(value)
assert(True)
elif(value['type'] == 'network-traffic'):
assert(int(value['src_ref']) in list(
range(0, len(objects)))), "Wrong value returned " + key + ":" + str(value)
assert(type(value['src_ref'])
is str), "Reference value should be a string"
assert(int(value['dst_ref']) in list(
range(0, len(objects)))), "Wrong value returned " + key + ":" + str(value)
assert(type(value['dst_ref'])
is str), "Reference value should be a string"
assert(value['protocols'] == ['tcp'])
assert(value['src_port'] == 3000)
assert(value['dst_port'] == 2000)
else:
assert(False), "Returned a non-mapped value " + \
key + ":" + str(value)
def test_change_cb_binary_api_results_to_stix(self):
data = json.loads("""
{
"terms": [
"md5:F5AE03DE0AD60F5B17B82F2CD68402FE"
],
"total_results": 1,
"highlights": [
{
"name": "PREPREPREF5AE03DE0AD60F5B17B82F2CD68402FEPOSTPOSTPOST",
"ids": [
"F5AE03DE0AD60F5B17B82F2CD68402FE"
]
}
],
"facets": {},
"results": [
{
"host_count": 13,
"alliance_updated_srstrust": "2016-09-04T04:59:53Z",
"original_filename": "Cmd.Exe.MUI",
"legal_copyright": "\u00a9 Microsoft Corporation. All rights reserved.",
"digsig_result": "Signed",
"observed_filename": [
"c:\\\\windows\\\\system32\\\\cmd.exe"
],
"product_version": "6.3.9600.16384",
"alliance_score_srstrust": -100,
"watchlists": [
{
"wid": "5",
"value": "2016-10-19T10:20:05.424Z"
}
],
"facet_id": 431419,
"copied_mod_len": 357376,
"server_added_timestamp": "2016-10-19T10:00:25.734Z",
"digsig_sign_time": "2014-11-07T08:02:00Z",
"orig_mod_len": 357376,
"alliance_data_srstrust": [
"f5ae03de0ad60f5b17b82f2cd68402fe"
],
"is_executable_image": true,
"is_64bit": true,
"md5": "F5AE03DE0AD60F5B17B82F2CD68402FE",
"digsig_publisher": "Microsoft Corporation",
"endpoint": [
"ADTWO|24",
"ADONE|26",
"CERT|27",
"REPO|29",
"adone|26",
"cert|27",
"adtwo|24",
"iestestmachine3|53",
"iestestmachine0|52",
"iestestmachine1|54"
],
"group": [ "CTF Lab", "Default Group", "ctf lab", "default group" ],
"event_partition_id": [ 97777295491072, 97794283536384, 97811271778304, 97828260020224, 97845247737856, 97862235979776, 97879224221696, 97896211152896, 97913199394816 ],
"digsig_result_code": "0",
"file_version": "6.3.9600.16384 (winblue_rtm.130821-1623)",
"signed": "Signed",
"alliance_link_srstrust": "https://services.bit9.com/Services/extinfo.aspx?ak=b8b4e631d4884ad1c56f50e4a5ee9279&sg=0313e1735f6cec221b1d686bd4de23ee&md5=f5ae03de0ad60f5b17b82f2cd68402fe",
"company_name": "Microsoft Corporation",
"internal_name": "cmd",
"timestamp": "2016-10-19T10:00:25.734Z",
"cb_version": 624,
"os_type": "Windows",
"file_desc": "Windows Command Processor",
"product_name": "Microsoft\u00ae Windows\u00ae Operating System",
"last_seen": "2019-01-14T03:19:05.687Z"
}
],
"elapsed": 0.02470088005065918,
"start": 0
}""")
binary_interface = carbonblack_translator.Translator()
binary_map_file = open(binary_interface.mapping_filepath).read()
binary_map_data = json.loads(binary_map_file)
results = data["results"]
result_bundle = json_to_stix_translator.convert_to_stix(data_source, binary_map_data, results, transformers.get_all_transformers(), options)
print(result_bundle)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'file')
file_obj = curr_obj # used in later test
assert(curr_obj is not None), 'file object type not found'
assert(curr_obj.keys() == {'type', 'name', 'created', 'hashes'})
assert(curr_obj['name'] =="Cmd.Exe.MUI")
assert(curr_obj['hashes']['MD5'] == "F5AE03DE0AD60F5B17B82F2CD68402FE")
def test_merge_results_mixed_to_stix(self):
process_data = json.loads("""
{
"terms": [
"process_name:cmd.exe"
],
"results": [
{
"process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41",
"sensor_id": 50,
"filtering_known_dlls": false,
"modload_count": 16,
"parent_unique_id": "00000032-0000-0a04-01d4-8bc245c6c9e6-000000000001",
"emet_count": 0,
"cmdline": "cmd /c \\"\\"C:\\\\ProgramData\\\\VMware\\\\VMware CAF\\\\pme\\\\\\\\config\\\\..\\\\scripts\\\\is-listener-running.bat\\" \\"",
"filemod_count": 0,
"id": "00000032-0000-0888-01d4-95e3b558aacb",
"parent_name": "managementagenthost.exe",
"parent_md5": "000000000000000000000000000000",
"group": "mdr redlab",
"parent_id": "00000032-0000-0a04-01d4-8bc245c6c9e6",
"hostname": "redlab-vuln2",
"last_update": "2018-12-17T08:37:13.396Z",
"start": "2018-12-17T08:37:13.318Z",
"comms_ip": 212262914,
"regmod_count": 0,
"interface_ip": 183439305,
"process_pid": 2184,
"username": "******",
"terminated": false,
"process_name": "cmd.exe",
"emet_config": "",
"last_server_update": "2019-02-01T18:44:10.53Z",
"path": "c:\\\\windows\\\\system32\\\\cmd.exe",
"netconn_count": 0,
"parent_pid": 2564,
"crossproc_count": 2,
"segment_id": 1549046650410,
"host_type": "workstation",
"processblock_count": 0,
"os_type": "windows",
"childproc_count": 8,
"unique_id": "00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a"
},
{
"process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41",
"sensor_id": 50,
"filtering_known_dlls": false,
"modload_count": 16,
"parent_unique_id": "00000032-0000-0a04-01d4-8bc245c6c9e6-000000000001",
"emet_count": 0,
"cmdline": "cmd /c \\"\\"C:\\\\ProgramData\\\\VMware\\\\VMware CAF\\\\pme\\\\\\\\config\\\\..\\\\scripts\\\\is-listener-running.bat\\" \\"",
"filemod_count": 0,
"id": "00000032-0000-0888-01d4-95e3b558aacb",
"parent_name": "managementagenthost.exe",
"parent_md5": "000000000000000000000000000000",
"group": "mdr redlab",
"parent_id": "00000032-0000-0a04-01d4-8bc245c6c9e6",
"hostname": "redlab-vuln2",
"last_update": "2018-12-17T08:37:13.396Z",
"start": "2018-12-17T08:37:13.318Z",
"comms_ip": 212262914,
"regmod_count": 0,
"interface_ip": 183439305,
"process_pid": 2184,
"username": "******",
"terminated": false,
"process_name": "cmd.exe",
"alliance_data_attackframework": [
"565594"
],
"emet_config": "",
"last_server_update": "2019-02-01T18:50:32.875Z",
"path": "c:\\\\windows\\\\system32\\\\cmd.exe",
"alliance_score_attackframework": 1,
"netconn_count": 0,
"parent_pid": 2564,
"crossproc_count": 2,
"alliance_link_attackframework": "https://attack.mitre.org/wiki/Technique/T1082",
"segment_id": 1549047032875,
"watchlists": [
{
"segments_hit": [
1549046650410
],
"wid": "1154",
"value": "2019-02-01T18:50:06.003Z"
}
],
"host_type": "workstation",
"processblock_count": 0,
"alliance_updated_attackframework": "2018-10-16T20:15:04Z",
"os_type": "windows",
"childproc_count": 8,
"unique_id": "00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b"
}
],
"elapsed": 0.023807048797607422,
"comprehensive_search": true,
"all_segments": true,
"total_results": 77835,
"highlights": [
{
"name": "PREPREPREcmd.exePOSTPOSTPOST",
"ids": [
"00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a",
"00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b"
]
},
{
"name": "C:\\\\Windows\\\\system32\\\\PREPREPREcmd.exePOSTPOSTPOST /S /D /c\\" echo\\"",
"ids": [
"00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a"
]
},
{
"name": "c:\\\\windows\\\\system32\\\\PREPREPREcmd.exePOSTPOSTPOST",
"ids": [
"00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a",
"00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b"
]
}
],
"facets": {},
"tagged_pids": {},
"start": 0,
"incomplete_results": false,
"filtered": {}
}
""")
binary_data = json.loads("""
{
"terms": [
"observed_filename:notepad.exe"
],
"total_results": 10,
"highlights": [
{
"name": "c:\\\\windows\\\\system32\\\\PREPREPREnotepad.exePOSTPOSTPOST",
"ids": [
"FC2EA5BD5307D2CFA5AAA38E0C0DDCE9",
"959A31D0CD013CEA0C66DB7C03BCBDDF"
]
}
],
"facets": {},
"results": [
{
"host_count": 4,
"alliance_updated_srstrust": "2017-11-05T07:05:38Z",
"original_filename": "NOTEPAD.EXE",
"legal_copyright": "\\u00a9 Microsoft Corporation. All rights reserved.",
"digsig_result": "Signed",
"observed_filename": [
"c:\\\\windows\\\\system32\\\\notepad.exe"
],
"product_version": "6.3.9600.17930",
"alliance_score_srstrust": -100,
"watchlists": [
{
"wid": "5",
"value": "2017-03-14T10:10:05.217Z"
}
],
"facet_id": 2272,
"copied_mod_len": 221184,
"server_added_timestamp": "2017-03-14T10:04:35.779Z",
"digsig_sign_time": "2015-07-11T00:18:00Z",
"orig_mod_len": 221184,
"alliance_data_srstrust": [
"fc2ea5bd5307d2cfa5aaa38e0c0ddce9"
],
"is_executable_image": true,
"is_64bit": true,
"md5": "FC2EA5BD5307D2CFA5AAA38E0C0DDCE9",
"digsig_publisher": "Microsoft Corporation",
"endpoint": [
"REPO|29",
"VSPHERE|28",
"vsphere|28",
"iestestmachine1|54"
],
"group": [
"CTF Lab",
"ctf lab",
"default group"
],
"event_partition_id": [
97777295491072,
98439833845760,
98847548112896,
99679970852864,
101310831263744
],
"digsig_result_code": "0",
"file_version": "6.3.9600.17930 (winblue_ltsb.150709-0600)",
"signed": "Signed",
"alliance_link_srstrust": "https://services.bit9.com/Services/extinfo.aspx?ak=b8b4e631d4884ad1c56f50e4a5ee9279&sg=0313e1735f6cec221b1d686bd4de23ee&md5=fc2ea5bd5307d2cfa5aaa38e0c0ddce9",
"company_name": "Microsoft Corporation",
"internal_name": "Notepad",
"timestamp": "2017-03-14T10:04:35.779Z",
"cb_version": 624,
"os_type": "Windows",
"file_desc": "Notepad",
"product_name": "Microsoft\\u00ae Windows\\u00ae Operating System",
"last_seen": "2018-12-29T12:41:54.355Z"
},
{
"host_count": 1,
"original_filename": "NOTEPAD.EXE",
"legal_copyright": "\\u00a9 Microsoft Corporation. All rights reserved.",
"digsig_result": "Signed",
"observed_filename": [
"c:\\\\windows\\\\system32\\\\notepad.exe"
],
"product_version": "6.3.9600.17415",
"watchlists": [
{
"wid": "5",
"value": "2017-04-12T21:10:04.604Z"
}
],
"facet_id": 87425,
"copied_mod_len": 221184,
"server_added_timestamp": "2017-04-12T21:06:15.216Z",
"digsig_sign_time": "2014-11-07T07:55:00Z",
"orig_mod_len": 221184,
"is_executable_image": true,
"is_64bit": true,
"md5": "959A31D0CD013CEA0C66DB7C03BCBDDF",
"digsig_publisher": "Microsoft Corporation",
"endpoint": [
"REPO|31"
],
"group": [
"Default Group"
],
"event_partition_id": [
97777295491072
],
"digsig_result_code": "0",
"file_version": "6.3.9600.17415 (winblue_r4.141028-1500)",
"signed": "Signed",
"company_name": "Microsoft Corporation",
"internal_name": "Notepad",
"timestamp": "2017-04-12T21:06:15.216Z",
"cb_version": 610,
"os_type": "Windows",
"file_desc": "Notepad",
"product_name": "Microsoft\\u00ae Windows\\u00ae Operating System",
"last_seen": "2017-04-12T21:10:06.095Z"
}
],
"elapsed": 0.011963844299316406,
"start": 0
}
""")
results = process_data["results"] + binary_data["results"] # we assume the data pipeline will combine the results in a list
result_bundle = json_to_stix_translator.convert_to_stix(data_source, map_data, results, transformers.get_all_transformers(), options)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
assert(len(result_bundle_objects) == 5)
objects = result_bundle_objects[1]['objects']
types = [o.get('type') for o in objects.values()]
assert (types == ['file', 'process', 'file', 'process', 'domain-name', 'ipv4-addr', 'network-traffic', 'ipv4-addr', 'user-account'])
objects = result_bundle_objects[4]['objects']
types = [o.get('type') for o in objects.values()]
assert (types == ['file'])
def test_cybox_observables(self):
payload = "SomeBase64Payload"
user_id = "someuserid2018"
url = "https://example.com"
source_ip = "fd80:655e:171d:30d4:fd80:655e:171d:30d4"
destination_ip = "255.255.255.1"
file_name = "somefile.exe"
source_mac = "00-00-5E-00-53-00"
destination_mac = "00-00-5A-00-55-01"
data = {
"sourceip": source_ip,
"destinationip": destination_ip,
"url": url,
"base64_payload": payload,
"username": user_id,
"protocol": 'TCP',
"sourceport": "3000",
"destinationport": 2000,
"filename": file_name,
"domainname": url,
"sourcemac": source_mac,
"destinationmac": destination_mac
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_object = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_object is not None), 'network-traffic object type not found'
assert (nt_object.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_object['src_port'] == 3000)
assert (nt_object['dst_port'] == 2000)
assert (nt_object['protocols'] == ['tcp'])
ip_ref = nt_object['dst_ref']
assert (
ip_ref
in objects), f"dst_ref with key {nt_object['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == destination_ip)
ip_ref = nt_object['src_ref']
assert (
ip_ref
in objects), f"src_ref with key {nt_object['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == source_ip)
curr_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert (curr_obj is not None), 'url object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == url)
curr_obj = TestTransform.get_first_of_type(objects.values(),
'artifact')
assert (curr_obj is not None), 'artifact object type not found'
assert (curr_obj.keys() == {'type', 'payload_bin'})
assert (curr_obj['payload_bin'] == payload)
curr_obj = TestTransform.get_first_of_type(objects.values(),
'user-account')
assert (curr_obj is not None), 'user-account object type not found'
assert (curr_obj.keys() == {'type', 'user_id'})
assert (curr_obj['user_id'] == user_id)
curr_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (curr_obj is not None), 'file object type not found'
assert (curr_obj.keys() == {'type', 'name'})
assert (curr_obj['name'] == file_name)
curr_obj = TestTransform.get_first_of_type(objects.values(),
'domain-name')
assert (curr_obj is not None), 'domain-name object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == 'example.com')
assert (objects.keys() == set(map(str, range(0, 10))))
def test_change_cb_process_api_results_to_stix(self):
data = json.loads("""
{
"terms": [
"process_name:cmd.exe",
"start:[2019-01-22T00:00:00 TO *]"
],
"results": [
{
"process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41",
"sensor_id": 49,
"filtering_known_dlls": true,
"modload_count": 3,
"parent_unique_id": "00000031-0000-09cc-01d4-b1e61979dd7c-000000000001",
"emet_count": 0,
"alliance_score_srstrust": -100,
"cmdline": "C:\\\\Windows\\\\system32\\\\cmd.exe /c tasklist",
"alliance_updated_srstrust": "2018-04-05T16:04:34Z",
"filemod_count": 0,
"id": "00000031-0000-0768-01d4-b1e6197c3edd",
"parent_name": "cmd.exe",
"parent_md5": "000000000000000000000000000000",
"group": "lab1",
"parent_id": "00000031-0000-09cc-01d4-b1e61979dd7c",
"hostname": "lab1-host1",
"last_update": "2019-01-22T00:04:52.937Z",
"start": "2019-01-22T00:04:52.875Z",
"alliance_link_srstrust": "https://example.com",
"comms_ip": -1051309706,
"regmod_count": 0,
"interface_ip": 183439304,
"process_pid": 1896,
"username": "******",
"terminated": true,
"alliance_data_srstrust": [
"5746bd7e255dd6a8afa06f7c42c1ba41"
],
"process_name": "cmd.exe",
"emet_config": "",
"last_server_update": "2019-01-22T00:07:07.064Z",
"path": "c:\\\\windows\\\\system32\\\\cmd.exe",
"netconn_count": 0,
"parent_pid": 2508,
"crossproc_count": 2,
"segment_id": 1548115627056,
"host_type": "workstation",
"processblock_count": 0,
"os_type": "windows",
"childproc_count": 4,
"unique_id": "00080031-0000-0748-01d4-b1e61c7c3edd-016872e1cb30"
}
],
"elapsed": 0.05147600173950195,
"comprehensive_search": true,
"all_segments": true,
"total_results": 1,
"highlights": [],
"facets": {},
"tagged_pids": {},
"start": 0,
"incomplete_results": false,
"filtered": {}
}""")
results = data["results"]
result_bundle = json_to_stix_translator.convert_to_stix(data_source, map_data, results, transformers.get_all_transformers(), options)
print(result_bundle)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'file')
file_obj = curr_obj # used in later test
assert(curr_obj is not None), 'file object type not found'
assert(curr_obj.keys() == {'type', 'name', 'hashes'})
assert(curr_obj['name'] == "cmd.exe")
assert(curr_obj['hashes']['MD5'] == "5746bd7e255dd6a8afa06f7c42c1ba41")
curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'user-account')
user_obj = curr_obj # used in later test
assert(curr_obj is not None), 'user-account object type not found'
assert(curr_obj.keys() == {'type', 'user_id'})
assert(curr_obj['user_id'] == "SYSTEM")
curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'network-traffic')
network_obj = curr_obj # used in later test
assert(curr_obj is not None), 'network-traffic object type not found'
assert(curr_obj.keys() == {'type', 'src_ref', 'dst_ref'})
assert(objects[curr_obj['src_ref']]['value'] == "10.239.15.200")
assert(objects[curr_obj['dst_ref']]['value'] == "193.86.73.118")
curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'process')
assert(curr_obj is not None), 'process object type not found'
assert(curr_obj.keys() == {'type', 'command_line', 'creator_user_ref', 'binary_ref', 'parent_ref', 'created', 'name', 'pid', 'opened_connection_refs'})
assert(curr_obj['command_line'] == "C:\\Windows\\system32\\cmd.exe /c tasklist")
assert(curr_obj['created'] == "2019-01-22T00:04:52.875Z")
assert(curr_obj['pid'] == 1896)
assert(network_obj == objects[curr_obj['opened_connection_refs'][0]]), 'open_connection_refs does not point to the correct object'
assert(file_obj == objects[curr_obj['binary_ref']]), 'process binary_ref does not point to the correct object'
assert(user_obj == objects[curr_obj['creator_user_ref']]), 'process creator_user_ref does not point to the correct object'
parent_index = curr_obj['parent_ref']
curr_obj = objects[parent_index]
assert(curr_obj is not None)
assert(curr_obj.keys() == {'type', 'pid', 'name', 'binary_ref'})
assert(curr_obj['pid'] == 2508)
assert(curr_obj['name'] == "cmd.exe")
assert(objects[curr_obj['binary_ref']]['name'] == "cmd.exe")
def test_process_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
pid = 0
name = "test_process"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
file_name = "sample.dll"
file_size = 25536
data = {
"event_count": count, "_time": time, "user": user,
"process_name": name, "process_id": pid, "file_path": filePath,
"file_create_time": create_time, "file_modify_time": modify_time,
"file_hash": file_hash, "file_size": file_size, "file_name": file_name
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert(validated_result.is_valid == True)
assert('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
proc_obj = TestTransform.get_first_of_type(objects.values(), 'process')
assert(proc_obj is not None), 'process object type not found'
assert(proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref'})
assert(proc_obj['name'] == "test_process")
assert(proc_obj['pid'] == 0)
user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account')
assert(user_obj is not None), 'user-account object type not found'
assert(user_obj.keys() == {'type', 'account_login', 'user_id'})
assert(user_obj['account_login'] == "test_user")
assert(user_obj['user_id'] == "test_user")
bin_ref = proc_obj['binary_ref']
assert(bin_ref in objects), f"binary_ref with key {proc_obj['binary_ref']} not found"
file_obj = objects[bin_ref]
assert(file_obj is not None), 'file object type not found'
assert(file_obj.keys() == {'type', 'parent_directory_ref', 'created', 'modified', 'size', 'name', 'hashes'})
assert(file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert(file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert(file_obj['name'] == "sample.dll")
assert(file_obj['size'] == 25536)
assert (file_obj['hashes']['SHA-256'] == "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
dir_ref = file_obj['parent_directory_ref']
assert(dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert(dir_obj is not None), 'directory object type not found'
assert(dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert(dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert(dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert(objects.keys() == set(map(str, range(0, 4))))
def test_cim_to_stix_no_tags(self):
data = {"src_ip": "169.250.0.1", "src_port": "1220", "src_mac": "aa:bb:cc:dd:11:22",
"dest_ip": "127.0.0.1", "dest_port": "1120", "dest_mac": "ee:dd:bb:aa:cc:11",
"file_hash": "cf23df2207d99a74fbe169e3eba035e633b65d94",
"user": "******", "url": "https://wally.fireeye.com/malware_analysis/analyses?maid=1",
"protocol": "tcp", "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114",
"_indextime": "1546960685",
"_raw": "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|"
"malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1"
" spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111"
" fileHash=41a26255d16d121dc525a6445144b895 proto=tcp "
"request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/"
"58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo"
" cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan"
" cn1=0 externalId=1 cs4Label=link "
"cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly"
" cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames",
"_serial": "0", "_si": ["splunk3-01.internal.resilientsystems.com", "main"],
"_sourcetype": "fe_cef_syslog", "_time": "2019-01-08T15:18:04.000+00:00", "event_count": 1
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup)
assert(result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert(validated_result.is_valid == True)
assert('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic')
assert(nt_obj is not None), 'network-traffic object type not found'
assert(nt_obj.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'})
assert(nt_obj['src_port'] == 1220)
assert(nt_obj['dst_port'] == 1120)
assert(nt_obj['protocols'] == ['tcp'])
nt_obj_2 = objects['2']
assert (nt_obj_2 is not None), 'network-traffic object type not found'
assert (nt_obj_2.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'})
assert (nt_obj_2['src_port'] == 1220)
assert (nt_obj_2['dst_port'] == 1120)
assert (nt_obj_2['protocols'] == ['tcp'])
mac_ref = nt_obj_2['dst_ref']
assert(mac_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found"
mac_obj = objects[mac_ref]
assert(mac_obj.keys() == {'type', 'value'})
assert(mac_obj['type'] == 'mac-addr')
assert(mac_obj['value'] == 'ee:dd:bb:aa:cc:11')
mac_ref = nt_obj_2['src_ref']
assert(mac_ref in objects), "src_ref with key {nt_obj['dst_ref']} not found"
mac_obj = objects[mac_ref]
assert(mac_obj.keys() == {'type', 'value'})
assert(mac_obj['type'] == 'mac-addr')
assert(mac_obj['value'] == 'aa:bb:cc:dd:11:22')
ip_ref = nt_obj['dst_ref']
assert(ip_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert(ip_obj.keys() == {'type', 'value'})
assert(ip_obj['type'] == 'ipv4-addr')
assert(ip_obj['value'] == '127.0.0.1')
ip_ref = nt_obj['src_ref']
assert(ip_ref in objects), "src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert(ip_obj.keys() == {'type', 'value'})
assert(ip_obj['type'] == 'ipv4-addr')
assert(ip_obj['value'] == '169.250.0.1')
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {'type', 'hashes'})
assert (file_obj['hashes']['SHA-1'] == "cf23df2207d99a74fbe169e3eba035e633b65d94")
user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account')
assert (user_obj is not None), 'user object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "sname")
assert (user_obj['user_id'] == "sname")
url_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert (url_obj is not None), 'url object type not found'
assert (url_obj.keys() == {'type', 'value'})
assert (url_obj['value'] == "https://wally.fireeye.com/malware_analysis/analyses?maid=1")
domain_obj = TestTransform.get_first_of_type(objects.values(), 'domain-name')
assert (domain_obj is not None), 'domain object type not found'
assert (domain_obj.keys() == {'type', 'value'})
assert (domain_obj['value'] == "wally.fireeye.com")
payload_obj = TestTransform.get_first_of_type(objects.values(), 'artifact')
assert (payload_obj is not None), 'payload object type not found'
assert (payload_obj.keys() == {'type', 'payload_bin'})
payload = 'SmFuIDA4IDIwMTkgMTU6MTg6MDQgMTkyLjE2OC4zMy4xMzEgZmVub3RpZnktMi5hbGVydDogQ0VGOjB8RmlyZUV5ZXxNQV' \
'N8Ni4yLjAuNzQyOTh8TU98bWFsd2FyZS1vYmplY3R8NHxydD1KYW4gMDggMjAxOSAxNToxODowNCBaIHNyYz0xNjkuMjUw' \
'LjAuMSBkcHQ9MTEyMCBkc3Q9MTI3LjAuMC4xIHNwdD0xMjIwIHNtYWM9QUE6QkI6Q0M6REQ6MTE6MjIgZG1hYz1FRTpERD' \
'pCQjpBQTpDQzoxMSBjbjJMYWJlbD1zaWQgY24yPTExMSBmaWxlSGFzaD00MWEyNjI1NWQxNmQxMjFkYzUyNWE2NDQ1MTQ0' \
'Yjg5NSBwcm90bz10Y3AgcmVxdWVzdD1odHRwOi8vcWEtc2VydmVyLmVuZy5maXJlZXllLmNvbS9RRS9Ob3RpZmljYXRpb2' \
'5QY2Fwcy81OC4yNTMuNjguMjlfODAtMTkyLjE2OC44NS4xMjhfMTE2NS0yMTE5MjgzMTA5X1QuZXhlIGNzM0xhYmVsPW9z' \
'aW5mbyBjczM9TWljcm9zb2Z0IFdpbmRvd3M3IFByb2Zlc3Npb25hbCA2LjEgc3AxIGR2Y2hvc3Q9d2FsbHkgZHZjPTEwLj' \
'IuMTAxLjEwMSBjbjFMYWJlbD12bGFuIGNuMT0wIGV4dGVybmFsSWQ9MSBjczRMYWJlbD1saW5rIGNzND1odHRwczovL3dh' \
'bGx5LmZpcmVleWUuY29tL21hbHdhcmVfYW5hbHlzaXMvYW5hbHlzZXM/bWFpZD0xIGNzMkxhYmVsPWFub21hbHkgY3MyPW' \
'1pc2MtYW5vbWFseSBjczFMYWJlbD1zbmFtZSBjczE9RkVfVVBYO1Ryb2phbi5QV1MuT25saW5lR2FtZXM='
assert (payload_obj['payload_bin'] == payload)
