def test_common_prop(self):
data = {"starttime": 1531169112, "eventcount": 5}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
assert (result_bundle_identity['id'] == data_source['id'])
assert (result_bundle_identity['name'] == data_source['name'])
assert (result_bundle_identity['identity_class'] ==
data_source['identity_class'])
observed_data = result_bundle_objects[1]
assert (observed_data['id'] is not None)
assert (observed_data['type'] == "observed-data")
assert (
observed_data['created_by_ref'] == result_bundle_identity['id'])
assert (observed_data['number_observed'] == 5)
assert (observed_data['created'] is not None)
assert (observed_data['modified'] is not None)
assert (observed_data['first_observed'] is not None)
assert (observed_data['last_observed'] is not None)
def translate_results(self, data_source, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX. Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
self.mapping_json = options['mapping'] if 'mapping' in options else {}
json_data = json.loads(data)
data_source = json.loads(data_source)
if (not self.mapping_json):
# If no mapping is passed in then we will use the default to_stix_map in the qradar module
map_file = open(self.default_mapping_file_path).read()
map_data = json.loads(map_file)
else:
map_data = self.mapping_json
results = json_to_stix_translator.convert_to_stix(
data_source, map_data, json_data,
transformers.get_all_transformers(), options)
return json.dumps(results, indent=4, sort_keys=False)
def translate_results(self, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX. Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
json_data = json.loads(data)
if (mapping is None):
# If no mapping is passed in then we will use the default to_stix_map in the qradar module
map_file = open(self.default_mapping_file_path).read()
map_data = json.loads(map_file)
else:
map_data = json.loads(mapping)
# todo: make datasource id/name dynamic
datasource = {
'id': '7c0de425-33bf-46be-9e38-e42319e36d95',
'name': 'events'
}
results = json_to_stix_translator.convert_to_stix(
datasource, map_data, json_data,
transformers.get_all_transformers(), options)
return json.dumps(results, indent=4, sort_keys=True)
def test_email_cim_to_stix(self):
tag = "email"
count = 3
time = "2018-08-21T15:11:55.000+00:00"
src_user = "******"
subject = "Test Subject"
multi = "False"
data = {
"tag": tag,
"event_count": count,
"_time": time,
"src_user": src_user,
"subject": subject,
"is_multipart": multi
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
msg_obj = TestTransform.get_first_of_type(objects.values(),
'email-message')
assert (msg_obj is not None), 'email-message object type not found'
assert (msg_obj.keys() == {
'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'
})
assert (msg_obj['subject'] == "Test Subject")
assert (msg_obj['is_multipart'] == False)
sender_ref = msg_obj['sender_ref']
assert (sender_ref in objects
), f"sender_ref with key {msg_obj['sender_ref']} not found"
addr_obj = objects[sender_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
from_ref = msg_obj['from_ref']
assert (
sender_ref
in objects), f"from_ref with key {msg_obj['from_ref']} not found"
addr_obj = objects[from_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
def test_to_string_transformer(self):
datasource = {'id': '123', 'name': 'sourcename'}
map_data = {
"destinationip": [{
"key": "ipv4-addr.value",
"type": "value"
}, {
"key": "ipv6-addr.value",
"type": "value"
}, {
"key": "network-traffic.dst_ref",
"type": "reference",
"linked": "nt",
"transformer": "ToString"
}],
"sourceip": [{
"key": "ipv4-addr.value",
"type": "value"
}, {
"key": "ipv6-addr.value",
"type": "value"
}, {
"key": "network-traffic.src_ref",
"type": "reference",
"linked": "nt",
"transformer": "ToString"
}]
}
options = {}
data = [{"sourceip": "1.1.1.1", "destinationip": "2.2.2.2"}]
result = json_to_stix_translator.convert_to_stix(
datasource, map_data, data, transformers.get_all_transformers(),
options)[0]
assert (result is not None)
assert ('objects' in result)
objects = result['objects']
assert (len(objects) == 3)
assert ('0' in objects) # destinationip
object0 = objects['0']
assert (object0['type'] == 'ipv4-addr')
assert (object0['value'] == "2.2.2.2")
assert ('1' in objects) # sourceip
object1 = objects['1']
assert (object1['type'] == 'ipv4-addr')
assert (object1['value'] == "1.1.1.1")
assert ('2' in objects)
object2 = objects['2']
assert (object2['dst_ref'] == '0')
assert (object2['src_ref'] == '1')
assert (object2['type'] == 'network-traffic')
def test_to_integer_transformer_error(self):
datasource = {'id': '123', 'name': 'sourcename'}
map_data = {
"eventCount": {
"key": "number_observed",
"type": "value",
"transformer": "ToInteger"
},
}
options = {}
data = [{"eventCount": "notaValidNumber"}]
result = json_to_stix_translator.convert_to_stix(
datasource, map_data, data, transformers.get_all_transformers(),
options)[0]
assert (result is not None)
assert ('number_observed' not in result)
def test_to_array_transformer(self):
data_source = {'id': '123', 'name': 'sourcename'}
map_data = {
"destinationport": {
"key": "network-traffic.dst_port",
"cybox": "true",
"linked": "nt",
"type": "value"
},
"sourceport": {
"key": "network-traffic.src_port",
"cybox": "true",
"type": "value",
"linked": "nt"
},
"protocol": {
"key": "network-traffic.protocols",
"cybox": "true",
"type": "value",
"linked": "nt",
"transformer": "ToArray"
}
}
test_data = [{
"protocol": "TCP",
"sourceport": 1,
"destinationport": 2
}]
options = {}
result = json_to_stix_translator.convert_to_stix(
data_source, map_data, test_data,
transformers.get_all_transformers(), options)[0]
assert (result is not None)
assert ('objects' in result)
objects = result['objects']
assert ('0' in objects)
network_traffic = objects['0']
assert (network_traffic['src_port'] == 1)
assert (network_traffic['dst_port'] == 2)
assert (isinstance(network_traffic['protocols'], list))
assert (network_traffic['protocols'][0] == 'tcp')
def test_to_integer_transformer(self):
datasource = {'id': '123', 'name': 'sourcename'}
map_data = {
"eventCount": {
"key": "number_observed",
"type": "value",
"transformer": "ToInteger"
},
}
options = {}
data = [{"eventCount": "5"}]
result = json_to_stix_translator.convert_to_stix(
datasource, map_data, data, transformers.get_all_transformers(),
options)[0]
assert (result is not None)
assert ('objects' in result)
objects = result['objects']
assert (len(objects) == 0)
assert ('number_observed' in result)
assert (result['number_observed'] == 5)
def test_custom_props(self):
data = {
"logsourceid": 126,
"qid": 55500004,
"identityip": "0.0.0.0",
"magnitude": 4,
"logsourcename": "someLogSourceName"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
observed_data = result_bundle['objects'][1]
assert ('x_com_ibm_ariel' in observed_data)
custom_props = observed_data['x_com_ibm_ariel']
assert (custom_props['identity_ip'] == data['identityip'])
assert (custom_props['log_source_id'] == data['logsourceid'])
assert (custom_props['qid'] == data['qid'])
assert (custom_props['magnitude'] == data['magnitude'])
assert (custom_props['log_source_name'] == data['logsourcename'])
def test_cybox_observables(self):
payload = "SomeBase64Payload"
user_id = "someuserid2018"
url = "https://example.com"
source_ip = "fd80:655e:171d:30d4:fd80:655e:171d:30d4"
destination_ip = "255.255.255.1"
file_name = "somefile.exe"
data = {
"sourceip": source_ip,
"destinationip": destination_ip,
"url": url,
"payload": payload,
"username": user_id,
"protocol": 'TCP',
"sourceport": 3000,
"destinationport": 2000,
"filename": file_name
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_object = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_object is not None), 'network-traffic object type not found'
assert (nt_object.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_object['src_port'] == 3000)
assert (nt_object['dst_port'] == 2000)
assert (nt_object['protocols'] == ['tcp'])
ip_ref = nt_object['dst_ref']
assert (
ip_ref
in objects), f"dst_ref with key {nt_object['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == destination_ip)
ip_ref = nt_object['src_ref']
assert (
ip_ref
in objects), f"src_ref with key {nt_object['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == source_ip)
curr_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert (curr_obj is not None), 'url object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == url)
curr_obj = TestTransform.get_first_of_type(objects.values(),
'artifact')
assert (curr_obj is not None), 'artifact object type not found'
assert (curr_obj.keys() == {'type', 'payload_bin'})
assert (curr_obj['payload_bin'] == payload)
curr_obj = TestTransform.get_first_of_type(objects.values(),
'user-account')
assert (curr_obj is not None), 'user-account object type not found'
assert (curr_obj.keys() == {'type', 'user_id'})
assert (curr_obj['user_id'] == user_id)
curr_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (curr_obj is not None), 'file object type not found'
assert (curr_obj.keys() == {'type', 'name'})
assert (curr_obj['name'] == file_name)
assert (objects.keys() == set(map(str, range(0, 7))))
def test_change_cim_to_stix(self):
tag = "change"
count = 1
time = "2018-08-21T15:11:55.000+00:00"
file_bytes = "300"
user = "******"
objPath = "hkey_local_machine\\system\\bar\\foo"
filePath = "C:\\Users\\someuser\\sample.dll"
create_time = "2018-08-15T15:11:55.676+00:00"
modify_time = "2018-08-15T18:10:30.456+00:00"
file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
file_name = "sample.dll"
file_size = 25536
data = {
"tag": tag,
"event_count": count,
"_time": time,
"user": user,
"bytes": file_bytes,
"object_path": objPath,
"file_path": filePath,
"file_create_time": create_time,
"file_modify_time": modify_time,
"file_hash": file_hash,
"file_size": file_size,
"file_name": file_name
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
wrk_obj = TestTransform.get_first_of_type(objects.values(),
'windows-registry-key')
assert (wrk_obj
is not None), 'windows-registry-key object type not found'
assert (wrk_obj.keys() == {'type', 'creator_user_ref', 'key'})
assert (wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo")
user_ref = wrk_obj['creator_user_ref']
assert (
user_ref in objects
), f"creator_user_ref with key {wrk_obj['creator_user_ref']} not found"
user_obj = objects[user_ref]
assert (user_obj is not None), 'user-account object type not found'
assert (user_obj.keys() == {'type', 'account_login', 'user_id'})
assert (user_obj['account_login'] == "ibm_user")
assert (user_obj['user_id'] == "ibm_user")
file_obj = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_obj is not None), 'file object type not found'
assert (file_obj.keys() == {
'type', 'parent_directory_ref', 'created', 'modified', 'hashes',
'name', 'size'
})
assert (file_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (file_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (file_obj['name'] == "sample.dll")
assert (file_obj['size'] == 25536)
assert (
file_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
dir_ref = file_obj['parent_directory_ref']
assert (
dir_ref in objects
), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found"
dir_obj = objects[dir_ref]
assert (dir_obj is not None), 'directory object type not found'
assert (dir_obj.keys() == {'type', 'path', 'created', 'modified'})
assert (dir_obj['path'] == "C:\\Users\\someuser\\sample.dll")
assert (dir_obj['created'] == "2018-08-15T15:11:55.676Z")
assert (dir_obj['modified'] == "2018-08-15T18:10:30.456Z")
assert (objects.keys() == set(map(str, range(0, 4))))
def test_network_cim_to_stix(self):
tag = "network"
count = 2
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
dest_ip = "127.0.0.1"
dest_port = "8090"
src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
src_port = "8080"
transport = "http"
data = {
"tag": tag,
"event_count": count,
"_time": time,
"user": user,
"dest_ip": dest_ip,
"dest_port": dest_port,
"src_ip": src_ip,
"src_port": src_port,
"transport": transport
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_obj is not None), 'network-traffic object type not found'
assert (nt_obj.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_obj['src_port'] == 8080)
assert (nt_obj['dst_port'] == 8090)
assert (nt_obj['protocols'] == ['http'])
ip_ref = nt_obj['dst_ref']
assert (ip_ref
in objects), f"dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == dest_ip)
ip_ref = nt_obj['src_ref']
assert (ip_ref
in objects), f"src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == src_ip)
def test_certificate_cim_to_stix(self):
tag = "certificate"
count = 1
time = "2018-08-21T15:11:55.000+00:00"
serial = "1234"
version = "1"
sig_algorithm = "md5WithRSAEncryption"
key_algorithm = "rsaEncryption"
issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new"
subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
data = {
"tag": tag,
"event_count": count,
"_time": time,
"ssl_serial": serial,
"ssl_version": version,
"ssl_signature_algorithm": sig_algorithm,
"ssl_issuer": issuer,
"ssl_subject": subject,
"ssl_hash": ssl_hash,
"ssl_publickey_algorithm": key_algorithm
}
result_bundle = cim_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
cert_obj = TestTransform.get_first_of_type(objects.values(),
'x509-certificate')
assert (cert_obj is not None), 'x509-certificate object type not found'
assert (cert_obj.keys() == {
'type', 'serial_number', 'version', "signature_algorithm",
"subject_public_key_algorithm", "issuer", "subject", "hashes"
})
assert (cert_obj['serial_number'] == "1234")
assert (cert_obj['version'] == "1")
assert (cert_obj['signature_algorithm'] == "md5WithRSAEncryption")
assert (cert_obj['issuer'] ==
"C=US, ST=California, O=www.example.com, OU=new, CN=new")
assert (
cert_obj['subject'] ==
"C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
)
assert (cert_obj['subject_public_key_algorithm'] == "rsaEncryption")
assert (
cert_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
assert (objects.keys() == set(map(str, range(0, 1))))