def translate_results(self, data_source, data):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:return: STIX formatted results
:rtype: str
"""
json_data = json.loads(data)
data_source = json.loads(data_source)
results = json_to_stix_translator.convert_to_stix(
data_source, self.map_data, json_data,
transformers.get_all_transformers(), self.options)
if len(results['objects']) - 1 == len(json_data):
for i in range(1, len(results['objects'])):
results['objects'][i]['number_observed'] = 1
else:
raise RuntimeError(
"Incorrect number of result objects after translation. Found: {}, expected: {}."
.format(len(results['objects']) - 1, len(json_data)))
return results
def test_custom_property(self):
"""
to test the custom stix object properties
"""
data = {
'computer_identity': '1626351170-xlcr.hcl.local',
'subQueryID': 1,
'sha256hash':
'89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5',
'sha1hash': '41838ed7a546aeefe184fb8515973ffee7c3ba7e',
'md5hash': '958d9ba84826e48094e361102a272fd6',
'file_path': '/tmp/big42E1.tmp',
'file_name': 'big42E1.tmp',
'file_size': '770',
'type': 'file',
'timestamp': '1567046172',
'event_count': '1'
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
custom_object = observed_data['x_com_bigfix_relevance']
assert custom_object.keys() == {'computer_identity'}
assert custom_object[
'computer_identity'] == '1626351170-xlcr.hcl.local'
def test_cybox_observables(self):
data = {
'author_id': 'IBMid-123',
'email': '*****@*****.**',
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type(
objects.values(), 'user-account')
assert (curr_obj is not None), 'user-account object type not found'
assert (curr_obj.keys() == {'type', 'user_id'})
assert (curr_obj['user_id'] == data['author_id'])
curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type(
objects.values(), 'email-addr')
assert (curr_obj is not None), 'eamil-addr object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == data['email'])
def test_common_prop(self):
data = {
"starttime": 1531169112,
"endtime": 1531169254,
"eventcount": 5
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert (result_bundle_identity['type'] == data_source['type'])
assert (result_bundle_identity['id'] == data_source['id'])
assert (result_bundle_identity['name'] == data_source['name'])
assert (result_bundle_identity['identity_class'] ==
data_source['identity_class'])
observed_data = result_bundle_objects[1]
assert (observed_data['id'] is not None)
assert (observed_data['type'] == "observed-data")
assert (
observed_data['created_by_ref'] == result_bundle_identity['id'])
assert (observed_data['number_observed'] == 5)
assert (observed_data['created'] is not None)
assert (observed_data['modified'] is not None)
assert (observed_data['first_observed'] is not None)
assert (observed_data['last_observed'] is not None)
def test_network_cim_to_stix(self):
count = 2
time = "2018-08-21T15:11:55.000+00:00"
user = "******"
dest_ip = "127.0.0.1"
dest_port = "8090"
src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
src_port = "8080"
transport = "http"
data = {
"event_count": count,
"_time": time,
"user": user,
"dest_ip": dest_ip,
"dest_port": dest_port,
"src_ip": src_ip,
"src_port": src_port,
"protocol": transport
}
print(data)
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_obj = TestTransform.get_first_of_type(objects.values(),
'network-traffic')
assert (nt_obj is not None), 'network-traffic object type not found'
assert (nt_obj.keys() == {
'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_obj['src_port'] == 8080)
assert (nt_obj['dst_port'] == 8090)
assert (nt_obj['protocols'] == ['http'])
ip_ref = nt_obj['dst_ref']
assert (ip_ref
in objects), f"dst_ref with key {nt_obj['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == dest_ip)
ip_ref = nt_obj['src_ref']
assert (ip_ref
in objects), f"src_ref with key {nt_obj['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value'})
assert (ip_obj['type'] == 'ipv6-addr')
assert (ip_obj['value'] == src_ip)
def test_certificate_cim_to_stix(self):
count = 1
time = "2018-08-21T15:11:55.000+00:00"
serial = "1234"
version = "1"
sig_algorithm = "md5WithRSAEncryption"
key_algorithm = "rsaEncryption"
issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new"
subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"
data = {
"event_count": count,
"_time": time,
"ssl_serial": serial,
"ssl_version": version,
"ssl_signature_algorithm": sig_algorithm,
"ssl_issuer": issuer,
"ssl_subject": subject,
"ssl_hash": ssl_hash,
"ssl_publickey_algorithm": key_algorithm
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
# Test objects in Stix observable data model after transform
cert_obj = TestTransform.get_first_of_type(objects.values(),
'x509-certificate')
assert (cert_obj is not None), 'x509-certificate object type not found'
assert (cert_obj.keys() == {
'type', 'serial_number', 'version', "signature_algorithm",
"subject_public_key_algorithm", "issuer", "subject", "hashes"
})
assert (cert_obj['serial_number'] == "1234")
assert (cert_obj['version'] == "1")
assert (cert_obj['signature_algorithm'] == "md5WithRSAEncryption")
assert (cert_obj['issuer'] ==
"C=US, ST=California, O=www.example.com, OU=new, CN=new")
assert (
cert_obj['subject'] ==
"C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]"
)
assert (cert_obj['subject_public_key_algorithm'] == "rsaEncryption")
assert (
cert_obj['hashes']['SHA-256'] ==
"aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f")
assert (objects.keys() == set(map(str, range(0, 1))))
def test_common_prop(self):
"""
to test the common stix object properties
"""
data = {'computer_identity': '1626351170-xlcr.hcl.local', 'subQueryID': 1,
'sha256hash': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5',
'sha1hash': '41838ed7a546aeefe184fb8515973ffee7c3ba7e', 'md5hash': '958d9ba84826e48094e361102a272fd6',
'file_path': '/tmp/big42E1.tmp', 'file_name': 'big42E1.tmp', 'file_size': '770', 'type': 'file',
'timestamp': '1567046172', 'event_count': '1'}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
assert result_bundle_identity['id'] == data_source['id']
assert result_bundle_identity['name'] == data_source['name']
assert result_bundle_identity['identity_class'] == data_source['identity_class']
observed_data = result_bundle_objects[1]
assert observed_data['id'] is not None
assert observed_data['type'] == "observed-data"
assert observed_data['created_by_ref'] == result_bundle_identity['id']
assert observed_data['modified'] is not None
assert observed_data['created'] is not None
assert observed_data['first_observed'] is not None
assert observed_data['last_observed'] is not None
assert observed_data['number_observed'] is not None
assert observed_data['x_bigfix_relevance'] is not None
def translate_results(self, data_source, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX.
Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
self.mapping = options['mapping'] if 'mapping' in options else {}
json_data = json.loads(data)
data_source = json.loads(data_source)
if (not self.mapping):
map_file = open(self.mapping_filepath).read()
map_data = json.loads(map_file)
else:
map_data = self.mapping
results = json_to_stix_translator.convert_to_stix(
data_source, map_data, json_data,
transformers.get_all_transformers(), options)
if len(results['objects']) - 1 == len(json_data):
for i in range(1, len(results['objects'])):
results['objects'][i]['number_observed'] = 1
else:
raise RuntimeError(
"Incorrect number of result objects after translation. Found: {}, expected: {}."
.format(len(results['objects']) - 1, len(json_data)))
return json.dumps(results, indent=4, sort_keys=False)
def test_vpc_flow_network_json_to_stix(self):
"""to test network stix object properties"""
data = {
'vpcflow': {'@timestamp': '2019-10-20 10:43:09.000', 'srcAddr': '54.239.29.61', 'dstAddr': '172.31.88.63',
'srcPort': '443', 'dstPort': '53866', 'protocol': 'tcp', 'start': '1571568189',
'end': '1571568248', 'accountId': '979326520502', 'interfaceId': 'eni-02e70b8e842c70a2f',
'@ptr':
'CloKIQodOTc5MzI2NTIwNTAyOlVTRWFzdDFfRmxvd0xvZ3MQBxI1GhgCBc2q4EYAAAACFMuFggAF2sO4QAAAAWIgA'
'SjoyP/F3i0wyPyNxt4tOCxA7TFI0ClQzyIQJRgB',
'event_count': 1}}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
network_obj = TestAwsResultsToStix.get_first_of_type(objects.values(), 'network-traffic')
assert network_obj is not None, 'network-traffic object type not found'
assert network_obj.keys() == {'type', 'src_ref', 'dst_ref', 'src_port', 'dst_port', 'protocols', 'start', 'end'}
assert network_obj['type'] == 'network-traffic'
assert network_obj['src_ref'] == '0'
assert network_obj['dst_ref'] == '2'
assert network_obj['src_port'] == 443
assert network_obj['dst_port'] == 53866
assert network_obj['protocols'] == ['tcp']
assert network_obj['start'] == '2019-10-20T10:43:09.000Z'
assert network_obj['end'] == '2019-10-20T10:44:08.000Z'
def test_mac_addr_json_to_stix(self):
"""
to test network stix object properties
"""
data = {
'computer_identity': '541866979-suse01',
'subQueryID': 1,
'local_address': '192.168.36.110',
'mac': '0a-ab-41-e0-89-f8',
'type': 'Address',
'event_count': '1'
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
network_obj = TestBigFixResultsToStix.get_first_of_type(
objects.values(), 'network-traffic')
assert network_obj is not None, 'network-traffic object type not found'
assert network_obj.keys() == {'type', 'src_ref'}
assert network_obj['type'] == 'network-traffic'
assert network_obj['src_ref'] == '0'
def test_process_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
proc_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'process')
assert (proc_object is not None), 'process object type not found'
assert (proc_object.keys() == {
'type', 'pid', 'name', 'created', 'creator_user_ref', 'binary_ref'
})
assert (proc_object['type'] == 'process')
assert (proc_object['pid'] == 609)
assert (proc_object['created'] == '2019-04-10T11:33:57.571Z')
creator_user_ref = proc_object['creator_user_ref']
assert (
creator_user_ref in objects
), f"dst_ref with key {proc_object['creator_user_ref']} not found"
creator_user_ref_obj = objects[creator_user_ref]
assert (creator_user_ref_obj.keys() == {'type', 'user_id'})
assert (creator_user_ref_obj['type'] == 'user-account')
assert (creator_user_ref_obj['user_id'] == '-')
def translate_results(self, data_source, data, options, mapping=None):
"""
Translates JSON data into STIX results based on a mapping file
:param data: JSON formatted data to translate into STIX format
:type data: str
:param mapping: The mapping file path to use as instructions on how to translate the given JSON data to STIX.
Defaults the path to whatever is passed into the constructor for JSONToSTIX (This should be the to_stix_map.json in the module's json directory)
:type mapping: str (filepath)
:return: STIX formatted results
:rtype: str
"""
self.mapping = options['mapping'] if 'mapping' in options else {}
try:
json_data = json.loads(data)
data_source = json.loads(data_source)
except Exception:
raise LoadJsonResultsException()
if (not self.mapping):
map_file = open(self.default_mapping_file_path).read()
map_data = json.loads(map_file)
else:
map_data = self.mapping
try:
results = json_to_stix_translator.convert_to_stix(
data_source, map_data, json_data,
transformers.get_all_transformers(), options, self.callback)
except Exception as ex:
raise TranslationResultException(
"Error when converting results to STIX: {}".format(ex))
return json.dumps(results, indent=4, sort_keys=False)
def test_network_json_to_stix_negative(self):
"""
to test negative test case for stix object
"""
data = {
'computer_identity': '550872812-WIN-N11M78AV7BP',
'subQueryID': 1,
'local_address': '192.168.36.10',
'local_port': '139',
'process_ppid': '0',
'process_user': '******',
'timestamp': '1565875693',
'process_name': 'System',
'process_id': '4',
'type': 'Socket',
'protocol': 'udp',
'event_count': '1'
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, data, transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
network_obj = TestBigFixResultsToStix.get_first_of_type(
objects.values(), 'file')
assert network_obj is None
def test_custom_property(self):
"""
to test the custom stix object properties
"""
data = {
'ProcessCreationEvents': {
'EventTime': '2019-09-20T06:57:11.8218304Z',
'MachineId': '8330ed311f1b21b861d63448984eb2632cc9c07c',
'ComputerName': 'desktop-536bt46',
'ActionType': 'ProcessCreated',
'FileName': 'consent.exe',
'FolderPath': 'C:\\Windows\\System32\\consent.exe',
'SHA1': '9329b2362078de27242dd4534f588af3264bf0bf',
'SHA256':
'8f112431143a22baaafb448eefd63bf90e7691c890ac69a296574fd07ba03ec6',
'MD5': '27992d7ebe51aec655a088de88bad5c9',
'ProcessId': 20948,
'ProcessCommandLine': 'consent.exe 10088 288 000001CB3AA92A80',
'ProcessIntegrityLevel': 'System',
'ProcessTokenElevation': 'TokenElevationTypeDefault',
'ProcessCreationTime': '2019-09-20T06:57:11.8212034Z',
'AccountDomain': 'nt authority',
'AccountName': 'system',
'AccountSid': 'S-1-5-18',
'LogonId': 999,
'InitiatingProcessAccountDomain': 'nt authority',
'InitiatingProcessAccountName': 'system',
'InitiatingProcessAccountSid': 'S-1-5-18',
'InitiatingProcessLogonId': 999,
'InitiatingProcessIntegrityLevel': 'System',
'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault',
'InitiatingProcessSHA1':
'a1385ce20ad79f55df235effd9780c31442aa234',
'InitiatingProcessMD5': '8a0a29438052faed8a2532da50455756',
'InitiatingProcessFileName': 'svchost.exe',
'InitiatingProcessId': 10088,
'InitiatingProcessCommandLine':
'svchost.exe -k netsvcs -p -s Appinfo',
'InitiatingProcessCreationTime': '2019-09-18T05:56:15.268893Z',
'InitiatingProcessFolderPath':
'c:\\windows\\system32\\svchost.exe',
'InitiatingProcessParentId': 856,
'InitiatingProcessParentFileName': 'services.exe',
'InitiatingProcessParentCreationTime':
'2019-09-17T14:54:59.5778638Z',
'ReportId': 12048,
'rn': 1,
'event_count': '1'
}
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
custom_object = observed_data['x_com_msatp']
assert custom_object.keys() == {'computer_name', 'machine_id'}
assert custom_object['computer_name'] == 'desktop-536bt46'
def test_network_json_to_stix(self):
"""
to test network stix object properties
"""
data = {'computer_identity': '550872812-WIN-N11M78AV7BP', 'subQueryID': 1, 'local_address': '192.168.36.10',
'local_port': '139', 'process_ppid': '0', 'process_user': '******',
'timestamp': '1565875693', 'process_name': 'System', 'process_id': '4', 'type': 'Socket',
'protocol': 'udp', 'event_count': '1'}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
network_obj = TestBigFixResultsToStix.get_first_of_type(objects.values(), 'network-traffic')
assert network_obj is not None, 'network-traffic object type not found'
assert network_obj.keys() == {'type', 'src_ref', 'src_port', 'protocols'}
assert network_obj['type'] == 'network-traffic'
assert network_obj['src_ref'] == '0'
assert network_obj['src_port'] == 139
assert network_obj['protocols'] == ['udp']
def test_common_prop(self):
"""
to test the common stix object properties
"""
data = {
'createTime': '2019-10-31T11:15:55.099615Z',
'updateTime': '2019-10-31T11:15:55.099635Z',
'occurence_count': 1
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
assert result_bundle_identity['id'] == data_source['id']
assert result_bundle_identity['name'] == data_source['name']
assert result_bundle_identity['identity_class'] == data_source[
'identity_class']
observed_data = result_bundle_objects[1]
assert observed_data['id'] is not None
assert observed_data['type'] == "observed-data"
assert observed_data['created_by_ref'] == result_bundle_identity['id']
assert observed_data['first_observed'] is not None
assert (observed_data['first_observed'] == data['createTime'])
assert observed_data['last_observed'] is not None
assert (observed_data['last_observed'] == data['updateTime'])
assert observed_data['number_observed'] is not None
assert (observed_data['number_observed'] == data['occurence_count'])
def test_process_json_to_stix(self):
"""
to test process stix object properties
"""
data = {'computer_identity': '13476923-archlinux', 'subQueryID': 1,
'sha256hash': '2f2f74f4083b95654a742a56a6c7318f3ab378c94b69009ceffc200fbc22d4d8',
'sha1hash': '0c8e8b1d4eb31e1e046fea1f1396ff85068a4c4a', 'md5hash': '148fd5f2a448b69a9f21d4c92098c4ca',
'file_path': '/usr/lib/systemd/systemd', 'process_ppid': '0', 'process_user': '******',
'timestamp': '1565616101', 'process_name': 'systemd', 'process_id': '1', 'file_size': '1468376',
'type': 'process', 'event_count': '1'}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
process_obj = TestBigFixResultsToStix.get_first_of_type(objects.values(), 'process')
assert process_obj is not None, 'process object type not found'
assert process_obj.keys() == {'type', 'binary_ref', 'parent_ref', 'creator_user_ref', 'name', 'pid'}
assert process_obj['type'] == 'process'
assert process_obj['name'] == 'systemd'
assert process_obj['pid'] == 1
assert process_obj['binary_ref'] == '0'
assert process_obj['parent_ref'] == '3'
assert process_obj['creator_user_ref'] == '4'
def test_file_json_to_stix(self):
"""
to test file stix object properties
"""
data = {'computer_identity': '1626351170-xlcr.hcl.local', 'subQueryID': 1,
'sha256hash': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5',
'sha1hash': '41838ed7a546aeefe184fb8515973ffee7c3ba7e', 'md5hash': '958d9ba84826e48094e361102a272fd6',
'file_path': '/tmp/big42E1.tmp', 'file_name': 'big42E1.tmp', 'file_size': '770', 'type': 'file',
'timestamp': '1567046172', 'event_count': '1'}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
observed_data = result_bundle_objects[1]
assert 'objects' in observed_data
objects = observed_data['objects']
file_obj = TestBigFixResultsToStix.get_first_of_type(objects.values(), 'file')
assert file_obj is not None, 'file object type not found'
assert file_obj.keys() == {'type', 'hashes', 'parent_directory_ref', 'name', 'size'}
assert file_obj['type'] == 'file'
assert file_obj['name'] == 'big42E1.tmp'
assert file_obj['hashes'] == {'SHA-256': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5',
'SHA-1': '41838ed7a546aeefe184fb8515973ffee7c3ba7e',
'MD5': '958d9ba84826e48094e361102a272fd6'}
assert file_obj['parent_directory_ref'] == '1'
assert file_obj['size'] == 770
def test_risk_finding(self):
data = {
"logsourceid": 126,
"qidname": "event name",
"creeventlist": ["one", "two"],
"crename": "cre name",
"credescription": "cre description",
"identityip": "0.0.0.0",
"severity": 4,
"devicetypename": "device type name",
"devicetype": 15,
"rulenames": ["one", "two"]
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
observed_data = result_bundle['objects'][1]
assert ('x_com_ibm_finding' in observed_data)
finding = observed_data['x_com_ibm_finding']
assert (finding['name'] == data['crename'])
assert (finding['description'] == data['credescription'])
assert ('x_com_ibm_ariel' in observed_data)
custom_prop = observed_data['x_com_ibm_ariel']
assert (custom_prop['severity'] == data['severity'])
assert (custom_prop['event_name'] == data['qidname'])
assert (custom_prop['device_type_name'] == data['devicetypename'])
assert (custom_prop['device_type'] == data['devicetype'])
assert (custom_prop['cre_event_list'] == data['creeventlist'])
assert (custom_prop['rule_names'] == data['rulenames'])
def __init__(self, pattern: Pattern, data_model_mapper):
self.dmm = data_model_mapper
self.pattern = pattern
# Now report_params_passed is a JSON object which is pointing to an array of JSON Objects (report_params_array)
self.report_params_passed = {}
self.report_params_array = []
self.report_params_array_size = 0
self.translated = self.parse_expression(pattern)
self.transformers = transformers.get_all_transformers()
# Read reference data
basepath = path.dirname(__file__)
filepath = path.abspath(
path.join(basepath, "json", "reference_data_types4Query.json"))
self.REFERENCE_DATA_TYPES = json.loads(open(filepath).read())
# Read report definition data
filepath = path.abspath(
path.join(basepath, "json", "guardium_reports_def.json"))
self.REPORT_DEF = json.loads(open(filepath).read())
# Read report definition data
filepath = path.abspath(
path.join(basepath, "json", "guardium_report_params_map.json"))
self.REPORT_PARAMS_MAP = json.loads(open(filepath).read())
def test_email_cim_to_stix(self):
count = 3
time = "2018-08-21T15:11:55.000+00:00"
src_user = "******"
subject = "Test Subject"
multi = "False"
data = {
"event_count": count,
"_time": time,
"src_user": src_user,
"subject": subject,
"is_multipart": multi
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
validated_result = validate_instance(observed_data)
assert (validated_result.is_valid == True)
assert ('objects' in observed_data)
objects = observed_data['objects']
msg_obj = TestTransform.get_first_of_type(objects.values(),
'email-message')
assert (msg_obj is not None), 'email-message object type not found'
assert (msg_obj.keys() == {
'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'
})
assert (msg_obj['subject'] == "Test Subject")
assert (msg_obj['is_multipart'] == False)
sender_ref = msg_obj['sender_ref']
assert (sender_ref in objects
), f"sender_ref with key {msg_obj['sender_ref']} not found"
addr_obj = objects[sender_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
from_ref = msg_obj['from_ref']
assert (
sender_ref
in objects), f"from_ref with key {msg_obj['from_ref']} not found"
addr_obj = objects[from_ref]
assert (addr_obj.keys() == {'type', 'value'})
assert (addr_obj['type'] == 'email-addr')
assert (addr_obj['value'] == src_user)
def test_common_prop():
"""
to test the common stix object properties
"""
data = {'id': '2518268485253060642_52b1a353-2fd8-4c45-8f8a-94db98dca29d',
'azureTenantId': 'b73e5ba8-34d5-495a-9901-06bdb84cf13e',
'azureSubscriptionId': '083de1fb-cd2d-4b7c-895a-2b5af1d091e8', 'category': 'SuspiciousSVCHOSTRareGroup',
'createdDateTime': '2019-12-04T09:38:05.2024952Z',
'description': 'The system process SVCHOST was observed running a rare service group. Malware often '
'use SVCHOST to masquerade its malicious activity.',
'eventDateTime': '2019-12-04T09:37:54.6939357Z', 'lastModifiedDateTime': '2019-12-04T09:38:06.7571701Z',
'recommendedActions_0': '1. Run Process Explorer and try to identify unknown running processes (see '
'https://technet.microsoft.com/en-us/sysinternals/bb896653.aspx)',
'recommendedActions_1': '2. Make sure the machine is completely updated and has an updated '
'anti-malware application installed',
'recommendedActions_2': '3. Run a full anti-malware scan and verify that the threat was removed',
'recommendedActions_3': '4. Install and run Microsoft’s Malicious Software Removal Tool (see '
'https://www.microsoft.com/en-us/download/malicious-software-removal-tool'
'-details.aspx)',
'recommendedActions_4': '5. Run Microsoft’s Autoruns utility and try to identify unknown applications '
'that are configured to run at login (see '
'https://technet.microsoft.com/en-us/sysinternals/bb963902.aspx)',
'severity': 'informational', 'status': 'newAlert', 'title': 'Rare SVCHOST service group executed',
'vendorInformation_provider': 'ASC', 'vendorInformation_subProvider': 'Detection',
'vendorInformation_vendor': 'Microsoft', 'fileStates_0_name': 'services.exe',
'fileStates_0_path': 'c:\\windows\\system32\\services.exe', 'fileStates_1_name': 'svchost.exe',
'fileStates_1_path': 'c:\\windows\\system32\\svchost.exe', 'hostStates_0_netBiosName': 'TEST-WINDOW',
'hostStates_0_os': 'Windows', 'processes_0_commandLine': '', 'processes_0_name': 'services.exe',
'processes_0_path': 'c:\\windows\\system32\\services.exe', 'processes_1_accountName': 'test-window$',
'processes_1_commandLine': 'c:\\windows\\system32\\svchost.exe -k clipboardsvcgroup -p -s cbdhsvc',
'processes_1_createdDateTime': '2019-12-04T09:37:54.6939357Z', 'processes_1_name': 'svchost.exe',
'processes_1_parentProcessName': 'services.exe',
'processes_1_path': 'c:\\windows\\system32\\svchost.exe', 'userStates_0_accountName': 'test-window$',
'userStates_0_domainName': 'WORKGROUP', 'userStates_0_emailRole': 'unknown',
'userStates_0_logonId': '0x3e7', 'userStates_0_onPremisesSecurityIdentifier': 'S-1-5-18',
'userStates_0_userPrincipalName': 'test-window$@TEST-WINDOW', 'event_count': '1'}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
result_bundle_identity = result_bundle_objects[0]
assert result_bundle_identity['type'] == data_source['type']
assert result_bundle_identity['id'] == data_source['id']
assert result_bundle_identity['name'] == data_source['name']
assert result_bundle_identity['identity_class'] == data_source['identity_class']
observed_data = result_bundle_objects[1]
assert observed_data['id'] is not None
assert observed_data['type'] == "observed-data"
assert observed_data['created_by_ref'] == result_bundle_identity['id']
assert observed_data['modified'] is not None
assert observed_data['created'] is not None
assert observed_data['first_observed'] is not None
assert observed_data['last_observed'] is not None
assert observed_data['number_observed'] is not None
def test_unmapped_attribute_alone(self):
data = {"unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
assert (objects == {})
def test_unmapped_attribute_with_mapped_attribute(self):
message = "\"GET /blog HTTP/1.1\" 200 2571"
data = {"message": message, "unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(), options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
assert (objects == {})
curr_obj = TestBigFixResultsToStix.get_first_of_type(objects.values(), 'message')
assert (curr_obj is None), 'url object type not found'
def test_custom_property(self):
"""
to test the custom stix object properties
"""
data = {
'author_accountId': 'test_id_1',
'name': 'test_id_1/providers/sec_adv/occurrences/853092',
'id': '853092',
'noteName': 'test_id_1/providers/sec_adv/notes/cert_mngr',
'updateTime': '2019-10-31T11:15:55.099635Z',
'createTime': '2019-10-31T11:15:55.099615Z',
'shortDescription': 'testing data tranlation',
'providerId': 'sec_adv',
'providerName': 'test_id_1/providers/sec_adv',
'longDescription': 'testing data tranlation',
'context_accountId': 'test_id_1',
'context_resourceName': 'mycluster',
'reportedBy_id': 'cert_mngr',
'reportedBy_title': 'Security Advisor',
'finding_severity': 'MEDIUM',
'finding_certainty': 'HIGH',
'occurence_count': 1
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert result_bundle['type'] == 'bundle'
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
custom_props = observed_data['x_security_advisor_finding']
assert (custom_props['author_accountId'] == data['author_accountId'])
assert (custom_props['name'] == data['name'])
assert (custom_props['noteName'] == data['noteName'])
assert (custom_props['shortDescription'] == data['shortDescription'])
assert (custom_props['longDescription'] == data['longDescription'])
assert (custom_props['context_accountId'] == data['context_accountId'])
assert (custom_props['context_resourceName'] ==
data['context_resourceName'])
assert (custom_props['finding_severity'] == data['finding_severity'])
assert (custom_props['finding_certainty'] == data['finding_certainty'])
assert (custom_props['providerId'] == data['providerId'])
assert (custom_props['providerName'] == data['providerName'])
assert (custom_props['reportedBy_id'] == data['reportedBy_id'])
assert (custom_props['reportedBy_title'] == data['reportedBy_title'])
def test_unmapped_attribute_with_mapped_attribute(self):
url = "https://example.com"
data = {"url": url, "unmapped": "nothing to see here"}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
assert (objects != {})
curr_obj = TestTransform.get_first_of_type(objects.values(), 'url')
assert (curr_obj is not None), 'url object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == url)
def test_artifact_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
artifact_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'artifact')
assert (artifact_object is not None), 'artifact object type not found'
assert (artifact_object.keys() == {'type', 'payload_bin'})
assert (artifact_object['type'] == 'artifact')
def test_unmapped_attribute_with_mapped_attribute(self):
accountId = 'test_id_1',
data = {
"author_accountId": accountId,
"unmapped": "nothing to see here"
}
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
assert (objects == {})
curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type(
objects.values(), 'author_accountId')
assert (curr_obj is None), 'author_accountId object type not found'
def test_network_traffic_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
nt_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'network-traffic')
assert (nt_object is not None), 'network-traffic object type not found'
assert (nt_object.keys() == {
'type', 'src_port', 'src_byte_count', 'src_packets', 'dst_port',
'dst_byte_count', 'dst_packets', 'src_ref', 'dst_ref', 'protocols'
})
assert (nt_object['type'] == 'network-traffic')
assert (nt_object['src_port'] == 49745)
assert (nt_object['dst_port'] == 443)
assert (nt_object['protocols'] == ['ipv4', 'tcp'])
ip_ref = nt_object['dst_ref']
assert (
ip_ref
in objects), f"dst_ref with key {nt_object['dst_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == '100.101.0.69')
assert (isinstance(ip_obj['resolves_to_refs'], list)
and isinstance(ip_obj['resolves_to_refs'][0], str))
ip_ref = nt_object['src_ref']
assert (
ip_ref
in objects), f"src_ref with key {nt_object['src_ref']} not found"
ip_obj = objects[ip_ref]
assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'})
assert (ip_obj['type'] == 'ipv4-addr')
assert (ip_obj['value'] == '107.0.0.48')
assert (isinstance(ip_obj['resolves_to_refs'], list)
and isinstance(ip_obj['resolves_to_refs'][0], str))
def test_file_prop(self):
result_bundle = json_to_stix_translator.convert_to_stix(
data_source, map_data, [data], transformers.get_all_transformers(),
options)
assert (result_bundle['type'] == 'bundle')
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'file')
assert (file_object is not None), 'file object type not found'
assert (file_object.keys() == {'type', 'name'})
assert (file_object['type'] == 'file')
assert (file_object['name'] == 'SubmitDiagInfo')
