def test_custom_props(self): data = {"logsourceid": 126, "qid": 55500004, "identityip": "0.0.0.0", "magnitude": 4, "logsourcename": "someLogSourceName"} result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) observed_data = result_bundle['objects'][1] assert('x_com_ibm_ariel' in observed_data) custom_props = observed_data['x_com_ibm_ariel'] assert(custom_props['identity_ip'] == data['identityip']) assert(custom_props['log_source_id'] == data['logsourceid']) assert(custom_props['qid'] == data['qid']) assert(custom_props['magnitude'] == data['magnitude']) assert(custom_props['log_source_name'] == data['logsourcename'])
def test_unmapped_attribute_with_mapped_attribute(self): url = "https://example.com" data = {"url": url, "unmapped": "nothing to see here"} result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] assert (objects != {}) curr_obj = TestTransform.get_first_of_type(objects.values(), 'url') assert (curr_obj is not None), 'url object type not found' assert (curr_obj.keys() == {'type', 'value'}) assert (curr_obj['value'] == url)
def test_email_cim_to_stix(self): count = 3 time = "2018-08-21T15:11:55.000+00:00" src_user = "******" subject = "Test Subject" multi = "False" data = {"event_count": count, "_time": time, "src_user": src_user, "subject": subject, "is_multipart": multi } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] msg_obj = TestTransform.get_first_of_type(objects.values(), 'email-message') assert(msg_obj is not None), 'email-message object type not found' assert(msg_obj.keys() == {'type', 'subject', 'sender_ref', 'from_ref', 'is_multipart'}) assert(msg_obj['subject'] == "Test Subject") assert(msg_obj['is_multipart'] == False) sender_ref = msg_obj['sender_ref'] assert(sender_ref in objects), f"sender_ref with key {msg_obj['sender_ref']} not found" addr_obj = objects[sender_ref] assert(addr_obj.keys() == {'type', 'value'}) assert(addr_obj['type'] == 'email-addr') assert(addr_obj['value'] == src_user) from_ref = msg_obj['from_ref'] assert(sender_ref in objects), f"from_ref with key {msg_obj['from_ref']} not found" addr_obj = objects[from_ref] assert(addr_obj.keys() == {'type', 'value'}) assert(addr_obj['type'] == 'email-addr') assert(addr_obj['value'] == src_user)
def test_network_cim_to_stix(self): count = 2 time = "2018-08-21T15:11:55.000+00:00" user = "******" dest_ip = "127.0.0.1" dest_port = "8090" src_ip = "2001:0db8:85a3:0000:0000:8a2e:0370:7334" src_port = "8080" transport = "http" data = {"event_count": count, "_time": time, "user": user, "dest_ip": dest_ip, "dest_port": dest_port, "src_ip": src_ip, "src_port": src_port, "protocol": transport } print(data) result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic') assert(nt_obj is not None), 'network-traffic object type not found' assert(nt_obj.keys() == {'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols'}) assert(nt_obj['src_port'] == 8080) assert(nt_obj['dst_port'] == 8090) assert(nt_obj['protocols'] == ['http']) ip_ref = nt_obj['dst_ref'] assert(ip_ref in objects), f"dst_ref with key {nt_obj['dst_ref']} not found" ip_obj = objects[ip_ref] assert(ip_obj.keys() == {'type', 'value'}) assert(ip_obj['type'] == 'ipv4-addr') assert(ip_obj['value'] == dest_ip) ip_ref = nt_obj['src_ref'] assert(ip_ref in objects), f"src_ref with key {nt_obj['src_ref']} not found" ip_obj = objects[ip_ref] assert(ip_obj.keys() == {'type', 'value'}) assert(ip_obj['type'] == 'ipv6-addr') assert(ip_obj['value'] == src_ip)
def test_file_json_to_stix(self): """ to test file stix object properties """ data = { 'computer_identity': '1626351170-xlcr.hcl.local', 'subQueryID': 1, 'sha256hash': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5', 'sha1hash': '41838ed7a546aeefe184fb8515973ffee7c3ba7e', 'md5hash': '958d9ba84826e48094e361102a272fd6', 'file_path': '/tmp/big42E1.tmp', 'file_name': 'big42E1.tmp', 'file_size': '770', 'type': 'file', 'timestamp': '1567046172', 'event_count': '1' } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data objects = observed_data['objects'] file_obj = TestBigFixResultsToStix.get_first_of_type( objects.values(), 'file') assert file_obj is not None, 'file object type not found' assert file_obj.keys() == { 'type', 'hashes', 'parent_directory_ref', 'name', 'size' } assert file_obj['type'] == 'file' assert file_obj['name'] == 'big42E1.tmp' assert file_obj['hashes'] == { 'SHA-256': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5', 'SHA-1': '41838ed7a546aeefe184fb8515973ffee7c3ba7e', 'MD5': '958d9ba84826e48094e361102a272fd6' } assert file_obj['parent_directory_ref'] == '1' assert file_obj['size'] == 770
def test_process_json_to_stix(self): """ to test process stix object properties """ data = { 'computer_identity': '13476923-archlinux', 'subQueryID': 1, 'sha256hash': '2f2f74f4083b95654a742a56a6c7318f3ab378c94b69009ceffc200fbc22d4d8', 'sha1hash': '0c8e8b1d4eb31e1e046fea1f1396ff85068a4c4a', 'md5hash': '148fd5f2a448b69a9f21d4c92098c4ca', 'file_path': '/usr/lib/systemd/systemd', 'process_ppid': '0', 'process_user': '******', 'timestamp': '1565616101', 'process_name': 'systemd', 'process_id': '1', 'file_size': '1468376', 'type': 'process', 'event_count': '1' } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data objects = observed_data['objects'] process_obj = TestBigFixResultsToStix.get_first_of_type( objects.values(), 'process') assert process_obj is not None, 'process object type not found' assert process_obj.keys() == { 'type', 'binary_ref', 'parent_ref', 'creator_user_ref', 'name', 'pid' } assert process_obj['type'] == 'process' assert process_obj['name'] == 'systemd' assert process_obj['pid'] == 1 assert process_obj['binary_ref'] == '0' assert process_obj['parent_ref'] == '3' assert process_obj['creator_user_ref'] == '4'
def test_vpc_flow_custom_attr_json_to_stix(self): """to test network stix object properties""" data = { 'vpcflow': { '@timestamp': '2019-10-20 10:43:09.000', 'srcAddr': '54.239.29.61', 'dstAddr': '172.31.88.63', 'srcPort': '443', 'dstPort': '53866', 'protocol': 'tcp', 'start': '1571568189', 'end': '1571568248', 'accountId': '979326520502', 'interfaceId': 'eni-02e70b8e842c70a2f', '@ptr': 'CloKIQodOTc5MzI2NTIwNTAyOlVTRWFzdDFfRmxvd0xvZ3MQBxI1GhgCBc2q4EYAAAACFMuFggAF2sO4QAAAAWIg' 'ASjoyP/F3i0wyPyNxt4tOCxA7TFI0ClQzyIQJRgB', 'event_count': 1 } } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data custom_object = observed_data['x_com_aws'] assert custom_object.keys() == {'account_id'} assert custom_object['account_id'] == '979326520502'
def test_payload_results(self): data = { "src_ip": "169.250.0.1", "src_port": "1220", "src_mac": "aa:bb:cc:dd:11:22", "dest_ip": "127.0.0.1", "dest_port": "1120", "dest_mac": "ee:dd:bb:aa:cc:11", "file_hash": "741ad92448fd12a089a13c6de49fb204e4693e1d3e9f7715471c292adf8c6bef", "user": "******", "url": "https://wally.fireeye.com/malware_analysis/analyses?maid=1", "protocol": "tcp", "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114", "_indextime": "1546960685", "_raw": "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1 spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111 fileHash=41a26255d16d121dc525a6445144b895 proto=tcp request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan cn1=0 externalId=1 cs4Label=link cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames \n", "_serial": "0", "_si": ["splunk3-01.internal.resilientsystems.com", "main"], "_sourcetype": "fe_cef_syslog", "_time": "2019-01-08T15:18:04.000+00:00", "event_count": 1 } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] # somehow breaking the stix validation # validated_result = validate_instance(observed_data) # assert(validated_result.is_valid == True) assert ('objects' in observed_data) objects = observed_data['objects'] utf8 = "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1 spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111 fileHash=41a26255d16d121dc525a6445144b895 proto=tcp request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan cn1=0 externalId=1 cs4Label=link cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames \n" assert (observed_data['x_com_splunk_spl']['utf8_payload'] == utf8)
def test_unmapped_attribute_with_mapped_attribute(self): accountId = 'test_id_1', data = { "author_accountId": accountId, "unmapped": "nothing to see here" } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] assert (objects == {}) curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type( objects.values(), 'author_accountId') assert (curr_obj is None), 'author_accountId object type not found'
def test_artifact_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] artifact_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'artifact') assert (artifact_object is not None), 'artifact object type not found' assert (artifact_object.keys() == {'type', 'payload_bin'}) assert (artifact_object['type'] == 'artifact')
def test_file_json_to_stix(self): """ to test file stix object properties """ data = { "computer_identity": "12369754-bigdata4545.canlab.ibm.com", "subQueryID": 1, "type": "file", "file_name": ".X0-lock", "sha256hash": "7236f966f07259a1de3ee0d48a3ef0ee47c4a551af7f0d76dcabbbb9d6e00940", "sha1hash": "8b5e953be1db90172af66631132f6f27dda402d2", "md5hash": "e5307d27f0eb9a27af8597a1ddc51e89", "file_path": "/tmp/.X0-lock", "modified_time": "1541424894" } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data objects = observed_data['objects'] file_obj = TestBigFixResultsToStix.get_first_of_type( objects.values(), 'file') assert file_obj is not None, 'file object type not found' assert file_obj.keys() == { 'type', 'name', 'hashes', 'parent_directory_ref' } assert file_obj['type'] == 'file' assert file_obj['name'] == '.X0-lock' assert file_obj['hashes'] == { 'SHA-256': '7236f966f07259a1de3ee0d48a3ef0ee47c4a551af7f0d76dcabbbb9d6e00940', 'SHA-1': '8b5e953be1db90172af66631132f6f27dda402d2', 'MD5': 'e5307d27f0eb9a27af8597a1ddc51e89' } assert file_obj['parent_directory_ref'] == '1'
def test_file_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'file') assert (file_object is not None), 'file object type not found' assert (file_object.keys() == {'type', 'name'}) assert (file_object['type'] == 'file') assert (file_object['name'] == 'SubmitDiagInfo')
def test_certificate_cim_to_stix(self): count = 1 time = "2018-08-21T15:11:55.000+00:00" serial = "1234" version = "1" sig_algorithm = "md5WithRSAEncryption" key_algorithm = "rsaEncryption" issuer = "C=US, ST=California, O=www.example.com, OU=new, CN=new" subject = "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]" ssl_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" data = { "event_count": count, "_time": time, "ssl_serial": serial, "ssl_version": version, "ssl_signature_algorithm": sig_algorithm, "ssl_issuer": issuer, "ssl_subject": subject, "ssl_hash": ssl_hash, "ssl_publickey_algorithm": key_algorithm } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] # Test objects in Stix observable data model after transform cert_obj = TestTransform.get_first_of_type(objects.values(), 'x509-certificate') assert(cert_obj is not None), 'x509-certificate object type not found' assert(cert_obj.keys() == {'type', 'serial_number', 'version', "signature_algorithm", "subject_public_key_algorithm", "issuer", "subject", "hashes"}) assert(cert_obj['serial_number'] == "1234") assert(cert_obj['version'] == "1") assert(cert_obj['signature_algorithm'] == "md5WithRSAEncryption") assert(cert_obj['issuer'] == "C=US, ST=California, O=www.example.com, OU=new, CN=new") assert(cert_obj['subject'] == "C=US, ST=Maryland, L=Baltimore, O=John Doe, OU=ExampleCorp, CN=www.example.com/[email protected]") assert(cert_obj['subject_public_key_algorithm'] == "rsaEncryption") assert(cert_obj['hashes']['SHA-256'] == "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f") assert(objects.keys() == set(map(str, range(0, 1))))
def test_common_prop(self): """ to test the common stix object properties """ data = { 'computer_identity': '1626351170-xlcr.hcl.local', 'subQueryID': 1, 'sha256hash': '89698504cb73fefacd012843a5ba2e0acda7fd8d5db4efaad22f7fe54fa422f5', 'sha1hash': '41838ed7a546aeefe184fb8515973ffee7c3ba7e', 'md5hash': '958d9ba84826e48094e361102a272fd6', 'file_path': '/tmp/big42E1.tmp', 'file_name': 'big42E1.tmp', 'file_size': '770', 'type': 'file', 'timestamp': '1567046172', 'event_count': '1' } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] assert result_bundle_identity['id'] == data_source['id'] assert result_bundle_identity['name'] == data_source['name'] assert result_bundle_identity['identity_class'] == data_source[ 'identity_class'] observed_data = result_bundle_objects[1] assert observed_data['id'] is not None assert observed_data['type'] == "observed-data" assert observed_data['created_by_ref'] == result_bundle_identity['id'] assert observed_data['modified'] is not None assert observed_data['created'] is not None assert observed_data['first_observed'] is not None assert observed_data['last_observed'] is not None assert observed_data['number_observed'] is not None assert observed_data['x_com_bigfix_relevance'] is not None
def test_common_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert (result_bundle_identity['type'] == data_source['type']) assert (result_bundle_identity['id'] == data_source['id']) assert (result_bundle_identity['name'] == data_source['name']) assert (result_bundle_identity['identity_class'] == data_source['identity_class']) observed_data = result_bundle_objects[1] assert (observed_data['id'] is not None) assert (observed_data['type'] == "observed-data") assert ( observed_data['created_by_ref'] == result_bundle_identity['id'])
def test_network_json_to_stix(self): """ to test network stix object properties """ data = { 'computer_identity': '550872812-WIN-N11M78AV7BP', 'subQueryID': 1, 'local_address': '192.168.36.10', 'local_port': '139', 'process_ppid': '0', 'process_user': '******', 'timestamp': '1565875693', 'process_name': 'System', 'process_id': '4', 'type': 'Socket', 'protocol': 'udp', 'event_count': '1' } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data objects = observed_data['objects'] network_obj = TestBigFixResultsToStix.get_first_of_type( objects.values(), 'network-traffic') assert network_obj is not None, 'network-traffic object type not found' assert network_obj.keys() == { 'type', 'src_ref', 'src_port', 'protocols' } assert network_obj['type'] == 'network-traffic' assert network_obj['src_ref'] == '0' assert network_obj['src_port'] == 139 assert network_obj['protocols'] == ['udp']
def test_process_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] proc_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'process') assert (proc_object is not None), 'process object type not found' assert (proc_object.keys() == { 'type', 'pid', 'command_line', 'created', 'image_ref', 'creator_user_ref' }) assert (proc_object['type'] == 'process') assert (proc_object['pid'] == 609) assert (proc_object['command_line'] == '/System/Library/CoreServices/SubmitDiagInfo') assert (proc_object['created'] == '2019-04-10T11:33:57.571Z') image_ref = proc_object['image_ref'] assert (image_ref in objects ), f"dst_ref with key {proc_object['image_ref']} not found" image_obj = objects[image_ref] assert (image_obj.keys() == {'type', 'name'}) assert (image_obj['type'] == 'file') assert (image_obj['name'] == 'SubmitDiagInfo') creator_user_ref = proc_object['creator_user_ref'] assert ( creator_user_ref in objects ), f"dst_ref with key {proc_object['creator_user_ref']} not found" creator_user_ref_obj = objects[creator_user_ref] assert (creator_user_ref_obj.keys() == {'type', 'user_id'}) assert (creator_user_ref_obj['type'] == 'user') assert (creator_user_ref_obj['user_id'] == '-')
def test_process_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] artifact_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'artifact') assert (artifact_object is not None), 'artifact object type not found' assert (artifact_object.keys() == {'type', 'payload_bin'}) assert (artifact_object['type'] == 'artifact') assert ( artifact_object['payload_bin'] == 'MTAuNDIuNDIuNDIgLSAtIFswNy9EZWMvMjAxODoxMTowNTowNyArMDEwMF0gIkdFVCAvYmxvZyBIVFRQLzEuMSIgMjAwIDI1NzEgIi0iICJNb3ppbGxhLzUuMCAoTWFjaW50b3NoOyBJbnRlbCBNYWMgT1MgWCAxMF8xNF8wKSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNzAuMC4zNTM4LjEwMiBTYWZhcmkvNTM3LjM2Ig==' )
def test_process_results_to_stix(self): process_name = 'systemd' data = {"computer_identity": "12369754-bigdata4545.canlab.ibm.com", "subQueryID": 1, "start_time": "1541424881", "type": "process", "process_name": "systemd", "process_id": "1", "sha256hash": "9c74c625b2aba7a2e8d8a42e2e94715c355367f7cbfa9bd5404ba52b726792a6", "sha1hash": "916933045c5c91ebcaa325e7f8302f3a732a0a3d", "md5hash": "28a9beb86c4d4c31ba572805bea8494f", "file_path": "/usr/lib/systemd/systemd"} result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) print(json.dumps(result_bundle, indent=2)) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert(result_bundle_identity['type'] == data_source['type']) observed_data = result_bundle_objects[1] assert('objects' in observed_data) objects = observed_data['objects'] process_obj = TestBigFixResultsToStix.get_first_of_type(objects.values(), 'process') assert(process_obj is not None), 'process object type not found' assert(process_obj.keys() == {'type', 'name', 'pid', 'binary_ref'}) assert(process_obj['name'] == process_name)
def test_network_traffic_prop(self): result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] nt_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'network-traffic') assert (nt_object is not None), 'network-traffic object type not found' assert (nt_object.keys() == { 'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols' }) assert (nt_object['type'] == 'network-traffic') assert (nt_object['src_port'] == 49745) assert (nt_object['dst_port'] == 443) assert (nt_object['protocols'] == ['ipv4', 'tcp']) ip_ref = nt_object['dst_ref'] assert ( ip_ref in objects), f"dst_ref with key {nt_object['dst_ref']} not found" ip_obj = objects[ip_ref] assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) assert (ip_obj['type'] == 'ipv4-addr') assert (ip_obj['value'] == '100.101.0.69') ip_ref = nt_object['src_ref'] assert ( ip_ref in objects), f"src_ref with key {nt_object['src_ref']} not found" ip_obj = objects[ip_ref] assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) assert (ip_obj['type'] == 'ipv4-addr') assert (ip_obj['value'] == '107.0.0.48')
def test_common_prop(self): """ to test the common stix object properties """ data = { "computer_identity": "12369754-bigdata4545.canlab.ibm.com", "subQueryID": 1, "start_time": "1541424881", "type": "process", "process_name": "systemd", "process_id": "1", "sha256hash": "9c74c625b2aba7a2e8d8a42e2e94715c355367f7cbfa9bd5404ba52b726792a6", "sha1hash": "916933045c5c91ebcaa325e7f8302f3a732a0a3d", "md5hash": "28a9beb86c4d4c31ba572805bea8494f", "file_path": "/usr/lib/systemd/systemd" } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert result_bundle['type'] == 'bundle' result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] assert result_bundle_identity['id'] == data_source['id'] assert result_bundle_identity['name'] == data_source['name'] assert result_bundle_identity['identity_class'] == data_source[ 'identity_class'] observed_data = result_bundle_objects[1] assert observed_data['id'] is not None assert observed_data['type'] == "observed-data" assert observed_data['created_by_ref'] == result_bundle_identity['id'] assert observed_data['created'] is not None assert observed_data['first_observed'] is not None assert observed_data['last_observed'] is not None
def test_process_cim_to_stix(self): count = 1 time = "2018-08-21T15:11:55.000+00:00" user = "******" pid = 0 name = "test_process" filePath = "C:\\Users\\someuser\\sample.dll" create_time = "2018-08-15T15:11:55.676+00:00" modify_time = "2018-08-15T18:10:30.456+00:00" file_hash = "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" file_name = "sample.dll" file_size = 25536 data = { "event_count": count, "_time": time, "user": user, "process_name": name, "process_id": pid, "file_path": filePath, "file_create_time": create_time, "file_modify_time": modify_time, "file_hash": file_hash, "file_size": file_size, "file_name": file_name } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] # Test objects in Stix observable data model after transform proc_obj = TestTransform.get_first_of_type(objects.values(), 'process') assert(proc_obj is not None), 'process object type not found' assert(proc_obj.keys() == {'type', 'name', 'pid', 'binary_ref'}) assert(proc_obj['name'] == "test_process") assert(proc_obj['pid'] == 0) user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') assert(user_obj is not None), 'user-account object type not found' assert(user_obj.keys() == {'type', 'account_login', 'user_id'}) assert(user_obj['account_login'] == "test_user") assert(user_obj['user_id'] == "test_user") bin_ref = proc_obj['binary_ref'] assert(bin_ref in objects), f"binary_ref with key {proc_obj['binary_ref']} not found" file_obj = objects[bin_ref] assert(file_obj is not None), 'file object type not found' assert(file_obj.keys() == {'type', 'parent_directory_ref', 'created', 'modified', 'size', 'name', 'hashes'}) assert(file_obj['created'] == "2018-08-15T15:11:55.676Z") assert(file_obj['modified'] == "2018-08-15T18:10:30.456Z") assert(file_obj['name'] == "sample.dll") assert(file_obj['size'] == 25536) assert (file_obj['hashes']['SHA-256'] == "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f") dir_ref = file_obj['parent_directory_ref'] assert(dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found" dir_obj = objects[dir_ref] assert(dir_obj is not None), 'directory object type not found' assert(dir_obj.keys() == {'type', 'path', 'created', 'modified'}) assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll") assert(dir_obj['created'] == "2018-08-15T15:11:55.676Z") assert(dir_obj['modified'] == "2018-08-15T18:10:30.456Z") assert(objects.keys() == set(map(str, range(0, 4))))
def test_cybox_observables(self): # transformer = None payload = "SomeBase64Payload" user_id = "someuserid2018" url = "https://example.com" domain = "example.com" source_ip = "127.0.0.1" destination_ip = "255.255.255.1" data = {"sourceip": source_ip, "destinationip": destination_ip, "url": url, "domain": domain, "payload": payload, "username": user_id, "protocol": 'TCP', "sourceport": 3000, "destinationport": 2000} # data = {"Network": {"A" : source_ip}, "destinationip": destination_ip, "url": url, # "domain": domain, "payload": payload, "username": user_id, "protocol": 'TCP', "sourceport": 3000, "destinationport": 2000} result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert('objects' in observed_data) objects = observed_data['objects'] # Test that each data element is properly mapped and input into the STIX JSON for key, value in objects.items(): assert(int(key) in list(range(0, len(objects)))) # Todo: handle case where there is both a source and destination ip, there will be more than one ipv4-addr if(value['type'] == 'ipv4-addr'): # assert( # value['value'] == source_ip), "Wrong value returned " + key + ":" + str(value) assert(True) elif(value['type'] == 'url'): assert(value['value'] == url), "Wrong value returned " + \ key + ":" + str(value) elif(value['type'] == 'domain-name'): assert( value['value'] == domain), "Wrong value returned " + key + ":" + str(value) elif(value['type'] == 'artifact'): assert( value['payload_bin'] == payload), "Wrong value returned " + key + ":" + str(value) elif(value['type'] == 'user-account'): assert( value['user_id'] == user_id), "Wrong value returned " + key + ":" + str(value) # Todo: should not be returned since the address passed in isn't ipv6, still needs to be fixed in logic elif(value['type'] == 'ipv6-addr'): # assert( # value['value'] == source_ip), "Wrong value returned " + key + ":" + str(value) assert(True) elif(value['type'] == 'network-traffic'): assert(int(value['src_ref']) in list( range(0, len(objects)))), "Wrong value returned " + key + ":" + str(value) assert(type(value['src_ref']) is str), "Reference value should be a string" assert(int(value['dst_ref']) in list( range(0, len(objects)))), "Wrong value returned " + key + ":" + str(value) assert(type(value['dst_ref']) is str), "Reference value should be a string" assert(value['protocols'] == ['tcp']) assert(value['src_port'] == 3000) assert(value['dst_port'] == 2000) else: assert(False), "Returned a non-mapped value " + \ key + ":" + str(value)
def test_guardduty_custom_attr_json_to_stix(self): """to test network stix object properties""" data = { 'guardduty': { 'NETWORK_CONNECTION': { 'version': '0', 'id': '617628ca-ae74-5875-6e9c-be4cd2e9e267', 'detail-type': 'GuardDuty Finding', 'time': '2019-10-17T09:30:05Z', 'region': 'us-east-1', 'detail_schemaVersion': '2.0', 'detail_accountId': '979326520502', 'detail_partition': 'aws', 'detail_arn': 'arn:aws:guardduty:us-east-1:979326520502:detector' '/6ab6e6ee780ed494f3b7ca56acdc74df/finding/6cb6e9975' '1fcbed76aae1a9a64bb96a8', 'detail_resource_instanceDetails_instanceId': 'i-0b8fd03ade35c' '681d', 'detail_resource_instanceDetails_instanceType': 't2.micro', 'detail_resource_instanceDetails_launchTime': '2019-10-14T12:51:' '57Z', 'detail_resource_instanceDetails_iamInstanceProfile' '_arn': 'arn:aws:iam::979326520502:instance-profile/EC2_Instances_' 'Full_Access', 'detail_resource_instanceDetails_iamInstanceProfile_id': 'AIPA6IBDIZS3ES3TI5TNQ', 'detail_resource_instanceDetails_networkInterfaces_0_' 'networkInterfaceId': 'eni-02e70b8e842c70a2f', 'detail_resource_instanceDetails_networkInterfaces_0_' 'privateDnsName': 'ip-172-31-88-63.ec2.internal', 'detail_resource_instanceDetails_networkInterfaces_0_' 'privateIpAddress': '172.31.88.63', 'detail_resource_instanceDetails_networkInterfaces_0_' 'privateIpAddresses_0_privateDnsName': 'ip-172-31-88-63.ec2.internal', 'detail_resource_instanceDetails_networkInterfaces_0_' 'privateIpAddresses_0_privateIpAddress': '172.31.88.63', 'detail_resource_instanceDetails_networkInterfaces_0_subnetId': 'subnet-c62a11e8', 'detail_resource_instanceDetails_networkInterfaces_0_vpcId': 'vpc-10db926a', 'detail_resource_instanceDetails_networkInterfaces_0_' 'securityGroups_0_groupName': 'launch-wizard-1', 'detail_resource_instanceDetails_networkInterfaces_0_' 'securityGroups_0_groupId': 'sg-0aa89ff4646f71594', 'detail_resource_instanceDetails_networkInterfaces_0_' 'publicDnsName': 'ec2-54-211-223-78.compute-1.amazonaws.com', 'detail_resource_instanceDetails_networkInterfaces_0_' 'publicIp': '54.211.223.78', 'detail_resource_instanceDetails_instanceState': 'running', 'detail_resource_instanceDetails_availabilityZone': 'us-east-1b', 'detail_resource_instanceDetails_imageId': 'ami-04763b3055de4860b', 'detail_resource_instanceDetails_imageDescription': 'Canonical, Ubuntu, 16.04 LTS, amd64 xenial image build on ' '2019-09-13', 'detail_service_serviceName': 'guardduty', 'detail_service_detectorId': '6ab6e6ee780ed494f3b7ca56acdc74df', 'detail_service_action_networkConnectionAction_' 'connectionDirection': 'INBOUND', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'ipAddressV4': '54.211.162.49', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'organization_asn': '14618', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'organization_asnOrg': 'Amazon.com, Inc.', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'organization_isp': 'Amazon.com', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'organization_org': 'Amazon.com', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'country_countryName': 'United States', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'city_cityName': 'Ashburn', 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'geoLocation_lat': 39.0481, 'detail_service_action_networkConnectionAction_remoteIpDetails_' 'geoLocation_lon': -77.4728, 'detail_service_action_networkConnectionAction_remotePortDetails_' 'port': 32820, 'detail_service_action_networkConnectionAction_remotePortDetails_' 'portName': 'Unknown', 'detail_service_action_networkConnectionAction_localPortDetails_' 'port': 22, 'detail_service_action_networkConnectionAction_localPortDetails_' 'portName': 'SSH', 'detail_service_action_networkConnectionAction_protocol': 'TCP', 'detail_service_action_networkConnectionAction_blocked': 'false', 'detail_service_additionalInfo': {} }, 'source': 'aws.guardduty', 'account': '979326520502', 'detail_region': 'us-east-1', 'detail_id': '6cb6e99751fcbed76aae1a9a64bb96a8', 'detail_type': 'UnauthorizedAccess:EC2/SSHBruteForce', 'detail_resource_resourceType': 'Instance', 'detail_service_action_actionType': 'NETWORK_CONNECTION', 'detail_service_resourceRole': 'TARGET', 'detail_service_eventFirstSeen': '2019-10-16T05:55:25Z', 'detail_service_eventLastSeen': '2019-10-17T09:05:51Z', 'detail_service_archived': 'false', 'detail_service_count': 16, 'detail_severity': 2, 'detail_createdAt': '2019-10-16T06:08:32.249Z', 'detail_updatedAt': '2019-10-17T09:20:25.038Z', 'detail_title': '54.211.162.49 is performing SSH brute force attacks against i-0b8fd03' 'ade35c681d. ', 'detail_description': '54.211.162.49 is performing SSH brute force attacks against ' 'i-0b8fd03ade35c681d. Brute force attacks are used to gain ' 'unauthorized access to your instance by guessing the SSH ' 'password.', '@timestamp': '2019-10-17 09:30:05.000', 'event_count': 1 } } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] observed_data = result_bundle_objects[1] assert 'objects' in observed_data custom_object = observed_data['x_com_aws_guardduty_finding'] assert custom_object.keys() == { 'severity', 'id', 'type', 'title', 'timestamp' } assert custom_object['id'] == '6cb6e99751fcbed76aae1a9a64bb96a8' assert custom_object['type'] == 'UnauthorizedAccess:EC2/SSHBruteForce' assert custom_object['severity'] == 2 assert custom_object['timestamp'] == '2019-10-17T09:30:05.000Z' assert custom_object[ 'title'] == '54.211.162.49 is performing SSH brute force attacks against i-0b8fd03ade35c681d. '
def test_cybox_observables(self): payload = "SomeBase64Payload" user_id = "someuserid2018" url = "https://example.com" source_ip = "fd80:655e:171d:30d4:fd80:655e:171d:30d4" destination_ip = "255.255.255.1" file_name = "somefile.exe" source_mac = "00-00-5E-00-53-00" destination_mac = "00-00-5A-00-55-01" data = { "sourceip": source_ip, "destinationip": destination_ip, "url": url, "base64_payload": payload, "username": user_id, "protocol": 'TCP', "sourceport": "3000", "destinationport": 2000, "filename": file_name, "domainname": url, "sourcemac": source_mac, "destinationmac": destination_mac } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options) assert (result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] nt_object = TestTransform.get_first_of_type(objects.values(), 'network-traffic') assert (nt_object is not None), 'network-traffic object type not found' assert (nt_object.keys() == { 'type', 'src_port', 'dst_port', 'src_ref', 'dst_ref', 'protocols' }) assert (nt_object['src_port'] == 3000) assert (nt_object['dst_port'] == 2000) assert (nt_object['protocols'] == ['tcp']) ip_ref = nt_object['dst_ref'] assert ( ip_ref in objects), f"dst_ref with key {nt_object['dst_ref']} not found" ip_obj = objects[ip_ref] assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) assert (ip_obj['type'] == 'ipv4-addr') assert (ip_obj['value'] == destination_ip) ip_ref = nt_object['src_ref'] assert ( ip_ref in objects), f"src_ref with key {nt_object['src_ref']} not found" ip_obj = objects[ip_ref] assert (ip_obj.keys() == {'type', 'value', 'resolves_to_refs'}) assert (ip_obj['type'] == 'ipv6-addr') assert (ip_obj['value'] == source_ip) curr_obj = TestTransform.get_first_of_type(objects.values(), 'url') assert (curr_obj is not None), 'url object type not found' assert (curr_obj.keys() == {'type', 'value'}) assert (curr_obj['value'] == url) curr_obj = TestTransform.get_first_of_type(objects.values(), 'artifact') assert (curr_obj is not None), 'artifact object type not found' assert (curr_obj.keys() == {'type', 'payload_bin'}) assert (curr_obj['payload_bin'] == payload) curr_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') assert (curr_obj is not None), 'user-account object type not found' assert (curr_obj.keys() == {'type', 'user_id'}) assert (curr_obj['user_id'] == user_id) curr_obj = TestTransform.get_first_of_type(objects.values(), 'file') assert (curr_obj is not None), 'file object type not found' assert (curr_obj.keys() == {'type', 'name'}) assert (curr_obj['name'] == file_name) curr_obj = TestTransform.get_first_of_type(objects.values(), 'domain-name') assert (curr_obj is not None), 'domain-name object type not found' assert (curr_obj.keys() == {'type', 'value'}) assert (curr_obj['value'] == 'example.com') assert (objects.keys() == set(map(str, range(0, 10))))
def test_change_cb_binary_api_results_to_stix(self): data = json.loads(""" { "terms": [ "md5:F5AE03DE0AD60F5B17B82F2CD68402FE" ], "total_results": 1, "highlights": [ { "name": "PREPREPREF5AE03DE0AD60F5B17B82F2CD68402FEPOSTPOSTPOST", "ids": [ "F5AE03DE0AD60F5B17B82F2CD68402FE" ] } ], "facets": {}, "results": [ { "host_count": 13, "alliance_updated_srstrust": "2016-09-04T04:59:53Z", "original_filename": "Cmd.Exe.MUI", "legal_copyright": "\u00a9 Microsoft Corporation. All rights reserved.", "digsig_result": "Signed", "observed_filename": [ "c:\\\\windows\\\\system32\\\\cmd.exe" ], "product_version": "6.3.9600.16384", "alliance_score_srstrust": -100, "watchlists": [ { "wid": "5", "value": "2016-10-19T10:20:05.424Z" } ], "facet_id": 431419, "copied_mod_len": 357376, "server_added_timestamp": "2016-10-19T10:00:25.734Z", "digsig_sign_time": "2014-11-07T08:02:00Z", "orig_mod_len": 357376, "alliance_data_srstrust": [ "f5ae03de0ad60f5b17b82f2cd68402fe" ], "is_executable_image": true, "is_64bit": true, "md5": "F5AE03DE0AD60F5B17B82F2CD68402FE", "digsig_publisher": "Microsoft Corporation", "endpoint": [ "ADTWO|24", "ADONE|26", "CERT|27", "REPO|29", "adone|26", "cert|27", "adtwo|24", "iestestmachine3|53", "iestestmachine0|52", "iestestmachine1|54" ], "group": [ "CTF Lab", "Default Group", "ctf lab", "default group" ], "event_partition_id": [ 97777295491072, 97794283536384, 97811271778304, 97828260020224, 97845247737856, 97862235979776, 97879224221696, 97896211152896, 97913199394816 ], "digsig_result_code": "0", "file_version": "6.3.9600.16384 (winblue_rtm.130821-1623)", "signed": "Signed", "alliance_link_srstrust": "https://services.bit9.com/Services/extinfo.aspx?ak=b8b4e631d4884ad1c56f50e4a5ee9279&sg=0313e1735f6cec221b1d686bd4de23ee&md5=f5ae03de0ad60f5b17b82f2cd68402fe", "company_name": "Microsoft Corporation", "internal_name": "cmd", "timestamp": "2016-10-19T10:00:25.734Z", "cb_version": 624, "os_type": "Windows", "file_desc": "Windows Command Processor", "product_name": "Microsoft\u00ae Windows\u00ae Operating System", "last_seen": "2019-01-14T03:19:05.687Z" } ], "elapsed": 0.02470088005065918, "start": 0 }""") binary_interface = carbonblack_translator.Translator() binary_map_file = open(binary_interface.mapping_filepath).read() binary_map_data = json.loads(binary_map_file) results = data["results"] result_bundle = json_to_stix_translator.convert_to_stix(data_source, binary_map_data, results, transformers.get_all_transformers(), options) print(result_bundle) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert('objects' in observed_data) objects = observed_data['objects'] curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'file') file_obj = curr_obj # used in later test assert(curr_obj is not None), 'file object type not found' assert(curr_obj.keys() == {'type', 'name', 'created', 'hashes'}) assert(curr_obj['name'] =="Cmd.Exe.MUI") assert(curr_obj['hashes']['MD5'] == "F5AE03DE0AD60F5B17B82F2CD68402FE")
def test_change_cim_to_stix(self): count = 1 time = "2018-08-21T15:11:55.000+00:00" file_bytes = "300" user = "******" objPath = "hkey_local_machine\\system\\bar\\foo" filePath = "C:\\Users\\someuser\\sample.dll" create_time = "2018-08-15T15:11:55.676+00:00" modify_time = "2018-08-15T18:10:30.456+00:00" file_hash = "41a26255d16d121dc525a6445144b895" file_name = "sample.dll" file_size = 25536 data = { "event_count": count, "_time": time, "user": user, "bytes": file_bytes, "object_path": objPath, "file_path": filePath, "file_create_time": create_time, "file_modify_time": modify_time, "file_hash": file_hash, "file_size": file_size, "file_name": file_name } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] # Test objects in Stix observable data model after transform wrk_obj = TestTransform.get_first_of_type(objects.values(), 'windows-registry-key') assert(wrk_obj is not None) assert(wrk_obj.keys() == {'type', 'key'}) assert(wrk_obj['key'] == "hkey_local_machine\\system\\bar\\foo") user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') assert(user_obj is not None), 'user-account object type not found' assert(user_obj.keys() == {'type', 'account_login', 'user_id'}) assert(user_obj['account_login'] == "ibm_user") assert(user_obj['user_id'] == "ibm_user") file_obj = TestTransform.get_first_of_type(objects.values(), 'file') assert(file_obj is not None), 'file object type not found' assert(file_obj.keys() == {'type', 'parent_directory_ref', 'created', 'modified', 'hashes', 'name', 'size'}) assert(file_obj['created'] == "2018-08-15T15:11:55.676Z") assert(file_obj['modified'] == "2018-08-15T18:10:30.456Z") assert(file_obj['name'] == "sample.dll") assert(file_obj['size'] == 25536) assert (file_obj['hashes']['MD5'] == "41a26255d16d121dc525a6445144b895") dir_ref = file_obj['parent_directory_ref'] assert(dir_ref in objects), f"parent_directory_ref with key {file_obj['parent_directory_ref']} not found" dir_obj = objects[dir_ref] assert(dir_obj is not None), 'directory object type not found' assert(dir_obj.keys() == {'type', 'path', 'created', 'modified'}) assert(dir_obj['path'] == "C:\\Users\\someuser\\sample.dll") assert(dir_obj['created'] == "2018-08-15T15:11:55.676Z") assert(dir_obj['modified'] == "2018-08-15T18:10:30.456Z") print(objects.keys()) print(result_bundle_objects) assert(objects.keys() == set(map(str, range(0, 5))))
def test_cim_to_stix_no_tags(self): data = {"src_ip": "169.250.0.1", "src_port": "1220", "src_mac": "aa:bb:cc:dd:11:22", "dest_ip": "127.0.0.1", "dest_port": "1120", "dest_mac": "ee:dd:bb:aa:cc:11", "file_hash": "cf23df2207d99a74fbe169e3eba035e633b65d94", "user": "******", "url": "https://wally.fireeye.com/malware_analysis/analyses?maid=1", "protocol": "tcp", "_bkt": "main~44~6D3E49A0-31FE-44C3-8373-C3AC6B1ABF06", "_cd": "44:12606114", "_indextime": "1546960685", "_raw": "Jan 08 2019 15:18:04 192.168.33.131 fenotify-2.alert: CEF:0|FireEye|MAS|6.2.0.74298|MO|" "malware-object|4|rt=Jan 08 2019 15:18:04 Z src=169.250.0.1 dpt=1120 dst=127.0.0.1" " spt=1220 smac=AA:BB:CC:DD:11:22 dmac=EE:DD:BB:AA:CC:11 cn2Label=sid cn2=111" " fileHash=41a26255d16d121dc525a6445144b895 proto=tcp " "request=http://qa-server.eng.fireeye.com/QE/NotificationPcaps/" "58.253.68.29_80-192.168.85.128_1165-2119283109_T.exe cs3Label=osinfo" " cs3=Microsoft Windows7 Professional 6.1 sp1 dvchost=wally dvc=10.2.101.101 cn1Label=vlan" " cn1=0 externalId=1 cs4Label=link " "cs4=https://wally.fireeye.com/malware_analysis/analyses?maid=1 cs2Label=anomaly" " cs2=misc-anomaly cs1Label=sname cs1=FE_UPX;Trojan.PWS.OnlineGames", "_serial": "0", "_si": ["splunk3-01.internal.resilientsystems.com", "main"], "_sourcetype": "fe_cef_syslog", "_time": "2019-01-08T15:18:04.000+00:00", "event_count": 1 } result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], transformers.get_all_transformers(), options, callback=hash_type_lookup) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] validated_result = validate_instance(observed_data) assert(validated_result.is_valid == True) assert('objects' in observed_data) objects = observed_data['objects'] nt_obj = TestTransform.get_first_of_type(objects.values(), 'network-traffic') assert(nt_obj is not None), 'network-traffic object type not found' assert(nt_obj.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'}) assert(nt_obj['src_port'] == 1220) assert(nt_obj['dst_port'] == 1120) assert(nt_obj['protocols'] == ['tcp']) nt_obj_2 = objects['2'] assert (nt_obj_2 is not None), 'network-traffic object type not found' assert (nt_obj_2.keys() == {'type', 'src_ref', 'src_port', 'dst_ref', 'dst_port', 'protocols'}) assert (nt_obj_2['src_port'] == 1220) assert (nt_obj_2['dst_port'] == 1120) assert (nt_obj_2['protocols'] == ['tcp']) mac_ref = nt_obj_2['dst_ref'] assert(mac_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found" mac_obj = objects[mac_ref] assert(mac_obj.keys() == {'type', 'value'}) assert(mac_obj['type'] == 'mac-addr') assert(mac_obj['value'] == 'ee:dd:bb:aa:cc:11') mac_ref = nt_obj_2['src_ref'] assert(mac_ref in objects), "src_ref with key {nt_obj['dst_ref']} not found" mac_obj = objects[mac_ref] assert(mac_obj.keys() == {'type', 'value'}) assert(mac_obj['type'] == 'mac-addr') assert(mac_obj['value'] == 'aa:bb:cc:dd:11:22') ip_ref = nt_obj['dst_ref'] assert(ip_ref in objects), "dst_ref with key {nt_obj['dst_ref']} not found" ip_obj = objects[ip_ref] assert(ip_obj.keys() == {'type', 'value'}) assert(ip_obj['type'] == 'ipv4-addr') assert(ip_obj['value'] == '127.0.0.1') ip_ref = nt_obj['src_ref'] assert(ip_ref in objects), "src_ref with key {nt_obj['src_ref']} not found" ip_obj = objects[ip_ref] assert(ip_obj.keys() == {'type', 'value'}) assert(ip_obj['type'] == 'ipv4-addr') assert(ip_obj['value'] == '169.250.0.1') file_obj = TestTransform.get_first_of_type(objects.values(), 'file') assert (file_obj is not None), 'file object type not found' assert (file_obj.keys() == {'type', 'hashes'}) assert (file_obj['hashes']['SHA-1'] == "cf23df2207d99a74fbe169e3eba035e633b65d94") user_obj = TestTransform.get_first_of_type(objects.values(), 'user-account') assert (user_obj is not None), 'user object type not found' assert (user_obj.keys() == {'type', 'account_login', 'user_id'}) assert (user_obj['account_login'] == "sname") assert (user_obj['user_id'] == "sname") url_obj = TestTransform.get_first_of_type(objects.values(), 'url') assert (url_obj is not None), 'url object type not found' assert (url_obj.keys() == {'type', 'value'}) assert (url_obj['value'] == "https://wally.fireeye.com/malware_analysis/analyses?maid=1") domain_obj = TestTransform.get_first_of_type(objects.values(), 'domain-name') assert (domain_obj is not None), 'domain object type not found' assert (domain_obj.keys() == {'type', 'value'}) assert (domain_obj['value'] == "wally.fireeye.com") payload_obj = TestTransform.get_first_of_type(objects.values(), 'artifact') assert (payload_obj is not None), 'payload object type not found' assert (payload_obj.keys() == {'type', 'payload_bin'}) payload = 'SmFuIDA4IDIwMTkgMTU6MTg6MDQgMTkyLjE2OC4zMy4xMzEgZmVub3RpZnktMi5hbGVydDogQ0VGOjB8RmlyZUV5ZXxNQV' \ 'N8Ni4yLjAuNzQyOTh8TU98bWFsd2FyZS1vYmplY3R8NHxydD1KYW4gMDggMjAxOSAxNToxODowNCBaIHNyYz0xNjkuMjUw' \ 'LjAuMSBkcHQ9MTEyMCBkc3Q9MTI3LjAuMC4xIHNwdD0xMjIwIHNtYWM9QUE6QkI6Q0M6REQ6MTE6MjIgZG1hYz1FRTpERD' \ 'pCQjpBQTpDQzoxMSBjbjJMYWJlbD1zaWQgY24yPTExMSBmaWxlSGFzaD00MWEyNjI1NWQxNmQxMjFkYzUyNWE2NDQ1MTQ0' \ 'Yjg5NSBwcm90bz10Y3AgcmVxdWVzdD1odHRwOi8vcWEtc2VydmVyLmVuZy5maXJlZXllLmNvbS9RRS9Ob3RpZmljYXRpb2' \ '5QY2Fwcy81OC4yNTMuNjguMjlfODAtMTkyLjE2OC44NS4xMjhfMTE2NS0yMTE5MjgzMTA5X1QuZXhlIGNzM0xhYmVsPW9z' \ 'aW5mbyBjczM9TWljcm9zb2Z0IFdpbmRvd3M3IFByb2Zlc3Npb25hbCA2LjEgc3AxIGR2Y2hvc3Q9d2FsbHkgZHZjPTEwLj' \ 'IuMTAxLjEwMSBjbjFMYWJlbD12bGFuIGNuMT0wIGV4dGVybmFsSWQ9MSBjczRMYWJlbD1saW5rIGNzND1odHRwczovL3dh' \ 'bGx5LmZpcmVleWUuY29tL21hbHdhcmVfYW5hbHlzaXMvYW5hbHlzZXM/bWFpZD0xIGNzMkxhYmVsPWFub21hbHkgY3MyPW' \ '1pc2MtYW5vbWFseSBjczFMYWJlbD1zbmFtZSBjczE9RkVfVVBYO1Ryb2phbi5QV1MuT25saW5lR2FtZXM=' assert (payload_obj['payload_bin'] == payload)
def test_merge_results_mixed_to_stix(self): process_data = json.loads(""" { "terms": [ "process_name:cmd.exe" ], "results": [ { "process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41", "sensor_id": 50, "filtering_known_dlls": false, "modload_count": 16, "parent_unique_id": "00000032-0000-0a04-01d4-8bc245c6c9e6-000000000001", "emet_count": 0, "cmdline": "cmd /c \\"\\"C:\\\\ProgramData\\\\VMware\\\\VMware CAF\\\\pme\\\\\\\\config\\\\..\\\\scripts\\\\is-listener-running.bat\\" \\"", "filemod_count": 0, "id": "00000032-0000-0888-01d4-95e3b558aacb", "parent_name": "managementagenthost.exe", "parent_md5": "000000000000000000000000000000", "group": "mdr redlab", "parent_id": "00000032-0000-0a04-01d4-8bc245c6c9e6", "hostname": "redlab-vuln2", "last_update": "2018-12-17T08:37:13.396Z", "start": "2018-12-17T08:37:13.318Z", "comms_ip": 212262914, "regmod_count": 0, "interface_ip": 183439305, "process_pid": 2184, "username": "******", "terminated": false, "process_name": "cmd.exe", "emet_config": "", "last_server_update": "2019-02-01T18:44:10.53Z", "path": "c:\\\\windows\\\\system32\\\\cmd.exe", "netconn_count": 0, "parent_pid": 2564, "crossproc_count": 2, "segment_id": 1549046650410, "host_type": "workstation", "processblock_count": 0, "os_type": "windows", "childproc_count": 8, "unique_id": "00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a" }, { "process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41", "sensor_id": 50, "filtering_known_dlls": false, "modload_count": 16, "parent_unique_id": "00000032-0000-0a04-01d4-8bc245c6c9e6-000000000001", "emet_count": 0, "cmdline": "cmd /c \\"\\"C:\\\\ProgramData\\\\VMware\\\\VMware CAF\\\\pme\\\\\\\\config\\\\..\\\\scripts\\\\is-listener-running.bat\\" \\"", "filemod_count": 0, "id": "00000032-0000-0888-01d4-95e3b558aacb", "parent_name": "managementagenthost.exe", "parent_md5": "000000000000000000000000000000", "group": "mdr redlab", "parent_id": "00000032-0000-0a04-01d4-8bc245c6c9e6", "hostname": "redlab-vuln2", "last_update": "2018-12-17T08:37:13.396Z", "start": "2018-12-17T08:37:13.318Z", "comms_ip": 212262914, "regmod_count": 0, "interface_ip": 183439305, "process_pid": 2184, "username": "******", "terminated": false, "process_name": "cmd.exe", "alliance_data_attackframework": [ "565594" ], "emet_config": "", "last_server_update": "2019-02-01T18:50:32.875Z", "path": "c:\\\\windows\\\\system32\\\\cmd.exe", "alliance_score_attackframework": 1, "netconn_count": 0, "parent_pid": 2564, "crossproc_count": 2, "alliance_link_attackframework": "https://attack.mitre.org/wiki/Technique/T1082", "segment_id": 1549047032875, "watchlists": [ { "segments_hit": [ 1549046650410 ], "wid": "1154", "value": "2019-02-01T18:50:06.003Z" } ], "host_type": "workstation", "processblock_count": 0, "alliance_updated_attackframework": "2018-10-16T20:15:04Z", "os_type": "windows", "childproc_count": 8, "unique_id": "00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b" } ], "elapsed": 0.023807048797607422, "comprehensive_search": true, "all_segments": true, "total_results": 77835, "highlights": [ { "name": "PREPREPREcmd.exePOSTPOSTPOST", "ids": [ "00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a", "00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b" ] }, { "name": "C:\\\\Windows\\\\system32\\\\PREPREPREcmd.exePOSTPOSTPOST /S /D /c\\" echo\\"", "ids": [ "00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a" ] }, { "name": "c:\\\\windows\\\\system32\\\\PREPREPREcmd.exePOSTPOSTPOST", "ids": [ "00000032-0000-0888-01d4-95e3b558aacb-0168aa60162a", "00000032-0000-0888-01d4-95e3b558aacb-0168aa65ec2b" ] } ], "facets": {}, "tagged_pids": {}, "start": 0, "incomplete_results": false, "filtered": {} } """) binary_data = json.loads(""" { "terms": [ "observed_filename:notepad.exe" ], "total_results": 10, "highlights": [ { "name": "c:\\\\windows\\\\system32\\\\PREPREPREnotepad.exePOSTPOSTPOST", "ids": [ "FC2EA5BD5307D2CFA5AAA38E0C0DDCE9", "959A31D0CD013CEA0C66DB7C03BCBDDF" ] } ], "facets": {}, "results": [ { "host_count": 4, "alliance_updated_srstrust": "2017-11-05T07:05:38Z", "original_filename": "NOTEPAD.EXE", "legal_copyright": "\\u00a9 Microsoft Corporation. All rights reserved.", "digsig_result": "Signed", "observed_filename": [ "c:\\\\windows\\\\system32\\\\notepad.exe" ], "product_version": "6.3.9600.17930", "alliance_score_srstrust": -100, "watchlists": [ { "wid": "5", "value": "2017-03-14T10:10:05.217Z" } ], "facet_id": 2272, "copied_mod_len": 221184, "server_added_timestamp": "2017-03-14T10:04:35.779Z", "digsig_sign_time": "2015-07-11T00:18:00Z", "orig_mod_len": 221184, "alliance_data_srstrust": [ "fc2ea5bd5307d2cfa5aaa38e0c0ddce9" ], "is_executable_image": true, "is_64bit": true, "md5": "FC2EA5BD5307D2CFA5AAA38E0C0DDCE9", "digsig_publisher": "Microsoft Corporation", "endpoint": [ "REPO|29", "VSPHERE|28", "vsphere|28", "iestestmachine1|54" ], "group": [ "CTF Lab", "ctf lab", "default group" ], "event_partition_id": [ 97777295491072, 98439833845760, 98847548112896, 99679970852864, 101310831263744 ], "digsig_result_code": "0", "file_version": "6.3.9600.17930 (winblue_ltsb.150709-0600)", "signed": "Signed", "alliance_link_srstrust": "https://services.bit9.com/Services/extinfo.aspx?ak=b8b4e631d4884ad1c56f50e4a5ee9279&sg=0313e1735f6cec221b1d686bd4de23ee&md5=fc2ea5bd5307d2cfa5aaa38e0c0ddce9", "company_name": "Microsoft Corporation", "internal_name": "Notepad", "timestamp": "2017-03-14T10:04:35.779Z", "cb_version": 624, "os_type": "Windows", "file_desc": "Notepad", "product_name": "Microsoft\\u00ae Windows\\u00ae Operating System", "last_seen": "2018-12-29T12:41:54.355Z" }, { "host_count": 1, "original_filename": "NOTEPAD.EXE", "legal_copyright": "\\u00a9 Microsoft Corporation. All rights reserved.", "digsig_result": "Signed", "observed_filename": [ "c:\\\\windows\\\\system32\\\\notepad.exe" ], "product_version": "6.3.9600.17415", "watchlists": [ { "wid": "5", "value": "2017-04-12T21:10:04.604Z" } ], "facet_id": 87425, "copied_mod_len": 221184, "server_added_timestamp": "2017-04-12T21:06:15.216Z", "digsig_sign_time": "2014-11-07T07:55:00Z", "orig_mod_len": 221184, "is_executable_image": true, "is_64bit": true, "md5": "959A31D0CD013CEA0C66DB7C03BCBDDF", "digsig_publisher": "Microsoft Corporation", "endpoint": [ "REPO|31" ], "group": [ "Default Group" ], "event_partition_id": [ 97777295491072 ], "digsig_result_code": "0", "file_version": "6.3.9600.17415 (winblue_r4.141028-1500)", "signed": "Signed", "company_name": "Microsoft Corporation", "internal_name": "Notepad", "timestamp": "2017-04-12T21:06:15.216Z", "cb_version": 610, "os_type": "Windows", "file_desc": "Notepad", "product_name": "Microsoft\\u00ae Windows\\u00ae Operating System", "last_seen": "2017-04-12T21:10:06.095Z" } ], "elapsed": 0.011963844299316406, "start": 0 } """) results = process_data["results"] + binary_data["results"] # we assume the data pipeline will combine the results in a list result_bundle = json_to_stix_translator.convert_to_stix(data_source, map_data, results, transformers.get_all_transformers(), options) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] assert(len(result_bundle_objects) == 5) objects = result_bundle_objects[1]['objects'] types = [o.get('type') for o in objects.values()] assert (types == ['file', 'process', 'file', 'process', 'domain-name', 'ipv4-addr', 'network-traffic', 'ipv4-addr', 'user-account']) objects = result_bundle_objects[4]['objects'] types = [o.get('type') for o in objects.values()] assert (types == ['file'])
def test_change_cb_process_api_results_to_stix(self): data = json.loads(""" { "terms": [ "process_name:cmd.exe", "start:[2019-01-22T00:00:00 TO *]" ], "results": [ { "process_md5": "5746bd7e255dd6a8afa06f7c42c1ba41", "sensor_id": 49, "filtering_known_dlls": true, "modload_count": 3, "parent_unique_id": "00000031-0000-09cc-01d4-b1e61979dd7c-000000000001", "emet_count": 0, "alliance_score_srstrust": -100, "cmdline": "C:\\\\Windows\\\\system32\\\\cmd.exe /c tasklist", "alliance_updated_srstrust": "2018-04-05T16:04:34Z", "filemod_count": 0, "id": "00000031-0000-0768-01d4-b1e6197c3edd", "parent_name": "cmd.exe", "parent_md5": "000000000000000000000000000000", "group": "lab1", "parent_id": "00000031-0000-09cc-01d4-b1e61979dd7c", "hostname": "lab1-host1", "last_update": "2019-01-22T00:04:52.937Z", "start": "2019-01-22T00:04:52.875Z", "alliance_link_srstrust": "https://example.com", "comms_ip": -1051309706, "regmod_count": 0, "interface_ip": 183439304, "process_pid": 1896, "username": "******", "terminated": true, "alliance_data_srstrust": [ "5746bd7e255dd6a8afa06f7c42c1ba41" ], "process_name": "cmd.exe", "emet_config": "", "last_server_update": "2019-01-22T00:07:07.064Z", "path": "c:\\\\windows\\\\system32\\\\cmd.exe", "netconn_count": 0, "parent_pid": 2508, "crossproc_count": 2, "segment_id": 1548115627056, "host_type": "workstation", "processblock_count": 0, "os_type": "windows", "childproc_count": 4, "unique_id": "00080031-0000-0748-01d4-b1e61c7c3edd-016872e1cb30" } ], "elapsed": 0.05147600173950195, "comprehensive_search": true, "all_segments": true, "total_results": 1, "highlights": [], "facets": {}, "tagged_pids": {}, "start": 0, "incomplete_results": false, "filtered": {} }""") results = data["results"] result_bundle = json_to_stix_translator.convert_to_stix(data_source, map_data, results, transformers.get_all_transformers(), options) print(result_bundle) assert(result_bundle['type'] == 'bundle') result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert('objects' in observed_data) objects = observed_data['objects'] curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'file') file_obj = curr_obj # used in later test assert(curr_obj is not None), 'file object type not found' assert(curr_obj.keys() == {'type', 'name', 'hashes'}) assert(curr_obj['name'] == "cmd.exe") assert(curr_obj['hashes']['MD5'] == "5746bd7e255dd6a8afa06f7c42c1ba41") curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'user-account') user_obj = curr_obj # used in later test assert(curr_obj is not None), 'user-account object type not found' assert(curr_obj.keys() == {'type', 'user_id'}) assert(curr_obj['user_id'] == "SYSTEM") curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'network-traffic') network_obj = curr_obj # used in later test assert(curr_obj is not None), 'network-traffic object type not found' assert(curr_obj.keys() == {'type', 'src_ref', 'dst_ref'}) assert(objects[curr_obj['src_ref']]['value'] == "10.239.15.200") assert(objects[curr_obj['dst_ref']]['value'] == "193.86.73.118") curr_obj = TestCarbonBlackTransformResults.get_first_of_type(objects.values(), 'process') assert(curr_obj is not None), 'process object type not found' assert(curr_obj.keys() == {'type', 'command_line', 'creator_user_ref', 'binary_ref', 'parent_ref', 'created', 'name', 'pid', 'opened_connection_refs'}) assert(curr_obj['command_line'] == "C:\\Windows\\system32\\cmd.exe /c tasklist") assert(curr_obj['created'] == "2019-01-22T00:04:52.875Z") assert(curr_obj['pid'] == 1896) assert(network_obj == objects[curr_obj['opened_connection_refs'][0]]), 'open_connection_refs does not point to the correct object' assert(file_obj == objects[curr_obj['binary_ref']]), 'process binary_ref does not point to the correct object' assert(user_obj == objects[curr_obj['creator_user_ref']]), 'process creator_user_ref does not point to the correct object' parent_index = curr_obj['parent_ref'] curr_obj = objects[parent_index] assert(curr_obj is not None) assert(curr_obj.keys() == {'type', 'pid', 'name', 'binary_ref'}) assert(curr_obj['pid'] == 2508) assert(curr_obj['name'] == "cmd.exe") assert(objects[curr_obj['binary_ref']]['name'] == "cmd.exe")