def test_custom_mapping(self): data_source_string = json.dumps(data_source) data = [{ "custompayload": "SomeBase64Payload", "url": "www.example.com", "filename": "somefile.exe", "username": "******" }] data_string = json.dumps(data) options = { "mapping": { "username": { "key": "user-account.user_id" }, "identityip": { "key": "x_com_ibm_ariel.identity_ip", "cybox": False }, "qidname": { "key": "x_com_ibm_ariel.qid_name", "cybox": False }, "url": { "key": "url.value" }, "custompayload": { "key": "artifact.payload_bin" } } } translation = stix_translation.StixTranslation() result = translation.translate('qradar', 'results', data_source_string, data_string, options) result_bundle = json.loads(result) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestTransform.get_first_of_type(objects.values(), 'file') assert ( file_object is None ), 'default file object type was returned even though it was not included in the custom mapping' curr_obj = TestTransform.get_first_of_type(objects.values(), 'artifact') assert (curr_obj is not None), 'artifact object type not found' assert (curr_obj.keys() == {'type', 'payload_bin'}) assert (curr_obj['payload_bin'] == "SomeBase64Payload")
def test_custom_mapping(self): data_source_string = json.dumps(data_source) data = [{ 'author_id': 'IBMid-123', 'author_email': '*****@*****.**', 'name': 'test_id_1/providers/sec_adv/occurrences/853092', 'id': '853092', }] data_string = json.dumps(data) options = { "mapping": { "author_id": { "key": "user-account.user_id" }, "id": { "key": "x_finding.id", "cybox": False }, "name": { "key": "x_finding.name", "cybox": False }, "author_email": { "key": "email-addr.value" }, } } translation = stix_translation.StixTranslation() result = translation.translate('security_advisor', 'results', data_source_string, data_string, options) result_bundle = json.loads(result) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type( objects.values(), 'user-account') assert (curr_obj is not None), 'user-account object type not found' assert (curr_obj.keys() == {'type', 'user_id'}) assert (curr_obj['user_id'] == data[0]['author_id']) curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type( objects.values(), 'email-addr') assert (curr_obj is not None), 'email-addr object type not found' assert (curr_obj.keys() == {'type', 'value'}) assert (curr_obj['value'] == "*****@*****.**")
def test_custom_mapping(self): data_source = "{\"type\": \"identity\", \"id\": \"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3\", \"name\": \"Splunk\", \"identity_class\": \"events\"}" data = "[{\"tag\":\"network\", \"src_ip\": \"127.0.0.1\"}]" options = { "mapping": { "tag_to_model": { "network": [ "network-traffic", "dst_ip", "src_ip" ] }, "event_count": { "key": "number_observed", "cybox": False, "transformer": "ToInteger" }, "src_ip": [ { "key": "ipv4-addr.value", "object": "src_ip" }, { "key": "ipv6-addr.value", "object": "src_ip" }, { "key": "network-traffic.src_ref", "object": "network-traffic", "references": "src_ip" } ] } } translation = stix_translation.StixTranslation() result = translation.translate('splunk', 'results', data_source, data, options) result_bundle = json.loads(result) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert('objects' in observed_data) objects = observed_data['objects'] curr_obj = TestTransform.get_first_of_type(objects.values(), 'ipv4-addr') assert(curr_obj is not None), 'ipv4-addr object type not found' assert(curr_obj.keys() == {'type', 'value'}) assert(curr_obj['value'] == "127.0.0.1")
def test_custom_mapping(self): data_source_string = json.dumps(data_source) data = [{ "custompayload": "SomeBase64Payload", "url": "www.example.com", "filename": "somefile.exe", "username": "******" }] data_string = json.dumps(data) options = { "mapping": { "default": { "to_stix": { "username": { "key": "user-account.user_id" }, "url": { "key": "url.value" }, "custompayload": { "key": "artifact.payload_bin" } } } } } translation = stix_translation.StixTranslation() result_bundle = translation.translate('elastic_ecs', 'results', data_source_string, data_string, options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestElasticEcsTransform.get_first_of_type( objects.values(), 'file') assert ( file_object is None ), 'default file object type was returned even though it was not included in the custom mapping' curr_obj = TestElasticEcsTransform.get_first_of_type( objects.values(), 'artifact')
def test_hashtype_lookup_without_matching_logsource_id(self): data_source_string = json.dumps(data_source) data = [{ "sha256hash": "someSHA-256hash", "filehash": "unknownTypeHash", "logsourceid": 123 }] data_string = json.dumps(data) options = { "hash_options": { "generic_name": "filehash", "log_source_id_map": { "2345": "sha-256", "65": "md5" } } } translation = stix_translation.StixTranslation() result = translation.translate('qradar', 'results', data_source_string, data_string, options) result_bundle = json.loads(result) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestTransform.get_first_of_type(objects.values(), 'file') assert (file_object is not None), 'file object not found' assert ('hashes' in file_object), 'file object did not contain hashes' assert ('type' in file_object), 'file object did not contain type' assert ( file_object['type'] == 'file'), 'file object had the wrong type' hashes = file_object['hashes'] assert ('SHA-256' in hashes), 'SHA-256 hash not included' assert ('MD5' not in hashes), 'MD5 hash included' assert ('SHA-1' not in hashes), 'SHA-1 hash included' assert ('UNKNOWN' in hashes), 'UNKNOWN hash not included' assert (hashes['SHA-256'] == 'someSHA-256hash') assert (hashes['UNKNOWN'] == 'unknownTypeHash')
def test_hashtype_lookup_by_length(self): data_source_string = json.dumps(DATA_SOURCE) hashes = {'SHA-256': '05503abea7b8ac0a01db3cb35179242c0c1d43c7002c51e5982318244bdcaba9', 'SHA-1': '05503abea7b8ac0a01db3cb35179242c0c1d43c7', 'MD5': '05503abea7b8ac0a01db3cb35179242c', 'UNKNOWN': '05503abea'} for key, value in hashes.items(): data = [{'filehash': value}] data_string = json.dumps(data) options = {} translation = stix_translation.StixTranslation() result_bundle = translation.translate(MODULE, RESULTS, data_source_string, data_string, options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] objects = observed_data['objects'] file_object = TestTransform.get_first_of_type(objects.values(), 'file') hashes = file_object['hashes'] assert(key in hashes), "{} hash not included".format(key) assert(hashes[key] == value)
def test_unmapped_fallback(self): data_source_string = json.dumps(DATA_SOURCE) data = [{ "sourceip": "127.0.0.1", "destinationip": "127.0.0.2", "sha256hash": "someSHA-256hash", "logsourceid": 123, "filename": "testfile.txt", "filepath": "/unix/files/system/testfile.txt", "unmapped1": "value1", "unmapped2": "value2", "unmapped3": None, "unmapped4": "" }] data_string = json.dumps(data) options = {} translation = stix_translation.StixTranslation() result_bundle = translation.translate('qradar', 'results', data_source_string, data_string, options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('first_observed' in observed_data) assert ('last_observed' in observed_data) assert ('created' in observed_data) assert ('modified' in observed_data) assert ('objects' in observed_data) objects = observed_data['objects'] custom_objects = TestTransform.get_first_of_type( objects.values(), 'x-QRadar') assert (custom_objects['unmapped1'] == "value1") assert (custom_objects['unmapped2'] == "value2") assert 'unmapped3' not in custom_objects.keys() assert 'unmapped4' not in custom_objects.keys()
def test_file_hash_mapping_with_type(self): data_source_string = json.dumps(data_source) data = [{ "filename": "somefile.exe", "sha256hash": "someSHA-256hash", "sha1hash": "someSHA-1hash", "md5hash": "someMD5hash", "logsourceid": 65 }] data_string = json.dumps(data) translation = stix_translation.StixTranslation() result = translation.translate('qradar', 'results', data_source_string, data_string, options) result_bundle = json.loads(result) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestTransform.get_first_of_type(objects.values(), 'file') assert (file_object is not None), 'file object not found' assert ('hashes' in file_object), 'file object did not contain hashes' assert ('name' in file_object), 'file object did not contain name' assert ('type' in file_object), 'file object did not contain type' assert ( file_object['type'] == 'file'), 'file object had the wrong type' assert (file_object['name'] == 'somefile.exe' ), 'file object did not contain the expected name' hashes = file_object['hashes'] assert ('SHA-256' in hashes), 'SHA-256 hash not included' assert ('SHA-1' in hashes), 'SHA-1 hash not included' assert ('MD5' in hashes), 'MD5 hash not included' assert (hashes['SHA-256'] == 'someSHA-256hash') assert (hashes['SHA-1'] == 'someSHA-1hash') assert (hashes['MD5'] == 'someMD5hash')
def test_hashtype_lookup_without_matching_generic_hash_name(self): data_source_string = json.dumps(DATA_SOURCE) data = [{ "filehash": "unknownTypeHash", "sha256hash": "someSHA-256hash", "logsourceid": 123, "filename": "someFile.exe" }] data_string = json.dumps(data) options = { "hash_options": { "generic_name": "someUnknownHashName", "log_source_id_map": { "2345": "sha-256", "65": "md5" } } } translation = stix_translation.StixTranslation() result_bundle = translation.translate(MODULE, RESULTS, data_source_string, data_string, options) result_bundle_objects = result_bundle['objects'] observed_data = result_bundle_objects[1] assert ('objects' in observed_data) objects = observed_data['objects'] file_object = TestTransform.get_first_of_type(objects.values(), 'file') assert (file_object is not None), 'file object not found' hashes = file_object['hashes'] assert ('UNKNOWN' in hashes), 'UNKNOWN hash not included' assert (hashes['UNKNOWN'] == 'unknownTypeHash')
def __main__(): """ Stix-shifter can either be called to either translate or transmit. In the case of translation, stix-shifter either translates a stix pattern to a datasource query, or converts data source query results into JSON of STIX observations. Arguments will take the form of... "translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options> The module and translate_type will determine what module and method gets executed. Option arguments comes in as: "{ "select_fields": <string array of fields in the datasource select statement> (In the case of QRadar), "mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>, "resultSizeLimit": <integer limit number for max results in the data source query>, "timeRange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent> }" In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval. Arguments will take the form of... "transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}', < query <query string>, status <search id>, results <search id> <offset> <length>, ping, is_async > """ # process arguments parent_parser = argparse.ArgumentParser(description='stix_shifter') parent_subparsers = parent_parser.add_subparsers(dest='command') # translate parser translate_parser = parent_subparsers.add_parser( TRANSLATE, help= 'Translate a query or result set using a specific translation module') # positional arguments translate_parser.add_argument('module', choices=stix_translation.TRANSLATION_MODULES, help='The translation module to use') translate_parser.add_argument('translate_type', choices=[ stix_translation.RESULTS, stix_translation.QUERY, stix_translation.PARSE ], help='The translation action to perform') translate_parser.add_argument( 'data_source', help='STIX identity object representing a datasource') translate_parser.add_argument( 'data', type=str, help='The STIX pattern or JSON results to be translated') translate_parser.add_argument('options', nargs='?', help='Options dictionary') translate_parser.add_argument( 'recursion_limit', type=int, nargs='?', help='Maximum depth of Python interpreter stack') # optional arguments translate_parser.add_argument( '-x', '--stix-validator', action='store_true', help='Run the STIX 2 validator against the translated results') # Only supported by Elastic and Splunk translate_parser.add_argument( '-m', '--data-mapper', help= 'optional module to use for Splunk or Elastic STIX-to-query mapping') # transmit parser transmit_parser = parent_subparsers.add_parser( TRANSMIT, help='Connect to a datasource and exectue a query...') # positional arguments transmit_parser.add_argument( 'module', choices=stix_transmission.TRANSMISSION_MODULES, help='Choose which connection module to use') transmit_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate') transmit_parser.add_argument('configuration', type=str, help='Data source authentication') # operation subparser operation_subparser = transmit_parser.add_subparsers( title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") query_operation_parser = operation_subparser.add_parser( stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') results_operation_parser = operation_subparser.add_parser( stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') status_operation_parser = operation_subparser.add_parser( stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') delete_operation_parser = operation_subparser.add_parser( stix_transmission.DELETE, help="Delete a running query on the data source") delete_operation_parser.add_argument('search_id', help='id of query to remove') operation_subparser.add_parser( stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') execute_parser = parent_subparsers.add_parser( EXECUTE, help='Translate and fully execute a query') # positional arguments execute_parser.add_argument('transmission_module', choices=stix_transmission.TRANSMISSION_MODULES, help='Which connection module to use') execute_parser.add_argument('translation_module', choices=stix_translation.TRANSLATION_MODULES, help='Which translation module to use') execute_parser.add_argument( 'data_source', type=str, help='STIX Identity object for the data source') execute_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate') execute_parser.add_argument('configuration', type=str, help='Data source authentication') execute_parser.add_argument('query', type=str, help='Query String') host_parser = parent_subparsers.add_parser( HOST, help='Host a local query service, for testing and development') host_parser.add_argument('data_source', type=str, help='STIX Identity object for the data source') host_parser.add_argument('host_address', type=str, help='Proxy Host:Port') args = parent_parser.parse_args() if args.command is None: parent_parser.print_help(sys.stderr) sys.exit(1) elif args.command == HOST: # Host means to start a local web service for STIX shifter, to use in combination with the proxy data source # module. This combination allows one to run and debug their stix-shifter code locally, while interacting with # it inside a service provider such as IBM Security Connect app = Flask("stix-shifter") @app.route('/transform_query', methods=['POST']) def transform_query(): host = ProxyHost() return host.transform_query() @app.route('/translate_results', methods=['POST']) def translate_results(): data_source_identity_object = args.data_source host = ProxyHost() return host.translate_results(data_source_identity_object) @app.route('/create_query_connection', methods=['POST']) def create_query_connection(): host = ProxyHost() return host.create_query_connection() @app.route('/create_status_connection', methods=['POST']) def create_status_connection(): host = ProxyHost() return host.create_status_connection() @app.route('/create_results_connection', methods=['POST']) def create_results_connection(): host = ProxyHost() return host.create_results_connection() @app.route('/delete_query_connection', methods=['POST']) def delete_query_connection(): host = ProxyHost() return host.delete_query_connection() @app.route('/ping', methods=['POST']) def ping(): host = ProxyHost() return host.ping() @app.route('/is_async', methods=['POST']) def is_async(): host = ProxyHost() return host.is_async() host_address = args.host_address.split(":") app.run(debug=True, port=int(host_address[1]), host=host_address[0]) elif args.command == EXECUTE: # Execute means take the STIX SCO pattern as input, execute query, and return STIX as output translation = stix_translation.StixTranslation() dsl = translation.translate(args.translation_module, 'query', args.data_source, args.query) connection_dict = json.loads(args.connection) configuration_dict = json.loads(args.configuration) transmission = stix_transmission.StixTransmission( args.transmission_module, connection_dict, configuration_dict) results = [] for query in dsl['queries']: search_result = transmission.query(query) if search_result["success"]: search_id = search_result["search_id"] if transmission.is_async(): time.sleep(1) status = transmission.status(search_id) if status['success']: while status['progress'] < 100 and status[ 'status'] == 'RUNNING': print(status) status = transmission.status(search_id) print(status) else: raise RuntimeError("Fetching status failed") result = transmission.results(search_id, 0, 9) if result["success"]: print("Search {} results is:\n{}".format( search_id, result["data"])) # Collect all results results += result["data"] else: raise RuntimeError( "Fetching results failed; see log for details") else: raise RuntimeError( "Search failed to execute; see log for details") # Translate results to STIX result = translation.translate(args.translation_module, 'results', args.data_source, json.dumps(results)) print(result) exit(0) elif args.command == TRANSLATE: options = json.loads(args.options) if bool(args.options) else {} if args.stix_validator: options['stix_validator'] = args.stix_validator if args.data_mapper: options['data_mapper'] = args.data_mapper recursion_limit = args.recursion_limit if args.recursion_limit else 1000 translation = stix_translation.StixTranslation() result = translation.translate(args.module, args.translate_type, args.data_source, args.data, options=options, recursion_limit=recursion_limit) elif args.command == TRANSMIT: result = transmit(args) # stix_transmission print(result) exit(0)
def transform_query(self): query = self.request_args["query"] translation = stix_translation.StixTranslation() dsl = translation.translate(self.module, 'query', '{}', query, self.options) return json.dumps(dsl['queries'])
def transform_query(): request_args = request.get_json( force=True ) translation = stix_translation.StixTranslation() dsl = translation.translate(args.translation_module, 'query', args.data_source, request_args["query"]) return json.dumps(dsl)
def main(): """ Stix-shifter can either be called to either translate or transmit. In the case of translation, stix-shifter either translates a stix pattern to a datasource query, or converts data source query results into JSON of STIX observations. Arguments will take the form of... "translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options> The module and translate_type will determine what module and method gets executed. Option arguments comes in as: "{ "mapping": <mapping hash for stix pattern to datasource and data results to stix observation objects>, "resultSizeLimit": <integer limit number for max results in the data source query>, "timeRange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent> }" In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval. Arguments will take the form of... "transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}', < query <query string>, status <search id>, results <search id> <offset> <length>, ping, is_async > """ # process arguments parent_parser = argparse.ArgumentParser(description='stix_shifter') parent_subparsers = parent_parser.add_subparsers(dest='command') # translate parser translate_parser = parent_subparsers.add_parser( TRANSLATE, help= 'Translate a query or result set using a specific translation module') # positional arguments translate_parser.add_argument('module', help='The translation module to use') translate_parser.add_argument('translate_type', choices=[ stix_translation.RESULTS, stix_translation.QUERY, stix_translation.PARSE ], help='The translation action to perform') translate_parser.add_argument( 'data_source', help='STIX identity object representing a datasource') translate_parser.add_argument( 'data', type=str, help='The STIX pattern or JSON results to be translated') translate_parser.add_argument('options', nargs='?', help='Options dictionary') translate_parser.add_argument( 'recursion_limit', type=int, nargs='?', help='Maximum depth of Python interpreter stack') # optional arguments translate_parser.add_argument( '-x', '--stix-validator', action='store_true', help='Run the STIX 2 validator against the translated results') translate_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') # modules parser parent_subparsers.add_parser(MODULES, help='Get modules list') # mapping parser mapping_parser = parent_subparsers.add_parser(MAPPING, help='Get module mapping') # positional arguments mapping_parser.add_argument('module', help='The translation module to use') # transmit parser transmit_parser = parent_subparsers.add_parser( TRANSMIT, help='Connect to a datasource and exectue a query...') # positional arguments transmit_parser.add_argument('module', help='Choose which connection module to use') transmit_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate') transmit_parser.add_argument('configuration', type=str, help='Data source authentication') transmit_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') # operation subparser operation_subparser = transmit_parser.add_subparsers( title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") query_operation_parser = operation_subparser.add_parser( stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') query_operation_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') results_operation_parser = operation_subparser.add_parser( stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') results_operation_parser.add_argument( '-d', '--debug', action='store_true', help='Print detail logs for debugging') status_operation_parser = operation_subparser.add_parser( stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') status_operation_parser.add_argument( '-d', '--debug', action='store_true', help='Print detail logs for debugging') delete_operation_parser = operation_subparser.add_parser( stix_transmission.DELETE, help="Delete a running query on the data source") delete_operation_parser.add_argument('search_id', help='id of query to remove') delete_operation_parser.add_argument( '-d', '--debug', action='store_true', help='Print detail logs for debugging') operation_subparser.add_parser( stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') execute_parser = parent_subparsers.add_parser( EXECUTE, help='Translate and fully execute a query') # positional arguments execute_parser.add_argument('transmission_module', help='Which connection module to use') execute_parser.add_argument( 'module', help='Which translation module to use for translation') execute_parser.add_argument( 'data_source', type=str, help='STIX Identity object for the data source') execute_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate') execute_parser.add_argument('configuration', type=str, help='Data source authentication') execute_parser.add_argument('query', type=str, help='Query String') execute_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') host_parser = parent_subparsers.add_parser( HOST, help='Host a local query service, for testing and development') host_parser.add_argument('data_source', type=str, help='STIX Identity object for the data source') host_parser.add_argument('host_address', type=str, help='Proxy Host:Port') host_parser.add_argument('-d', '--debug', action='store_true', help='Print detail logs for debugging') args = parent_parser.parse_args() help_and_exit = args.command is None if 'debug' in args and args.debug: utils_logger.init(logging.DEBUG) else: utils_logger.init(logging.INFO) log = utils_logger.set_logger(__name__) if 'module' in args: args_module_dialects = args.module options = {} if 'options' in args and args.options: options = json.loads(args.options) module = process_dialects(args_module_dialects, options)[0] try: importlib.import_module("stix_shifter_modules." + module + ".entry_point") except Exception as ex: log.debug(exception_to_string(ex)) log.error('Module {} not found'.format(module)) log.debug(exception_to_string(ex)) help_and_exit = True if help_and_exit: parent_parser.print_help(sys.stderr) sys.exit(1) elif args.command == HOST: # Host means to start a local web service for STIX shifter, to use in combination with the proxy data source # module. This combination allows one to run and debug their stix-shifter code locally, while interacting with # it inside a service provider such as IBM Security Connect app = Flask("stix-shifter") @app.route('/transform_query', methods=['POST']) def transform_query(): host = ProxyHost() return host.transform_query() @app.route('/translate_results', methods=['POST']) def translate_results(): data_source_identity_object = args.data_source host = ProxyHost() return host.translate_results(data_source_identity_object) @app.route('/create_query_connection', methods=['POST']) def create_query_connection(): host = ProxyHost() return host.create_query_connection() @app.route('/create_status_connection', methods=['POST']) def create_status_connection(): host = ProxyHost() return host.create_status_connection() @app.route('/create_results_connection', methods=['POST']) def create_results_connection(): host = ProxyHost() return host.create_results_connection() @app.route('/delete_query_connection', methods=['POST']) def delete_query_connection(): host = ProxyHost() return host.delete_query_connection() @app.route('/ping', methods=['POST']) def ping_connection(): host = ProxyHost() return host.ping_connection() @app.route('/is_async', methods=['POST']) def is_async(): host = ProxyHost() return host.is_async() host_address = args.host_address.split(":") app.run(debug=False, port=int(host_address[1]), host=host_address[0]) elif args.command == EXECUTE: # Execute means take the STIX SCO pattern as input, execute query, and return STIX as output translation = stix_translation.StixTranslation() connection_dict = json.loads(args.connection) configuration_dict = json.loads(args.configuration) translation_options = copy.deepcopy(connection_dict.get('options', {})) options['validate_pattern'] = True dsl = translation.translate(args.module, 'query', args.data_source, args.query, translation_options) transmission = stix_transmission.StixTransmission( args.transmission_module, connection_dict, configuration_dict) results = [] log.info('Translated Queries: \n' + json.dumps(dsl, indent=4)) if 'queries' not in dsl: exit(1) for query in dsl['queries']: search_result = transmission.query(query) if search_result["success"]: search_id = search_result["search_id"] if transmission.is_async(): time.sleep(1) status = transmission.status(search_id) if status['success']: while status['progress'] < 100 and status[ 'status'] == 'RUNNING': log.debug(status) status = transmission.status(search_id) log.debug(status) else: raise RuntimeError("Fetching status failed") result = transmission.results(search_id, 0, 9) if result["success"]: log.debug("Search {} results is:\n{}".format( search_id, result["data"])) # Collect all results results += result["data"] else: raise RuntimeError( "Fetching results failed; see log for details") else: log.error(str(search_result)) exit(0) # Translate results to STIX translation_options = copy.deepcopy(connection_dict.get('options', {})) options['validate_pattern'] = True result = translation.translate(args.module, 'results', args.data_source, json.dumps(results), translation_options) log.info('STIX Results: \n' + json.dumps(result, indent=4, sort_keys=False)) exit(0) elif args.command == TRANSLATE: data = args.data if not data: data_lines = [] for line in sys.stdin: data_lines.append(line) data = '\n'.join(data_lines) if args.stix_validator: options['stix_validator'] = args.stix_validator recursion_limit = args.recursion_limit if args.recursion_limit else 1000 translation = stix_translation.StixTranslation() result = translation.translate(args.module, args.translate_type, args.data_source, data, options=options, recursion_limit=recursion_limit) elif args.command == MAPPING: translation = stix_translation.StixTranslation() result = translation.translate(args.module, stix_translation.MAPPING, None, None, options=options) elif args.command == MODULES: translation = stix_translation.StixTranslation() result = {} all_modules = modules_list() for m in all_modules: result[m] = translation.translate(m, stix_translation.DIALECTS, None, None) elif args.command == TRANSMIT: result = transmit(args) # stix_transmission print(json.dumps(result, indent=4, sort_keys=False)) exit(0)
async def query_indicator( indicator: Indicator, module: str, opts: dict, sightings_queue: asyncio.Queue ): """ Translates an indicator into a module-specific query and executes it. E.g., if the module is `splunk`, the indicator's pattern is first translated into a valid Splunk query and then executed via the Splunk REST API. @param indicator The indicator to translate and query @param module The module's name, e.g., `splunk` @param opts The module configuration directly taken from the user-defined configuration file `config.yaml` with which this app was started @param sightings_queue The queue to put sightings into """ max_results = opts["max_results"] connection_opts = opts["connection"] transmission_opts = opts.get("transmission", {}) translation_opts = opts.get("translation", {}) data_source = opts["data_source"] ## Translate the pattern to a module-specific query. translation = stix_translation.StixTranslation() dsl = translation.translate( module, "query", indicator, indicator.pattern, translation_opts ) if not dsl.get("queries", None): logger.error( f"Failed to translate STIX-2 indicator with ID '{indicator.id}' to query for module '{module}': {dsl}" ) return logger.debug(f"Translated pattern to {module} query: {dsl}") ## Run the query against the configured endpoint for this module. transmission = stix_transmission.StixTransmission( module, connection_opts, transmission_opts ) query_results = [] for query in dsl["queries"]: search_result = transmission.query(query) if not search_result["success"]: logger.error(str(search_result)) continue search_id = search_result["search_id"] if transmission.is_async(): status = transmission.status(search_id) if not status.get("success", None): logger.error(f"Fetching query status failed for module '{module}'") return while status["progress"] < 100 and status["status"] == "RUNNING": status = transmission.status(search_id) await asyncio.sleep(0.05) result = transmission.results(search_id, 0, max_results) if result["success"]: # Collect all results query_results += result["data"] else: logger.error(f"Fetching results failed for module '{module}': {result}") ## Translate query_results to STIX. if not query_results: return stix_results = translation.translate( module, "results", json.dumps(data_source), json.dumps(query_results), translation_opts, ) ## Parse output and report back sightings to Threat Bus ## The stix_results is always a self-made bundle with at least an `objects` ## field present. The bundle may be invalid STIX though, so we cannot simply ## invoke `parse()`. See this link for details on the bundle stucture: ## https://github.com/opencybersecurityalliance/stix-shifter/blob/3.4.5/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py#L12 objs = stix_results.get("objects", None) if objs is None: logger.error( f"Received STIX bundle without `objects` field, cannot generate sightings: {stix_results}" ) return for sighting in map_bundle_to_sightings(indicator, objs): await sightings_queue.put(sighting)
def __main__(): """ Stix-shifter can either be called to either translate or transmit. In the case of translation, stix-shifter either translates a stix pattern to a datasource query, or converts data source query results into JSON of STIX observations. Arguments will take the form of... "translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options> The module and translate_type will determine what module and method gets executed. Option arguments comes in as: "{ "select_fields": <string array of fields in the datasource select statement> (In the case of QRadar), "mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>, "result_limit": <integer limit number for max results in the data source query>, "timerange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent> }" In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval. Arguments will take the form of... "transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}', < query <query string>, status <search id>, results <search id> <offset> <length>, ping, is_async > """ # process arguments parent_parser = argparse.ArgumentParser(description='stix_shifter') parent_subparsers = parent_parser.add_subparsers(dest='command') # translate parser translate_parser = parent_subparsers.add_parser( TRANSLATE, help='Translate a query or result set using a specific translation module') # positional arguments translate_parser.add_argument( 'module', choices=stix_translation.TRANSLATION_MODULES, help='what translation module to use') translate_parser.add_argument('translate_type', choices=[ stix_translation.RESULTS, stix_translation.QUERY], help='what translation action to perform') translate_parser.add_argument( 'data_source', help='STIX identity object representing a datasource') translate_parser.add_argument( 'data', type=str, help='the data to be translated') translate_parser.add_argument('options', nargs='?', help='options that can be passed in') # optional arguments translate_parser.add_argument('-x', '--stix-validator', action='store_true', help='run stix2 validator against the converted results') translate_parser.add_argument('-m', '--data-mapper', help='module to use for the data mapper') # transmit parser transmit_parser = parent_subparsers.add_parser( TRANSMIT, help='Connect to a datasource and exectue a query...') # positional arguments transmit_parser.add_argument( 'module', choices=stix_transmission.TRANSMISSION_MODULES, help='choose which connection module to use' ) transmit_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate' ) transmit_parser.add_argument( 'configuration', type=str, help='Data source authentication' ) # operation subparser operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') delete_operation_parser = operation_subparser.add_parser(stix_transmission.DELETE, help="Delete a running query on the data source") delete_operation_parser.add_argument('search_id', help='id of query to remove') operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') execute_parser = parent_subparsers.add_parser(EXECUTE, help='Translate and fully execute a query') # positional arguments execute_parser.add_argument( 'transmission_module', choices=stix_transmission.TRANSMISSION_MODULES, help='Which connection module to use' ) execute_parser.add_argument( 'translation_module', choices=stix_translation.TRANSLATION_MODULES, help='Which translation module to use' ) execute_parser.add_argument( 'data_source', type=str, help='STIX Identity object for the data source' ) execute_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate' ) execute_parser.add_argument( 'configuration', type=str, help='Data source authentication' ) execute_parser.add_argument( 'query', type=str, help='Query String' ) args = parent_parser.parse_args() if args.command is None: parent_parser.print_help(sys.stderr) sys.exit(1) if args.command == EXECUTE: #Execute means take the STIX SCO pattern as input, execute query, and return STIX as output translation = stix_translation.StixTranslation() dsl = translation.translate(args.translation_module, 'query', args.data_source, args.query) print("DSL Translation returned {}".format(dsl)) connection_dict = json.loads(args.connection) configuration_dict = json.loads(args.configuration) transmission = stix_transmission.StixTransmission(args.transmission_module, connection_dict, configuration_dict) results = [] for query in dsl['queries']: search_result = transmission.query(query) print("Executed search; returned id is {}".format(search_result)) if search_result["success"]: search_id = search_result["search_id"] if transmission.is_async(): time.sleep(1) status = transmission.status(search_id) while status['progress'] < 100: print( status['progress'] ) result = transmission.results(search_id, 0, 9) if result["success"]: print("Search {} results is:\n{}".format(search_id,result["data"])) # Collect all results results += result["data"] else: raise RuntimeError("Fetching results failed; see log for details") else: raise RuntimeError("Search failed to execute; see log for details") # Translate results to STIX result = translation.translate(args.translation_module, 'results', args.data_source, json.dumps(results) ) print( result ) exit(0) elif args.command == TRANSLATE: options = json.loads(args.options) if bool(args.options) else {} if args.stix_validator: options['stix_validator'] = args.stix_validator if args.data_mapper: options['data_mapper'] = args.data_mapper translation = stix_translation.StixTranslation() result = translation.translate( args.module, args.translate_type, args.data_source, args.data, options=options) elif args.command == TRANSMIT: result = transmit(args) # stix_transmission print(result) exit(0)
def translate_results(): request_args = request.get_json( force=True ) translation = stix_translation.StixTranslation() print( json.dumps(request_args["results"])) dsl = translation.translate(args.translation_module, 'results', args.data_source, request_args["results"]) return json.dumps(dsl)
def __main__(): """ Stix-shifter can either be called to either translate or transmit. In the case of translation, stix-shifter either translates a stix pattern to a datasource query, or converts data source query results into JSON of STIX observations. Arguments will take the form of... "translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options> The module and translate_type will determine what module and method gets executed. Option arguments comes in as: "{ "select_fields": <string array of fields in the datasource select statement> (In the case of QRadar), "mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>, "result_limit": <integer limit number for max results in the data source query>, "timerange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent> }" In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval. Arguments will take the form of... "transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}', < query <query string>, status <search id>, results <search id> <offset> <length>, ping, is_async > """ # process arguments parent_parser = argparse.ArgumentParser(description='stix_shifter') parent_subparsers = parent_parser.add_subparsers(dest='command') # translate parser translate_parser = parent_subparsers.add_parser( TRANSLATE, help='Translate a query or result set using a specific translation module') # positional arguments translate_parser.add_argument( 'module', choices=stix_translation.TRANSLATION_MODULES, help='what translation module to use') translate_parser.add_argument('translate_type', choices=[ stix_translation.RESULTS, stix_translation.QUERY], help='what translation action to perform') translate_parser.add_argument( 'data_source', help='STIX identity object representing a datasource') translate_parser.add_argument( 'data', type=str, help='the data to be translated') translate_parser.add_argument('options', nargs='?', help='options that can be passed in') # optional arguments translate_parser.add_argument('-x', '--stix-validator', action='store_true', help='run stix2 validator against the converted results') translate_parser.add_argument('-m', '--data-mapper', help='module to use for the data mapper') # transmit parser transmit_parser = parent_subparsers.add_parser( TRANSMIT, help='Connect to a datasource and exectue a query...') # positional arguments transmit_parser.add_argument( 'module', choices=stix_transmission.TRANSMISSION_MODULES, help='choose which connection module to use' ) transmit_parser.add_argument( 'connection', type=str, help='Data source connection with host, port, and certificate' ) transmit_parser.add_argument( 'configuration', type=str, help='Data source authentication' ) # operation subparser operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command") operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source") query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source") query_operation_parser.add_argument('query_string', help='native datasource query string') results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query") results_operation_parser.add_argument('search_id', help='uuid of executed query') results_operation_parser.add_argument('offset', help='offset of results') results_operation_parser.add_argument('length', help='length of results') status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query") status_operation_parser.add_argument('search_id', help='uuid of executed query') operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous') args = parent_parser.parse_args() if args.command is None: parent_parser.print_help(sys.stderr) sys.exit(1) if args.command == TRANSLATE: options = json.loads(args.options) if bool(args.options) else {} if args.stix_validator: options['stix_validator'] = args.stix_validator if args.data_mapper: options['data_mapper'] = args.data_mapper translation = stix_translation.StixTranslation() result = translation.translate( args.module, args.translate_type, args.data_source, args.data, options=options) elif args.command == TRANSMIT: result = transmit(args) # stix_transmission print(result) exit(0)
from stix_shifter.stix_translation import stix_translation from stix_shifter_utils.utils.error_response import ErrorCode import unittest import re translation = stix_translation.StixTranslation() def _remove_timestamp_from_query(queries): pattern1 = r"to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}\s*and\s*to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}" pattern2 = r"\{'from':\s*\d{0,13},\s*'to':\s*\d{0,13}\}" if isinstance(queries, list): modified_queries = [] for query in queries: replace_pat1 = re.sub(pattern1, '', str(query)) replace_pat2 = re.sub(pattern2, '{}', replace_pat1) modified_queries.append(replace_pat2) return modified_queries elif isinstance(queries, str): replace_pat1 = re.sub(pattern1, '', queries) return re.sub(pattern2, '{}', replace_pat1) all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6," \ "dst_agent_ip_addresses_v6," \ "action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received," \ "action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name," \ "os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5," \ "action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2," \ "action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time," \ "actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time," \
def stix_shiter_execute(self, config_name: str, stix_query: str): # Execute means take the STIX SCO pattern as input, execute query, and return STIX as output # ref: https://github.com/opencybersecurityalliance/stix-shifter/blob/ee4bdf754fc9c2a80cb5b5607210e53dd2657b72/stix_shifter/scripts/stix_shifter.py#L251 # TODO: wrapper stix-shifter's cml tool to be function to replace this method. config = self.configs[config_name] if 'translation_module' not in config or 'transmission_module' not in config \ or 'connection' not in config or 'configuration' not in config: raise Exception( 'transmission_module, translation_module, connection and configuration should be in config.' ) connection_dict, configuration_dict = config['connection'], config[ 'configuration'], translation_module, transmission_module, data_source = config[ 'translation_module'], config['transmission_module'], {} options = {} if 'options' in connection_dict: options.update(connection_dict['options']) options['validate_pattern'] = True translation = stix_translation.StixTranslation() dsl = translation.translate(translation_module, 'query', data_source, stix_query, options) logging.debug('Translated Queries: ' + json.dumps(dsl)) transmission = stix_transmission.StixTransmission( transmission_module, connection_dict, configuration_dict) results = [] for query in dsl['queries']: search_result = transmission.query(query) if search_result["success"]: search_id = search_result["search_id"] if transmission.is_async(): time.sleep(1) status = transmission.status(search_id) if status['success']: while status['progress'] < 100 and status[ 'status'] == 'RUNNING': logging.debug(status) status = transmission.status(search_id) logging.debug(status) else: raise RuntimeError("Fetching status failed") # TODO: allow user to pass No. of records result = transmission.results(search_id, 0, 1000) if result["success"]: logging.debug("Search {} results is:\n{}".format( search_id, result["data"])) # Collect all results results += result["data"] else: raise RuntimeError( "Fetching results failed; see log for details") else: logging.error(str(search_result)) raise Exception(str( search_result)) # TODO: how to deal with this situation # Translate results to STIX data_source = config['data_source'] result = translation.translate(translation_module, 'results', data_source, json.dumps(results), {"stix_validator": True}) return result