def test_custom_mapping(self):
data_source_string = json.dumps(data_source)
data = [{
"custompayload": "SomeBase64Payload",
"url": "www.example.com",
"filename": "somefile.exe",
"username": "******"
}]
data_string = json.dumps(data)
options = {
"mapping": {
"username": {
"key": "user-account.user_id"
},
"identityip": {
"key": "x_com_ibm_ariel.identity_ip",
"cybox": False
},
"qidname": {
"key": "x_com_ibm_ariel.qid_name",
"cybox": False
},
"url": {
"key": "url.value"
},
"custompayload": {
"key": "artifact.payload_bin"
}
}
}
translation = stix_translation.StixTranslation()
result = translation.translate('qradar', 'results', data_source_string,
data_string, options)
result_bundle = json.loads(result)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestTransform.get_first_of_type(objects.values(), 'file')
assert (
file_object is None
), 'default file object type was returned even though it was not included in the custom mapping'
curr_obj = TestTransform.get_first_of_type(objects.values(),
'artifact')
assert (curr_obj is not None), 'artifact object type not found'
assert (curr_obj.keys() == {'type', 'payload_bin'})
assert (curr_obj['payload_bin'] == "SomeBase64Payload")
def test_custom_mapping(self):
data_source_string = json.dumps(data_source)
data = [{
'author_id': 'IBMid-123',
'author_email': '*****@*****.**',
'name': 'test_id_1/providers/sec_adv/occurrences/853092',
'id': '853092',
}]
data_string = json.dumps(data)
options = {
"mapping": {
"author_id": {
"key": "user-account.user_id"
},
"id": {
"key": "x_finding.id",
"cybox": False
},
"name": {
"key": "x_finding.name",
"cybox": False
},
"author_email": {
"key": "email-addr.value"
},
}
}
translation = stix_translation.StixTranslation()
result = translation.translate('security_advisor', 'results',
data_source_string, data_string,
options)
result_bundle = json.loads(result)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type(
objects.values(), 'user-account')
assert (curr_obj is not None), 'user-account object type not found'
assert (curr_obj.keys() == {'type', 'user_id'})
assert (curr_obj['user_id'] == data[0]['author_id'])
curr_obj = TestSecurityAdvisorResultsToStix.get_first_of_type(
objects.values(), 'email-addr')
assert (curr_obj is not None), 'email-addr object type not found'
assert (curr_obj.keys() == {'type', 'value'})
assert (curr_obj['value'] == "*****@*****.**")
def test_custom_mapping(self):
data_source = "{\"type\": \"identity\", \"id\": \"identity--3532c56d-ea72-48be-a2ad-1a53f4c9c6d3\", \"name\": \"Splunk\", \"identity_class\": \"events\"}"
data = "[{\"tag\":\"network\", \"src_ip\": \"127.0.0.1\"}]"
options = {
"mapping": {
"tag_to_model": {
"network": [
"network-traffic",
"dst_ip",
"src_ip"
]
},
"event_count": {
"key": "number_observed",
"cybox": False,
"transformer": "ToInteger"
},
"src_ip": [
{
"key": "ipv4-addr.value",
"object": "src_ip"
},
{
"key": "ipv6-addr.value",
"object": "src_ip"
},
{
"key": "network-traffic.src_ref",
"object": "network-traffic",
"references": "src_ip"
}
]
}
}
translation = stix_translation.StixTranslation()
result = translation.translate('splunk', 'results', data_source, data, options)
result_bundle = json.loads(result)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert('objects' in observed_data)
objects = observed_data['objects']
curr_obj = TestTransform.get_first_of_type(objects.values(), 'ipv4-addr')
assert(curr_obj is not None), 'ipv4-addr object type not found'
assert(curr_obj.keys() == {'type', 'value'})
assert(curr_obj['value'] == "127.0.0.1")
def test_custom_mapping(self):
data_source_string = json.dumps(data_source)
data = [{
"custompayload": "SomeBase64Payload",
"url": "www.example.com",
"filename": "somefile.exe",
"username": "******"
}]
data_string = json.dumps(data)
options = {
"mapping": {
"default": {
"to_stix": {
"username": {
"key": "user-account.user_id"
},
"url": {
"key": "url.value"
},
"custompayload": {
"key": "artifact.payload_bin"
}
}
}
}
}
translation = stix_translation.StixTranslation()
result_bundle = translation.translate('elastic_ecs', 'results',
data_source_string, data_string,
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'file')
assert (
file_object is None
), 'default file object type was returned even though it was not included in the custom mapping'
curr_obj = TestElasticEcsTransform.get_first_of_type(
objects.values(), 'artifact')
def test_hashtype_lookup_without_matching_logsource_id(self):
data_source_string = json.dumps(data_source)
data = [{
"sha256hash": "someSHA-256hash",
"filehash": "unknownTypeHash",
"logsourceid": 123
}]
data_string = json.dumps(data)
options = {
"hash_options": {
"generic_name": "filehash",
"log_source_id_map": {
"2345": "sha-256",
"65": "md5"
}
}
}
translation = stix_translation.StixTranslation()
result = translation.translate('qradar', 'results', data_source_string,
data_string, options)
result_bundle = json.loads(result)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_object is not None), 'file object not found'
assert ('hashes' in file_object), 'file object did not contain hashes'
assert ('type' in file_object), 'file object did not contain type'
assert (
file_object['type'] == 'file'), 'file object had the wrong type'
hashes = file_object['hashes']
assert ('SHA-256' in hashes), 'SHA-256 hash not included'
assert ('MD5' not in hashes), 'MD5 hash included'
assert ('SHA-1' not in hashes), 'SHA-1 hash included'
assert ('UNKNOWN' in hashes), 'UNKNOWN hash not included'
assert (hashes['SHA-256'] == 'someSHA-256hash')
assert (hashes['UNKNOWN'] == 'unknownTypeHash')
def test_hashtype_lookup_by_length(self):
data_source_string = json.dumps(DATA_SOURCE)
hashes = {'SHA-256': '05503abea7b8ac0a01db3cb35179242c0c1d43c7002c51e5982318244bdcaba9',
'SHA-1': '05503abea7b8ac0a01db3cb35179242c0c1d43c7',
'MD5': '05503abea7b8ac0a01db3cb35179242c',
'UNKNOWN': '05503abea'}
for key, value in hashes.items():
data = [{'filehash': value}]
data_string = json.dumps(data)
options = {}
translation = stix_translation.StixTranslation()
result_bundle = translation.translate(MODULE, RESULTS, data_source_string, data_string, options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
objects = observed_data['objects']
file_object = TestTransform.get_first_of_type(objects.values(), 'file')
hashes = file_object['hashes']
assert(key in hashes), "{} hash not included".format(key)
assert(hashes[key] == value)
def test_unmapped_fallback(self):
data_source_string = json.dumps(DATA_SOURCE)
data = [{
"sourceip": "127.0.0.1",
"destinationip": "127.0.0.2",
"sha256hash": "someSHA-256hash",
"logsourceid": 123,
"filename": "testfile.txt",
"filepath": "/unix/files/system/testfile.txt",
"unmapped1": "value1",
"unmapped2": "value2",
"unmapped3": None,
"unmapped4": ""
}]
data_string = json.dumps(data)
options = {}
translation = stix_translation.StixTranslation()
result_bundle = translation.translate('qradar', 'results',
data_source_string, data_string,
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('first_observed' in observed_data)
assert ('last_observed' in observed_data)
assert ('created' in observed_data)
assert ('modified' in observed_data)
assert ('objects' in observed_data)
objects = observed_data['objects']
custom_objects = TestTransform.get_first_of_type(
objects.values(), 'x-QRadar')
assert (custom_objects['unmapped1'] == "value1")
assert (custom_objects['unmapped2'] == "value2")
assert 'unmapped3' not in custom_objects.keys()
assert 'unmapped4' not in custom_objects.keys()
def test_file_hash_mapping_with_type(self):
data_source_string = json.dumps(data_source)
data = [{
"filename": "somefile.exe",
"sha256hash": "someSHA-256hash",
"sha1hash": "someSHA-1hash",
"md5hash": "someMD5hash",
"logsourceid": 65
}]
data_string = json.dumps(data)
translation = stix_translation.StixTranslation()
result = translation.translate('qradar', 'results', data_source_string,
data_string, options)
result_bundle = json.loads(result)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_object is not None), 'file object not found'
assert ('hashes' in file_object), 'file object did not contain hashes'
assert ('name' in file_object), 'file object did not contain name'
assert ('type' in file_object), 'file object did not contain type'
assert (
file_object['type'] == 'file'), 'file object had the wrong type'
assert (file_object['name'] == 'somefile.exe'
), 'file object did not contain the expected name'
hashes = file_object['hashes']
assert ('SHA-256' in hashes), 'SHA-256 hash not included'
assert ('SHA-1' in hashes), 'SHA-1 hash not included'
assert ('MD5' in hashes), 'MD5 hash not included'
assert (hashes['SHA-256'] == 'someSHA-256hash')
assert (hashes['SHA-1'] == 'someSHA-1hash')
assert (hashes['MD5'] == 'someMD5hash')
def test_hashtype_lookup_without_matching_generic_hash_name(self):
data_source_string = json.dumps(DATA_SOURCE)
data = [{
"filehash": "unknownTypeHash",
"sha256hash": "someSHA-256hash",
"logsourceid": 123,
"filename": "someFile.exe"
}]
data_string = json.dumps(data)
options = {
"hash_options": {
"generic_name": "someUnknownHashName",
"log_source_id_map": {
"2345": "sha-256",
"65": "md5"
}
}
}
translation = stix_translation.StixTranslation()
result_bundle = translation.translate(MODULE, RESULTS,
data_source_string, data_string,
options)
result_bundle_objects = result_bundle['objects']
observed_data = result_bundle_objects[1]
assert ('objects' in observed_data)
objects = observed_data['objects']
file_object = TestTransform.get_first_of_type(objects.values(), 'file')
assert (file_object is not None), 'file object not found'
hashes = file_object['hashes']
assert ('UNKNOWN' in hashes), 'UNKNOWN hash not included'
assert (hashes['UNKNOWN'] == 'unknownTypeHash')
def __main__():
"""
Stix-shifter can either be called to either translate or transmit.
In the case of translation, stix-shifter either translates a stix pattern to a datasource query,
or converts data source query results into JSON of STIX observations.
Arguments will take the form of...
"translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options>
The module and translate_type will determine what module and method gets executed.
Option arguments comes in as:
"{
"select_fields": <string array of fields in the datasource select statement> (In the case of QRadar),
"mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>,
"resultSizeLimit": <integer limit number for max results in the data source query>,
"timeRange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent>
}"
In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval.
Arguments will take the form of...
"transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
<
query <query string>,
status <search id>,
results <search id> <offset> <length>,
ping,
is_async
>
"""
# process arguments
parent_parser = argparse.ArgumentParser(description='stix_shifter')
parent_subparsers = parent_parser.add_subparsers(dest='command')
# translate parser
translate_parser = parent_subparsers.add_parser(
TRANSLATE,
help=
'Translate a query or result set using a specific translation module')
# positional arguments
translate_parser.add_argument('module',
choices=stix_translation.TRANSLATION_MODULES,
help='The translation module to use')
translate_parser.add_argument('translate_type',
choices=[
stix_translation.RESULTS,
stix_translation.QUERY,
stix_translation.PARSE
],
help='The translation action to perform')
translate_parser.add_argument(
'data_source', help='STIX identity object representing a datasource')
translate_parser.add_argument(
'data',
type=str,
help='The STIX pattern or JSON results to be translated')
translate_parser.add_argument('options',
nargs='?',
help='Options dictionary')
translate_parser.add_argument(
'recursion_limit',
type=int,
nargs='?',
help='Maximum depth of Python interpreter stack')
# optional arguments
translate_parser.add_argument(
'-x',
'--stix-validator',
action='store_true',
help='Run the STIX 2 validator against the translated results')
# Only supported by Elastic and Splunk
translate_parser.add_argument(
'-m',
'--data-mapper',
help=
'optional module to use for Splunk or Elastic STIX-to-query mapping')
# transmit parser
transmit_parser = parent_subparsers.add_parser(
TRANSMIT, help='Connect to a datasource and exectue a query...')
# positional arguments
transmit_parser.add_argument(
'module',
choices=stix_transmission.TRANSMISSION_MODULES,
help='Choose which connection module to use')
transmit_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
transmit_parser.add_argument('configuration',
type=str,
help='Data source authentication')
# operation subparser
operation_subparser = transmit_parser.add_subparsers(
title="operation", dest="operation_command")
operation_subparser.add_parser(stix_transmission.PING,
help="Pings the data source")
query_operation_parser = operation_subparser.add_parser(
stix_transmission.QUERY, help="Executes a query on the data source")
query_operation_parser.add_argument('query_string',
help='native datasource query string')
results_operation_parser = operation_subparser.add_parser(
stix_transmission.RESULTS,
help="Fetches the results of the data source query")
results_operation_parser.add_argument('search_id',
help='uuid of executed query')
results_operation_parser.add_argument('offset', help='offset of results')
results_operation_parser.add_argument('length', help='length of results')
status_operation_parser = operation_subparser.add_parser(
stix_transmission.STATUS, help="Gets the current status of the query")
status_operation_parser.add_argument('search_id',
help='uuid of executed query')
delete_operation_parser = operation_subparser.add_parser(
stix_transmission.DELETE,
help="Delete a running query on the data source")
delete_operation_parser.add_argument('search_id',
help='id of query to remove')
operation_subparser.add_parser(
stix_transmission.IS_ASYNC,
help='Checks if the query operation is asynchronous')
execute_parser = parent_subparsers.add_parser(
EXECUTE, help='Translate and fully execute a query')
# positional arguments
execute_parser.add_argument('transmission_module',
choices=stix_transmission.TRANSMISSION_MODULES,
help='Which connection module to use')
execute_parser.add_argument('translation_module',
choices=stix_translation.TRANSLATION_MODULES,
help='Which translation module to use')
execute_parser.add_argument(
'data_source',
type=str,
help='STIX Identity object for the data source')
execute_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
execute_parser.add_argument('configuration',
type=str,
help='Data source authentication')
execute_parser.add_argument('query', type=str, help='Query String')
host_parser = parent_subparsers.add_parser(
HOST, help='Host a local query service, for testing and development')
host_parser.add_argument('data_source',
type=str,
help='STIX Identity object for the data source')
host_parser.add_argument('host_address', type=str, help='Proxy Host:Port')
args = parent_parser.parse_args()
if args.command is None:
parent_parser.print_help(sys.stderr)
sys.exit(1)
elif args.command == HOST:
# Host means to start a local web service for STIX shifter, to use in combination with the proxy data source
# module. This combination allows one to run and debug their stix-shifter code locally, while interacting with
# it inside a service provider such as IBM Security Connect
app = Flask("stix-shifter")
@app.route('/transform_query', methods=['POST'])
def transform_query():
host = ProxyHost()
return host.transform_query()
@app.route('/translate_results', methods=['POST'])
def translate_results():
data_source_identity_object = args.data_source
host = ProxyHost()
return host.translate_results(data_source_identity_object)
@app.route('/create_query_connection', methods=['POST'])
def create_query_connection():
host = ProxyHost()
return host.create_query_connection()
@app.route('/create_status_connection', methods=['POST'])
def create_status_connection():
host = ProxyHost()
return host.create_status_connection()
@app.route('/create_results_connection', methods=['POST'])
def create_results_connection():
host = ProxyHost()
return host.create_results_connection()
@app.route('/delete_query_connection', methods=['POST'])
def delete_query_connection():
host = ProxyHost()
return host.delete_query_connection()
@app.route('/ping', methods=['POST'])
def ping():
host = ProxyHost()
return host.ping()
@app.route('/is_async', methods=['POST'])
def is_async():
host = ProxyHost()
return host.is_async()
host_address = args.host_address.split(":")
app.run(debug=True, port=int(host_address[1]), host=host_address[0])
elif args.command == EXECUTE:
# Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
translation = stix_translation.StixTranslation()
dsl = translation.translate(args.translation_module, 'query',
args.data_source, args.query)
connection_dict = json.loads(args.connection)
configuration_dict = json.loads(args.configuration)
transmission = stix_transmission.StixTransmission(
args.transmission_module, connection_dict, configuration_dict)
results = []
for query in dsl['queries']:
search_result = transmission.query(query)
if search_result["success"]:
search_id = search_result["search_id"]
if transmission.is_async():
time.sleep(1)
status = transmission.status(search_id)
if status['success']:
while status['progress'] < 100 and status[
'status'] == 'RUNNING':
print(status)
status = transmission.status(search_id)
print(status)
else:
raise RuntimeError("Fetching status failed")
result = transmission.results(search_id, 0, 9)
if result["success"]:
print("Search {} results is:\n{}".format(
search_id, result["data"]))
# Collect all results
results += result["data"]
else:
raise RuntimeError(
"Fetching results failed; see log for details")
else:
raise RuntimeError(
"Search failed to execute; see log for details")
# Translate results to STIX
result = translation.translate(args.translation_module, 'results',
args.data_source, json.dumps(results))
print(result)
exit(0)
elif args.command == TRANSLATE:
options = json.loads(args.options) if bool(args.options) else {}
if args.stix_validator:
options['stix_validator'] = args.stix_validator
if args.data_mapper:
options['data_mapper'] = args.data_mapper
recursion_limit = args.recursion_limit if args.recursion_limit else 1000
translation = stix_translation.StixTranslation()
result = translation.translate(args.module,
args.translate_type,
args.data_source,
args.data,
options=options,
recursion_limit=recursion_limit)
elif args.command == TRANSMIT:
result = transmit(args) # stix_transmission
print(result)
exit(0)
def transform_query(self):
query = self.request_args["query"]
translation = stix_translation.StixTranslation()
dsl = translation.translate(self.module, 'query', '{}', query,
self.options)
return json.dumps(dsl['queries'])
def transform_query():
request_args = request.get_json( force=True )
translation = stix_translation.StixTranslation()
dsl = translation.translate(args.translation_module, 'query', args.data_source, request_args["query"])
return json.dumps(dsl)
def main():
"""
Stix-shifter can either be called to either translate or transmit.
In the case of translation, stix-shifter either translates a stix pattern to a datasource query,
or converts data source query results into JSON of STIX observations.
Arguments will take the form of...
"translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options>
The module and translate_type will determine what module and method gets executed.
Option arguments comes in as:
"{
"mapping": <mapping hash for stix pattern to datasource and data results to stix observation objects>,
"resultSizeLimit": <integer limit number for max results in the data source query>,
"timeRange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent>
}"
In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval.
Arguments will take the form of...
"transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
<
query <query string>,
status <search id>,
results <search id> <offset> <length>,
ping,
is_async
>
"""
# process arguments
parent_parser = argparse.ArgumentParser(description='stix_shifter')
parent_subparsers = parent_parser.add_subparsers(dest='command')
# translate parser
translate_parser = parent_subparsers.add_parser(
TRANSLATE,
help=
'Translate a query or result set using a specific translation module')
# positional arguments
translate_parser.add_argument('module',
help='The translation module to use')
translate_parser.add_argument('translate_type',
choices=[
stix_translation.RESULTS,
stix_translation.QUERY,
stix_translation.PARSE
],
help='The translation action to perform')
translate_parser.add_argument(
'data_source', help='STIX identity object representing a datasource')
translate_parser.add_argument(
'data',
type=str,
help='The STIX pattern or JSON results to be translated')
translate_parser.add_argument('options',
nargs='?',
help='Options dictionary')
translate_parser.add_argument(
'recursion_limit',
type=int,
nargs='?',
help='Maximum depth of Python interpreter stack')
# optional arguments
translate_parser.add_argument(
'-x',
'--stix-validator',
action='store_true',
help='Run the STIX 2 validator against the translated results')
translate_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
# modules parser
parent_subparsers.add_parser(MODULES, help='Get modules list')
# mapping parser
mapping_parser = parent_subparsers.add_parser(MAPPING,
help='Get module mapping')
# positional arguments
mapping_parser.add_argument('module', help='The translation module to use')
# transmit parser
transmit_parser = parent_subparsers.add_parser(
TRANSMIT, help='Connect to a datasource and exectue a query...')
# positional arguments
transmit_parser.add_argument('module',
help='Choose which connection module to use')
transmit_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
transmit_parser.add_argument('configuration',
type=str,
help='Data source authentication')
transmit_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
# operation subparser
operation_subparser = transmit_parser.add_subparsers(
title="operation", dest="operation_command")
operation_subparser.add_parser(stix_transmission.PING,
help="Pings the data source")
query_operation_parser = operation_subparser.add_parser(
stix_transmission.QUERY, help="Executes a query on the data source")
query_operation_parser.add_argument('query_string',
help='native datasource query string')
query_operation_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
results_operation_parser = operation_subparser.add_parser(
stix_transmission.RESULTS,
help="Fetches the results of the data source query")
results_operation_parser.add_argument('search_id',
help='uuid of executed query')
results_operation_parser.add_argument('offset', help='offset of results')
results_operation_parser.add_argument('length', help='length of results')
results_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
status_operation_parser = operation_subparser.add_parser(
stix_transmission.STATUS, help="Gets the current status of the query")
status_operation_parser.add_argument('search_id',
help='uuid of executed query')
status_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
delete_operation_parser = operation_subparser.add_parser(
stix_transmission.DELETE,
help="Delete a running query on the data source")
delete_operation_parser.add_argument('search_id',
help='id of query to remove')
delete_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
operation_subparser.add_parser(
stix_transmission.IS_ASYNC,
help='Checks if the query operation is asynchronous')
execute_parser = parent_subparsers.add_parser(
EXECUTE, help='Translate and fully execute a query')
# positional arguments
execute_parser.add_argument('transmission_module',
help='Which connection module to use')
execute_parser.add_argument(
'module', help='Which translation module to use for translation')
execute_parser.add_argument(
'data_source',
type=str,
help='STIX Identity object for the data source')
execute_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
execute_parser.add_argument('configuration',
type=str,
help='Data source authentication')
execute_parser.add_argument('query', type=str, help='Query String')
execute_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
host_parser = parent_subparsers.add_parser(
HOST, help='Host a local query service, for testing and development')
host_parser.add_argument('data_source',
type=str,
help='STIX Identity object for the data source')
host_parser.add_argument('host_address', type=str, help='Proxy Host:Port')
host_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
args = parent_parser.parse_args()
help_and_exit = args.command is None
if 'debug' in args and args.debug:
utils_logger.init(logging.DEBUG)
else:
utils_logger.init(logging.INFO)
log = utils_logger.set_logger(__name__)
if 'module' in args:
args_module_dialects = args.module
options = {}
if 'options' in args and args.options:
options = json.loads(args.options)
module = process_dialects(args_module_dialects, options)[0]
try:
importlib.import_module("stix_shifter_modules." + module +
".entry_point")
except Exception as ex:
log.debug(exception_to_string(ex))
log.error('Module {} not found'.format(module))
log.debug(exception_to_string(ex))
help_and_exit = True
if help_and_exit:
parent_parser.print_help(sys.stderr)
sys.exit(1)
elif args.command == HOST:
# Host means to start a local web service for STIX shifter, to use in combination with the proxy data source
# module. This combination allows one to run and debug their stix-shifter code locally, while interacting with
# it inside a service provider such as IBM Security Connect
app = Flask("stix-shifter")
@app.route('/transform_query', methods=['POST'])
def transform_query():
host = ProxyHost()
return host.transform_query()
@app.route('/translate_results', methods=['POST'])
def translate_results():
data_source_identity_object = args.data_source
host = ProxyHost()
return host.translate_results(data_source_identity_object)
@app.route('/create_query_connection', methods=['POST'])
def create_query_connection():
host = ProxyHost()
return host.create_query_connection()
@app.route('/create_status_connection', methods=['POST'])
def create_status_connection():
host = ProxyHost()
return host.create_status_connection()
@app.route('/create_results_connection', methods=['POST'])
def create_results_connection():
host = ProxyHost()
return host.create_results_connection()
@app.route('/delete_query_connection', methods=['POST'])
def delete_query_connection():
host = ProxyHost()
return host.delete_query_connection()
@app.route('/ping', methods=['POST'])
def ping_connection():
host = ProxyHost()
return host.ping_connection()
@app.route('/is_async', methods=['POST'])
def is_async():
host = ProxyHost()
return host.is_async()
host_address = args.host_address.split(":")
app.run(debug=False, port=int(host_address[1]), host=host_address[0])
elif args.command == EXECUTE:
# Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
translation = stix_translation.StixTranslation()
connection_dict = json.loads(args.connection)
configuration_dict = json.loads(args.configuration)
translation_options = copy.deepcopy(connection_dict.get('options', {}))
options['validate_pattern'] = True
dsl = translation.translate(args.module, 'query', args.data_source,
args.query, translation_options)
transmission = stix_transmission.StixTransmission(
args.transmission_module, connection_dict, configuration_dict)
results = []
log.info('Translated Queries: \n' + json.dumps(dsl, indent=4))
if 'queries' not in dsl:
exit(1)
for query in dsl['queries']:
search_result = transmission.query(query)
if search_result["success"]:
search_id = search_result["search_id"]
if transmission.is_async():
time.sleep(1)
status = transmission.status(search_id)
if status['success']:
while status['progress'] < 100 and status[
'status'] == 'RUNNING':
log.debug(status)
status = transmission.status(search_id)
log.debug(status)
else:
raise RuntimeError("Fetching status failed")
result = transmission.results(search_id, 0, 9)
if result["success"]:
log.debug("Search {} results is:\n{}".format(
search_id, result["data"]))
# Collect all results
results += result["data"]
else:
raise RuntimeError(
"Fetching results failed; see log for details")
else:
log.error(str(search_result))
exit(0)
# Translate results to STIX
translation_options = copy.deepcopy(connection_dict.get('options', {}))
options['validate_pattern'] = True
result = translation.translate(args.module, 'results',
args.data_source, json.dumps(results),
translation_options)
log.info('STIX Results: \n' +
json.dumps(result, indent=4, sort_keys=False))
exit(0)
elif args.command == TRANSLATE:
data = args.data
if not data:
data_lines = []
for line in sys.stdin:
data_lines.append(line)
data = '\n'.join(data_lines)
if args.stix_validator:
options['stix_validator'] = args.stix_validator
recursion_limit = args.recursion_limit if args.recursion_limit else 1000
translation = stix_translation.StixTranslation()
result = translation.translate(args.module,
args.translate_type,
args.data_source,
data,
options=options,
recursion_limit=recursion_limit)
elif args.command == MAPPING:
translation = stix_translation.StixTranslation()
result = translation.translate(args.module,
stix_translation.MAPPING,
None,
None,
options=options)
elif args.command == MODULES:
translation = stix_translation.StixTranslation()
result = {}
all_modules = modules_list()
for m in all_modules:
result[m] = translation.translate(m, stix_translation.DIALECTS,
None, None)
elif args.command == TRANSMIT:
result = transmit(args) # stix_transmission
print(json.dumps(result, indent=4, sort_keys=False))
exit(0)
async def query_indicator(
indicator: Indicator, module: str, opts: dict, sightings_queue: asyncio.Queue
):
"""
Translates an indicator into a module-specific query and executes it. E.g.,
if the module is `splunk`, the indicator's pattern is first translated into
a valid Splunk query and then executed via the Splunk REST API.
@param indicator The indicator to translate and query
@param module The module's name, e.g., `splunk`
@param opts The module configuration directly taken from the user-defined
configuration file `config.yaml` with which this app was started
@param sightings_queue The queue to put sightings into
"""
max_results = opts["max_results"]
connection_opts = opts["connection"]
transmission_opts = opts.get("transmission", {})
translation_opts = opts.get("translation", {})
data_source = opts["data_source"]
## Translate the pattern to a module-specific query.
translation = stix_translation.StixTranslation()
dsl = translation.translate(
module, "query", indicator, indicator.pattern, translation_opts
)
if not dsl.get("queries", None):
logger.error(
f"Failed to translate STIX-2 indicator with ID '{indicator.id}' to query for module '{module}': {dsl}"
)
return
logger.debug(f"Translated pattern to {module} query: {dsl}")
## Run the query against the configured endpoint for this module.
transmission = stix_transmission.StixTransmission(
module, connection_opts, transmission_opts
)
query_results = []
for query in dsl["queries"]:
search_result = transmission.query(query)
if not search_result["success"]:
logger.error(str(search_result))
continue
search_id = search_result["search_id"]
if transmission.is_async():
status = transmission.status(search_id)
if not status.get("success", None):
logger.error(f"Fetching query status failed for module '{module}'")
return
while status["progress"] < 100 and status["status"] == "RUNNING":
status = transmission.status(search_id)
await asyncio.sleep(0.05)
result = transmission.results(search_id, 0, max_results)
if result["success"]:
# Collect all results
query_results += result["data"]
else:
logger.error(f"Fetching results failed for module '{module}': {result}")
## Translate query_results to STIX.
if not query_results:
return
stix_results = translation.translate(
module,
"results",
json.dumps(data_source),
json.dumps(query_results),
translation_opts,
)
## Parse output and report back sightings to Threat Bus
## The stix_results is always a self-made bundle with at least an `objects`
## field present. The bundle may be invalid STIX though, so we cannot simply
## invoke `parse()`. See this link for details on the bundle stucture:
## https://github.com/opencybersecurityalliance/stix-shifter/blob/3.4.5/stix_shifter_utils/stix_translation/src/json_to_stix/json_to_stix_translator.py#L12
objs = stix_results.get("objects", None)
if objs is None:
logger.error(
f"Received STIX bundle without `objects` field, cannot generate sightings: {stix_results}"
)
return
for sighting in map_bundle_to_sightings(indicator, objs):
await sightings_queue.put(sighting)
def __main__():
"""
Stix-shifter can either be called to either translate or transmit.
In the case of translation, stix-shifter either translates a stix pattern to a datasource query,
or converts data source query results into JSON of STIX observations.
Arguments will take the form of...
"translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options>
The module and translate_type will determine what module and method gets executed.
Option arguments comes in as:
"{
"select_fields": <string array of fields in the datasource select statement> (In the case of QRadar),
"mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>,
"result_limit": <integer limit number for max results in the data source query>,
"timerange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent>
}"
In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval.
Arguments will take the form of...
"transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
<
query <query string>,
status <search id>,
results <search id> <offset> <length>,
ping,
is_async
>
"""
# process arguments
parent_parser = argparse.ArgumentParser(description='stix_shifter')
parent_subparsers = parent_parser.add_subparsers(dest='command')
# translate parser
translate_parser = parent_subparsers.add_parser(
TRANSLATE, help='Translate a query or result set using a specific translation module')
# positional arguments
translate_parser.add_argument(
'module', choices=stix_translation.TRANSLATION_MODULES, help='what translation module to use')
translate_parser.add_argument('translate_type', choices=[
stix_translation.RESULTS, stix_translation.QUERY], help='what translation action to perform')
translate_parser.add_argument(
'data_source', help='STIX identity object representing a datasource')
translate_parser.add_argument(
'data', type=str, help='the data to be translated')
translate_parser.add_argument('options', nargs='?', help='options that can be passed in')
# optional arguments
translate_parser.add_argument('-x', '--stix-validator', action='store_true',
help='run stix2 validator against the converted results')
translate_parser.add_argument('-m', '--data-mapper',
help='module to use for the data mapper')
# transmit parser
transmit_parser = parent_subparsers.add_parser(
TRANSMIT, help='Connect to a datasource and exectue a query...')
# positional arguments
transmit_parser.add_argument(
'module', choices=stix_transmission.TRANSMISSION_MODULES,
help='choose which connection module to use'
)
transmit_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate'
)
transmit_parser.add_argument(
'configuration',
type=str,
help='Data source authentication'
)
# operation subparser
operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command")
operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source")
query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source")
query_operation_parser.add_argument('query_string', help='native datasource query string')
results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query")
results_operation_parser.add_argument('search_id', help='uuid of executed query')
results_operation_parser.add_argument('offset', help='offset of results')
results_operation_parser.add_argument('length', help='length of results')
status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query")
status_operation_parser.add_argument('search_id', help='uuid of executed query')
delete_operation_parser = operation_subparser.add_parser(stix_transmission.DELETE, help="Delete a running query on the data source")
delete_operation_parser.add_argument('search_id', help='id of query to remove')
operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous')
execute_parser = parent_subparsers.add_parser(EXECUTE, help='Translate and fully execute a query')
# positional arguments
execute_parser.add_argument(
'transmission_module', choices=stix_transmission.TRANSMISSION_MODULES,
help='Which connection module to use'
)
execute_parser.add_argument(
'translation_module', choices=stix_translation.TRANSLATION_MODULES,
help='Which translation module to use'
)
execute_parser.add_argument(
'data_source',
type=str,
help='STIX Identity object for the data source'
)
execute_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate'
)
execute_parser.add_argument(
'configuration',
type=str,
help='Data source authentication'
)
execute_parser.add_argument(
'query',
type=str,
help='Query String'
)
args = parent_parser.parse_args()
if args.command is None:
parent_parser.print_help(sys.stderr)
sys.exit(1)
if args.command == EXECUTE:
#Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
translation = stix_translation.StixTranslation()
dsl = translation.translate(args.translation_module, 'query', args.data_source, args.query)
print("DSL Translation returned {}".format(dsl))
connection_dict = json.loads(args.connection)
configuration_dict = json.loads(args.configuration)
transmission = stix_transmission.StixTransmission(args.transmission_module, connection_dict, configuration_dict)
results = []
for query in dsl['queries']:
search_result = transmission.query(query)
print("Executed search; returned id is {}".format(search_result))
if search_result["success"]:
search_id = search_result["search_id"]
if transmission.is_async():
time.sleep(1)
status = transmission.status(search_id)
while status['progress'] < 100:
print( status['progress'] )
result = transmission.results(search_id, 0, 9)
if result["success"]:
print("Search {} results is:\n{}".format(search_id,result["data"]))
# Collect all results
results += result["data"]
else:
raise RuntimeError("Fetching results failed; see log for details")
else:
raise RuntimeError("Search failed to execute; see log for details")
# Translate results to STIX
result = translation.translate(args.translation_module, 'results', args.data_source, json.dumps(results) )
print( result )
exit(0)
elif args.command == TRANSLATE:
options = json.loads(args.options) if bool(args.options) else {}
if args.stix_validator:
options['stix_validator'] = args.stix_validator
if args.data_mapper:
options['data_mapper'] = args.data_mapper
translation = stix_translation.StixTranslation()
result = translation.translate(
args.module, args.translate_type, args.data_source, args.data, options=options)
elif args.command == TRANSMIT:
result = transmit(args) # stix_transmission
print(result)
exit(0)
def translate_results():
request_args = request.get_json( force=True )
translation = stix_translation.StixTranslation()
print( json.dumps(request_args["results"]))
dsl = translation.translate(args.translation_module, 'results', args.data_source, request_args["results"])
return json.dumps(dsl)
def __main__():
"""
Stix-shifter can either be called to either translate or transmit.
In the case of translation, stix-shifter either translates a stix pattern to a datasource query,
or converts data source query results into JSON of STIX observations.
Arguments will take the form of...
"translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options>
The module and translate_type will determine what module and method gets executed.
Option arguments comes in as:
"{
"select_fields": <string array of fields in the datasource select statement> (In the case of QRadar),
"mapping": <mapping hash for either stix pattern to datasource or mapping hash for data results to stix observation objects>,
"result_limit": <integer limit number for max results in the data source query>,
"timerange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent>
}"
In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval.
Arguments will take the form of...
"transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
<
query <query string>,
status <search id>,
results <search id> <offset> <length>,
ping,
is_async
>
"""
# process arguments
parent_parser = argparse.ArgumentParser(description='stix_shifter')
parent_subparsers = parent_parser.add_subparsers(dest='command')
# translate parser
translate_parser = parent_subparsers.add_parser(
TRANSLATE, help='Translate a query or result set using a specific translation module')
# positional arguments
translate_parser.add_argument(
'module', choices=stix_translation.TRANSLATION_MODULES, help='what translation module to use')
translate_parser.add_argument('translate_type', choices=[
stix_translation.RESULTS, stix_translation.QUERY], help='what translation action to perform')
translate_parser.add_argument(
'data_source', help='STIX identity object representing a datasource')
translate_parser.add_argument(
'data', type=str, help='the data to be translated')
translate_parser.add_argument('options', nargs='?', help='options that can be passed in')
# optional arguments
translate_parser.add_argument('-x', '--stix-validator', action='store_true',
help='run stix2 validator against the converted results')
translate_parser.add_argument('-m', '--data-mapper',
help='module to use for the data mapper')
# transmit parser
transmit_parser = parent_subparsers.add_parser(
TRANSMIT, help='Connect to a datasource and exectue a query...')
# positional arguments
transmit_parser.add_argument(
'module', choices=stix_transmission.TRANSMISSION_MODULES,
help='choose which connection module to use'
)
transmit_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate'
)
transmit_parser.add_argument(
'configuration',
type=str,
help='Data source authentication'
)
# operation subparser
operation_subparser = transmit_parser.add_subparsers(title="operation", dest="operation_command")
operation_subparser.add_parser(stix_transmission.PING, help="Pings the data source")
query_operation_parser = operation_subparser.add_parser(stix_transmission.QUERY, help="Executes a query on the data source")
query_operation_parser.add_argument('query_string', help='native datasource query string')
results_operation_parser = operation_subparser.add_parser(stix_transmission.RESULTS, help="Fetches the results of the data source query")
results_operation_parser.add_argument('search_id', help='uuid of executed query')
results_operation_parser.add_argument('offset', help='offset of results')
results_operation_parser.add_argument('length', help='length of results')
status_operation_parser = operation_subparser.add_parser(stix_transmission.STATUS, help="Gets the current status of the query")
status_operation_parser.add_argument('search_id', help='uuid of executed query')
operation_subparser.add_parser(stix_transmission.IS_ASYNC, help='Checks if the query operation is asynchronous')
args = parent_parser.parse_args()
if args.command is None:
parent_parser.print_help(sys.stderr)
sys.exit(1)
if args.command == TRANSLATE:
options = json.loads(args.options) if bool(args.options) else {}
if args.stix_validator:
options['stix_validator'] = args.stix_validator
if args.data_mapper:
options['data_mapper'] = args.data_mapper
translation = stix_translation.StixTranslation()
result = translation.translate(
args.module, args.translate_type, args.data_source, args.data, options=options)
elif args.command == TRANSMIT:
result = transmit(args) # stix_transmission
print(result)
exit(0)
from stix_shifter.stix_translation import stix_translation
from stix_shifter_utils.utils.error_response import ErrorCode
import unittest
import re
translation = stix_translation.StixTranslation()
def _remove_timestamp_from_query(queries):
pattern1 = r"to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}\s*and\s*to_epoch\(_time,\"millis\"\)\s*.=\s*\d{0,13}"
pattern2 = r"\{'from':\s*\d{0,13},\s*'to':\s*\d{0,13}\}"
if isinstance(queries, list):
modified_queries = []
for query in queries:
replace_pat1 = re.sub(pattern1, '', str(query))
replace_pat2 = re.sub(pattern2, '{}', replace_pat1)
modified_queries.append(replace_pat2)
return modified_queries
elif isinstance(queries, str):
replace_pat1 = re.sub(pattern1, '', queries)
return re.sub(pattern2, '{}', replace_pat1)
all_fields = "dataset_name,action_local_ip,action_remote_ip,agent_ip_addresses,agent_ip_addresses_v6," \
"dst_agent_ip_addresses_v6," \
"action_local_port,action_remote_port,action_network_protocol,action_pkts_sent,action_pkts_received," \
"action_file_name,action_process_image_name,actor_process_image_name,causality_actor_process_image_name," \
"os_actor_process_image_name,action_file_size,action_file_md5,action_module_md5," \
"action_process_image_md5,action_file_authenticode_sha1,action_file_authenticode_sha2," \
"action_file_sha256,action_module_sha256,action_process_image_sha256,action_file_access_time," \
"actor_process_file_access_time,os_actor_process_file_access_time,action_file_mod_time," \
def stix_shiter_execute(self, config_name: str, stix_query: str):
# Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
# ref: https://github.com/opencybersecurityalliance/stix-shifter/blob/ee4bdf754fc9c2a80cb5b5607210e53dd2657b72/stix_shifter/scripts/stix_shifter.py#L251
# TODO: wrapper stix-shifter's cml tool to be function to replace this method.
config = self.configs[config_name]
if 'translation_module' not in config or 'transmission_module' not in config \
or 'connection' not in config or 'configuration' not in config:
raise Exception(
'transmission_module, translation_module, connection and configuration should be in config.'
)
connection_dict, configuration_dict = config['connection'], config[
'configuration'],
translation_module, transmission_module, data_source = config[
'translation_module'], config['transmission_module'], {}
options = {}
if 'options' in connection_dict:
options.update(connection_dict['options'])
options['validate_pattern'] = True
translation = stix_translation.StixTranslation()
dsl = translation.translate(translation_module, 'query', data_source,
stix_query, options)
logging.debug('Translated Queries: ' + json.dumps(dsl))
transmission = stix_transmission.StixTransmission(
transmission_module, connection_dict, configuration_dict)
results = []
for query in dsl['queries']:
search_result = transmission.query(query)
if search_result["success"]:
search_id = search_result["search_id"]
if transmission.is_async():
time.sleep(1)
status = transmission.status(search_id)
if status['success']:
while status['progress'] < 100 and status[
'status'] == 'RUNNING':
logging.debug(status)
status = transmission.status(search_id)
logging.debug(status)
else:
raise RuntimeError("Fetching status failed")
# TODO: allow user to pass No. of records
result = transmission.results(search_id, 0, 1000)
if result["success"]:
logging.debug("Search {} results is:\n{}".format(
search_id, result["data"]))
# Collect all results
results += result["data"]
else:
raise RuntimeError(
"Fetching results failed; see log for details")
else:
logging.error(str(search_result))
raise Exception(str(
search_result)) # TODO: how to deal with this situation
# Translate results to STIX
data_source = config['data_source']
result = translation.translate(translation_module, 'results',
data_source, json.dumps(results),
{"stix_validator": True})
return result
