def translate(self,
module,
translate_type,
data_source,
data,
options={},
recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of connector modules: 'qradar', 'template'
:param translate_type: translation of a query or result set must be one of: 'parse', 'mapping' 'query', 'results'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
module, dialects = process_dialects(module, options)
try:
try:
connector_module = importlib.import_module(
"stix_shifter_modules." + module + ".entry_point")
except Exception as ex:
raise UnsupportedDataSourceException(
"{} is an unsupported data source.".format(module))
try:
if not translate_type == DIALECTS:
validated_options = param_validator(
module, options, 'connection.options')
else:
validated_options = {}
entry_point = connector_module.EntryPoint(
options=validated_options)
except Exception as ex:
track = traceback.format_exc()
self.logger.error(ex)
self.logger.debug(track)
raise
if translate_type == DIALECTS:
dialects = entry_point.get_dialects_full()
return dialects
if len(dialects) == 0:
dialects = entry_point.get_dialects()
language = validated_options['language']
else:
language = options.get('language')
if translate_type == QUERY or translate_type == PARSE:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
self.logger.debug(
"Changing Python recursion limit from {} to {}".format(
current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
if translate_type == QUERY:
# Carbon Black combines the mapping files into one JSON using process and binary keys.
# The query constructor has some logic around which of the two are used.
queries = []
unmapped_stix_collection = []
unmapped_operator_collection = []
dialects_used = 0
for dialect in dialects:
query_translator = entry_point.get_query_translator(
dialect)
if not language or language == query_translator.get_language(
):
dialects_used += 1
transform_result = entry_point.transform_query(
dialect, data)
if 'async_call' in transform_result:
queries.append(transform_result)
else:
queries.extend(
transform_result.get('queries', []))
unmapped_stix_collection.extend(
transform_result.get('unmapped_attributes',
[]))
unmapped_operator_collection.extend(
transform_result.get('unmapped_operator', []))
if not dialects_used:
raise UnsupportedLanguageException(language)
unmapped_stix_collection = list(
set(unmapped_stix_collection))
unmapped_operator_collection = list(
set(unmapped_operator_collection))
if not queries:
if unmapped_stix_collection and unmapped_operator_collection:
raise DataMappingException(
"{} {} and Operators: {} to data source fields"
.format(MAPPING_ERROR,
unmapped_stix_collection,
unmapped_operator_collection))
elif unmapped_stix_collection:
raise DataMappingException(
"{} {} to data source fields".format(
ATTRIBUTE_MAPPING_ERROR,
unmapped_stix_collection))
elif unmapped_operator_collection:
raise DataMappingException(
"{} {} to data source fields".format(
OPERATOR_MAPPING_ERROR,
unmapped_operator_collection))
return {'queries': queries}
else:
return entry_point.parse_query(data)
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return entry_point.translate_results(data_source, data)
elif translate_type == MAPPING:
mappings = entry_point.get_mapping()
return mappings
elif translate_type == SUPPORTED_ATTRIBUTES:
# Return mapped STIX attributes supported by the data source
result = {}
for dialect in dialects:
query_translator = entry_point.get_query_translator(
dialect)
result[dialect] = query_translator.map_data
return {'supported_attributes': result}
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
self.logger.error('Caught exception: ' + str(ex) + " " +
str(type(ex)))
self.logger.debug(exception_to_string(ex))
response = dict()
ErrorResponder.fill_error(response,
message_struct={'exception': ex},
connector=module)
return response
def translate(self,
module,
translate_type,
data_source,
data,
options={},
recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of connector modules: 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be one of: 'parse', 'mapping' 'query', 'results'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
module, dialects = process_dialects(module, options)
try:
try:
connector_module = importlib.import_module(
"stix_shifter_modules." + module + ".entry_point")
except Exception as ex:
raise UnsupportedDataSourceException(
"{} is an unsupported data source.".format(module))
try:
if options:
validated_options = param_validator(
module, options, 'connection.options')
entry_point = connector_module.EntryPoint(
options=validated_options)
except Exception as ex:
track = traceback.format_exc()
self.logger.error(ex)
self.logger.error(track)
raise
if len(dialects) == 0:
dialects = entry_point.get_dialects()
if translate_type == QUERY or translate_type == PARSE:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
self.logger.debug(
"Changing Python recursion limit from {} to {}".format(
current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
if translate_type == QUERY:
# Carbon Black combines the mapping files into one JSON using process and binary keys.
# The query constructor has some logic around which of the two are used.
if validated_options.get('validate_pattern'):
self._validate_pattern(data)
queries = []
unmapped_stix_collection = []
for dialect in dialects:
antlr_parsing = generate_query(data)
query_translator = entry_point.get_query_translator(
dialect)
if query_translator and not isinstance(
query_translator, EmptyQueryTranslator):
stripped_parsing = strip_unmapped_attributes(
antlr_parsing, query_translator)
antlr_parsing = stripped_parsing.get('parsing')
unmapped_stix = stripped_parsing.get(
'unmapped_stix')
if unmapped_stix:
unmapped_stix_collection.append(unmapped_stix)
if not antlr_parsing:
continue
translated_queries = entry_point.transform_query(
dialect, data, antlr_parsing)
if isinstance(translated_queries, str):
translated_queries = [translated_queries]
for query in translated_queries:
queries.append(query)
if not queries:
raise DataMappingException("{} {}".format(
MAPPING_ERROR, unmapped_stix_collection))
return {'queries': queries}
else:
self._validate_pattern(data)
antlr_parsing = generate_query(data)
# Extract pattern elements into parsed stix object
parsed_stix_dictionary = parse_stix(
antlr_parsing, validated_options['time_range'])
parsed_stix = parsed_stix_dictionary['parsed_stix']
start_time = parsed_stix_dictionary['start_time']
end_time = parsed_stix_dictionary['end_time']
return {
'parsed_stix': parsed_stix,
'start_time': start_time,
'end_time': end_time
}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return entry_point.translate_results(data_source, data)
elif translate_type == MAPPING:
mappings = {}
for dialect in dialects:
mapping = entry_point.get_mapping(dialect)
mappings[dialect] = mapping
return mappings
elif translate_type == SUPPORTED_ATTRIBUTES:
# Return mapped STIX attributes supported by the data source
result = {}
for dialect in dialects:
query_translator = entry_point.get_query_translator(
dialect)
result[dialect] = query_translator.map_data
return {'supported_attributes': result}
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
self.logger.error('Caught exception: ' + str(ex) + " " +
str(type(ex)))
response = dict()
ErrorResponder.fill_error(response,
message_struct={'exception': ex})
return response
def main():
"""
Stix-shifter can either be called to either translate or transmit.
In the case of translation, stix-shifter either translates a stix pattern to a datasource query,
or converts data source query results into JSON of STIX observations.
Arguments will take the form of...
"translate" <module> <translate_type (query or results)> <data (STIX pattern or query results)> <options>
The module and translate_type will determine what module and method gets executed.
Option arguments comes in as:
"{
"mapping": <mapping hash for stix pattern to datasource and data results to stix observation objects>,
"resultSizeLimit": <integer limit number for max results in the data source query>,
"timeRange": <integer time range for LAST x MINUTES used in the data source query when START STOP qualifiers are absent>
}"
In the case of transmission, stix-shifter connects to a datasource to execute queries, status updates, and result retrieval.
Arguments will take the form of...
"transmit" <module> '{"host": <host IP>, "port": <port>, "cert": <certificate>}', '{"auth": <authentication>}',
<
query <query string>,
status <search id>,
results <search id> <offset> <length>,
ping,
is_async
>
"""
# process arguments
parent_parser = argparse.ArgumentParser(description='stix_shifter')
parent_subparsers = parent_parser.add_subparsers(dest='command')
# translate parser
translate_parser = parent_subparsers.add_parser(
TRANSLATE,
help=
'Translate a query or result set using a specific translation module')
# positional arguments
translate_parser.add_argument('module',
help='The translation module to use')
translate_parser.add_argument('translate_type',
choices=[
stix_translation.RESULTS,
stix_translation.QUERY,
stix_translation.PARSE
],
help='The translation action to perform')
translate_parser.add_argument(
'data_source', help='STIX identity object representing a datasource')
translate_parser.add_argument(
'data',
type=str,
help='The STIX pattern or JSON results to be translated')
translate_parser.add_argument('options',
nargs='?',
help='Options dictionary')
translate_parser.add_argument(
'recursion_limit',
type=int,
nargs='?',
help='Maximum depth of Python interpreter stack')
# optional arguments
translate_parser.add_argument(
'-x',
'--stix-validator',
action='store_true',
help='Run the STIX 2 validator against the translated results')
translate_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
# modules parser
parent_subparsers.add_parser(MODULES, help='Get modules list')
# mapping parser
mapping_parser = parent_subparsers.add_parser(MAPPING,
help='Get module mapping')
# positional arguments
mapping_parser.add_argument('module', help='The translation module to use')
# transmit parser
transmit_parser = parent_subparsers.add_parser(
TRANSMIT, help='Connect to a datasource and exectue a query...')
# positional arguments
transmit_parser.add_argument('module',
help='Choose which connection module to use')
transmit_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
transmit_parser.add_argument('configuration',
type=str,
help='Data source authentication')
transmit_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
# operation subparser
operation_subparser = transmit_parser.add_subparsers(
title="operation", dest="operation_command")
operation_subparser.add_parser(stix_transmission.PING,
help="Pings the data source")
query_operation_parser = operation_subparser.add_parser(
stix_transmission.QUERY, help="Executes a query on the data source")
query_operation_parser.add_argument('query_string',
help='native datasource query string')
query_operation_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
results_operation_parser = operation_subparser.add_parser(
stix_transmission.RESULTS,
help="Fetches the results of the data source query")
results_operation_parser.add_argument('search_id',
help='uuid of executed query')
results_operation_parser.add_argument('offset', help='offset of results')
results_operation_parser.add_argument('length', help='length of results')
results_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
status_operation_parser = operation_subparser.add_parser(
stix_transmission.STATUS, help="Gets the current status of the query")
status_operation_parser.add_argument('search_id',
help='uuid of executed query')
status_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
delete_operation_parser = operation_subparser.add_parser(
stix_transmission.DELETE,
help="Delete a running query on the data source")
delete_operation_parser.add_argument('search_id',
help='id of query to remove')
delete_operation_parser.add_argument(
'-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
operation_subparser.add_parser(
stix_transmission.IS_ASYNC,
help='Checks if the query operation is asynchronous')
execute_parser = parent_subparsers.add_parser(
EXECUTE, help='Translate and fully execute a query')
# positional arguments
execute_parser.add_argument('transmission_module',
help='Which connection module to use')
execute_parser.add_argument(
'module', help='Which translation module to use for translation')
execute_parser.add_argument(
'data_source',
type=str,
help='STIX Identity object for the data source')
execute_parser.add_argument(
'connection',
type=str,
help='Data source connection with host, port, and certificate')
execute_parser.add_argument('configuration',
type=str,
help='Data source authentication')
execute_parser.add_argument('query', type=str, help='Query String')
execute_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
host_parser = parent_subparsers.add_parser(
HOST, help='Host a local query service, for testing and development')
host_parser.add_argument('data_source',
type=str,
help='STIX Identity object for the data source')
host_parser.add_argument('host_address', type=str, help='Proxy Host:Port')
host_parser.add_argument('-d',
'--debug',
action='store_true',
help='Print detail logs for debugging')
args = parent_parser.parse_args()
help_and_exit = args.command is None
if 'debug' in args and args.debug:
utils_logger.init(logging.DEBUG)
else:
utils_logger.init(logging.INFO)
log = utils_logger.set_logger(__name__)
if 'module' in args:
args_module_dialects = args.module
options = {}
if 'options' in args and args.options:
options = json.loads(args.options)
module = process_dialects(args_module_dialects, options)[0]
try:
importlib.import_module("stix_shifter_modules." + module +
".entry_point")
except Exception as ex:
log.debug(exception_to_string(ex))
log.error('Module {} not found'.format(module))
log.debug(exception_to_string(ex))
help_and_exit = True
if help_and_exit:
parent_parser.print_help(sys.stderr)
sys.exit(1)
elif args.command == HOST:
# Host means to start a local web service for STIX shifter, to use in combination with the proxy data source
# module. This combination allows one to run and debug their stix-shifter code locally, while interacting with
# it inside a service provider such as IBM Security Connect
app = Flask("stix-shifter")
@app.route('/transform_query', methods=['POST'])
def transform_query():
host = ProxyHost()
return host.transform_query()
@app.route('/translate_results', methods=['POST'])
def translate_results():
data_source_identity_object = args.data_source
host = ProxyHost()
return host.translate_results(data_source_identity_object)
@app.route('/create_query_connection', methods=['POST'])
def create_query_connection():
host = ProxyHost()
return host.create_query_connection()
@app.route('/create_status_connection', methods=['POST'])
def create_status_connection():
host = ProxyHost()
return host.create_status_connection()
@app.route('/create_results_connection', methods=['POST'])
def create_results_connection():
host = ProxyHost()
return host.create_results_connection()
@app.route('/delete_query_connection', methods=['POST'])
def delete_query_connection():
host = ProxyHost()
return host.delete_query_connection()
@app.route('/ping', methods=['POST'])
def ping_connection():
host = ProxyHost()
return host.ping_connection()
@app.route('/is_async', methods=['POST'])
def is_async():
host = ProxyHost()
return host.is_async()
host_address = args.host_address.split(":")
app.run(debug=False, port=int(host_address[1]), host=host_address[0])
elif args.command == EXECUTE:
# Execute means take the STIX SCO pattern as input, execute query, and return STIX as output
translation = stix_translation.StixTranslation()
connection_dict = json.loads(args.connection)
configuration_dict = json.loads(args.configuration)
translation_options = copy.deepcopy(connection_dict.get('options', {}))
options['validate_pattern'] = True
dsl = translation.translate(args.module, 'query', args.data_source,
args.query, translation_options)
transmission = stix_transmission.StixTransmission(
args.transmission_module, connection_dict, configuration_dict)
results = []
log.info('Translated Queries: \n' + json.dumps(dsl, indent=4))
if 'queries' not in dsl:
exit(1)
for query in dsl['queries']:
search_result = transmission.query(query)
if search_result["success"]:
search_id = search_result["search_id"]
if transmission.is_async():
time.sleep(1)
status = transmission.status(search_id)
if status['success']:
while status['progress'] < 100 and status[
'status'] == 'RUNNING':
log.debug(status)
status = transmission.status(search_id)
log.debug(status)
else:
raise RuntimeError("Fetching status failed")
result = transmission.results(search_id, 0, 9)
if result["success"]:
log.debug("Search {} results is:\n{}".format(
search_id, result["data"]))
# Collect all results
results += result["data"]
else:
raise RuntimeError(
"Fetching results failed; see log for details")
else:
log.error(str(search_result))
exit(0)
# Translate results to STIX
translation_options = copy.deepcopy(connection_dict.get('options', {}))
options['validate_pattern'] = True
result = translation.translate(args.module, 'results',
args.data_source, json.dumps(results),
translation_options)
log.info('STIX Results: \n' +
json.dumps(result, indent=4, sort_keys=False))
exit(0)
elif args.command == TRANSLATE:
data = args.data
if not data:
data_lines = []
for line in sys.stdin:
data_lines.append(line)
data = '\n'.join(data_lines)
if args.stix_validator:
options['stix_validator'] = args.stix_validator
recursion_limit = args.recursion_limit if args.recursion_limit else 1000
translation = stix_translation.StixTranslation()
result = translation.translate(args.module,
args.translate_type,
args.data_source,
data,
options=options,
recursion_limit=recursion_limit)
elif args.command == MAPPING:
translation = stix_translation.StixTranslation()
result = translation.translate(args.module,
stix_translation.MAPPING,
None,
None,
options=options)
elif args.command == MODULES:
translation = stix_translation.StixTranslation()
result = {}
all_modules = modules_list()
for m in all_modules:
result[m] = translation.translate(m, stix_translation.DIALECTS,
None, None)
elif args.command == TRANSMIT:
result = transmit(args) # stix_transmission
print(json.dumps(result, indent=4, sort_keys=False))
exit(0)