def transform_query(self, data, options, mapping=None):
query_object = generate_query(data)
data_model_mapper = data_mapping.DataMapper(options)
query_string = query_constructor.translate_pattern(
query_object, data_model_mapper)
return query_string
def translate(self, module, translate_type, data_source, data, options={}):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:return: translated results
:rtype: str
"""
dialect = None
mod_dia = module.split(':', 1)
module = mod_dia[0]
if len(mod_dia) > 1:
dialect = mod_dia[1]
if module not in TRANSLATION_MODULES:
raise NotImplementedError
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." +
module + "_translator")
if dialect is not None:
interface = translator_module.Translator(dialect=dialect)
else:
interface = translator_module.Translator()
if translate_type == QUERY:
errors = []
# Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format
start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP"
pattern_match = bool(re.search(start_stop_pattern, data))
if (not pattern_match):
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(
errors))
else:
# Translating STIX pattern to antlr query object
query_object = generate_query(data)
# Converting query object to datasource query
parsed_stix = parse_stix(query_object)
# Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls.
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, options)
return {'queries': queries, 'parsed_stix': parsed_stix}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return interface.translate_results(data_source, data, options)
else:
raise NotImplementedError
def translate(self,
module,
translate_type,
data_source,
data,
options={},
recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
dialect = None
mod_dia = module.split(':', 1)
module = mod_dia[0]
if len(mod_dia) > 1:
dialect = mod_dia[1]
try:
if module not in TRANSLATION_MODULES:
raise UnsupportedDataSourceException(
"{} is an unsupported data source.".format(module))
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." +
module + "_translator")
if dialect is not None:
interface = translator_module.Translator(dialect=dialect)
else:
interface = translator_module.Translator()
if translate_type == QUERY:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
print(
"Changing Python recursion limit from {} to {}".format(
current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
if 'result_limit' not in options:
options['result_limit'] = DEFAULT_LIMIT
if 'timerange' not in options:
options['timerange'] = DEFAULT_TIMERANGE
errors = []
# Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format
start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP"
pattern_match = re.search(start_stop_pattern, data)
if (not pattern_match):
errors = run_validator(data)
if (errors != []):
raise StixValidationException(
"The STIX pattern has the following errors: {}".format(
errors))
else:
# Translating STIX pattern to antlr query object
query_object = generate_query(data)
# Converting query object to datasource query
parsed_stix_dictionary = parse_stix(
query_object, options['timerange'])
parsed_stix = parsed_stix_dictionary['parsed_stix']
start_time = parsed_stix_dictionary['start_time']
end_time = parsed_stix_dictionary['end_time']
# Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls.
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, options)
return {
'queries': queries,
'parsed_stix': parsed_stix,
'start_time': start_time,
'end_time': end_time
}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
try:
return interface.translate_results(data_source, data,
options)
except Exception:
raise TranslationResultException()
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
print('caught exception: ' + str(ex) + " " + str(type(ex)))
response = dict()
ErrorResponder.fill_error(response,
message_struct={'exception': ex})
return response
def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
dialect = None
mod_dia = module.split(':', 1)
module = mod_dia[0]
if len(mod_dia) > 1:
dialect = mod_dia[1]
try:
if module not in TRANSLATION_MODULES:
raise UnsupportedDataSourceException("{} is an unsupported data source.".format(module))
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator")
if dialect is not None:
interface = translator_module.Translator(dialect=dialect)
options['dialect'] = dialect
else:
interface = translator_module.Translator()
if translate_type == QUERY or translate_type == PARSE:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
print("Changing Python recursion limit from {} to {}".format(current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
if 'result_limit' not in options:
options['result_limit'] = DEFAULT_LIMIT
if 'timerange' not in options:
options['timerange'] = DEFAULT_TIMERANGE
if translate_type == QUERY:
if 'validate_pattern' in options and options['validate_pattern'] == "true":
self._validate_pattern(data)
try:
data_model = importlib.import_module("stix_shifter.stix_translation.src.modules." + module + ".data_mapping")
data_model_mapper = data_model.DataMapper(options)
except Exception as ex:
print("Data model mapper not found for {} so attempting to use CAR or CIM".format(module))
data_model_mapper = self._cim_or_car_data_mapper(module, options)
antlr_parsing = generate_query(data)
if data_model_mapper:
# Remove unmapped STIX attributes from antlr parsing
antlr_parsing = strip_unmapped_attributes(antlr_parsing, data_model_mapper)
# Converting STIX pattern to datasource query
queries = interface.transform_query(data, antlr_parsing, data_model_mapper, options)
return {'queries': queries}
else:
self._validate_pattern(data)
# Translating STIX pattern to antlr query object
antlr_parsing = generate_query(data)
# Extract pattern elements into parsed stix object
parsed_stix_dictionary = parse_stix(antlr_parsing, options['timerange'])
parsed_stix = parsed_stix_dictionary['parsed_stix']
start_time = parsed_stix_dictionary['start_time']
end_time = parsed_stix_dictionary['end_time']
return {'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
try:
return interface.translate_results(data_source, data, options)
except Exception:
raise TranslationResultException()
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
print('Caught exception: ' + str(ex) + " " + str(type(ex)))
response = dict()
ErrorResponder.fill_error(response, message_struct={'exception': ex})
return response
def translate(self,
module,
translate_type,
data_source,
data,
options={},
recursion_limit=1000):
"""
Translated queries to a specified format
:param module: What module to use
:type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
:param translate_type: translation of a query or result set must be either 'results' or 'query'
:type translate_type: str
:param data: the data to translate
:type data: str
:param options: translation options { stix_validator: bool }
:type options: dict
:param recursion_limit: maximum depth of Python interpreter stack
:type recursion_limit: int
:return: translated results
:rtype: str
"""
module, dialects = self._collect_dialects(module)
try:
if module not in TRANSLATION_MODULES:
raise UnsupportedDataSourceException(
"{} is an unsupported data source.".format(module))
translator_module = importlib.import_module(
"stix_shifter.stix_translation.src.modules." + module + "." +
module + "_translator")
if not dialects[0] == DEFAULT_DIALECT:
# Todo: This will only work if there is one dialect.
# To handle a case such as <MODULE>:<DIALECT_01>:<DIALECT_02> this may need to go in a loop.
interface = translator_module.Translator(dialect=dialects[0])
else:
interface = translator_module.Translator()
if translate_type == QUERY or translate_type == PARSE:
# Increase the python recursion limit to allow ANTLR to parse large patterns
current_recursion_limit = sys.getrecursionlimit()
if current_recursion_limit < recursion_limit:
print(
"Changing Python recursion limit from {} to {}".format(
current_recursion_limit, recursion_limit))
sys.setrecursionlimit(recursion_limit)
options['result_limit'] = options.get('resultSizeLimit',
DEFAULT_LIMIT)
options['timerange'] = options.get('timeRange',
DEFAULT_TIMERANGE)
if translate_type == QUERY:
# Carbon Black combines the mapping files into one JSON using process and binary keys.
# The query constructor has some logic around which of the two are used.
if options.get('validate_pattern'):
self._validate_pattern(data)
queries = []
unmapped_stix_collection = []
for dia in dialects:
options['dialect'] = dia
antlr_parsing = generate_query(data)
data_model_mapper = self._build_data_mapper(
module, options)
if data_model_mapper:
stripped_parsing = strip_unmapped_attributes(
antlr_parsing, data_model_mapper)
antlr_parsing = stripped_parsing.get('parsing')
unmapped_stix = stripped_parsing.get(
'unmapped_stix')
if unmapped_stix:
unmapped_stix_collection.append(unmapped_stix)
if not antlr_parsing:
continue
translated_queries = interface.transform_query(
data, antlr_parsing, data_model_mapper, options)
if isinstance(translated_queries, str):
translated_queries = [translated_queries]
for query in translated_queries:
queries.append(query)
if not queries:
raise DataMappingException("{} {}".format(
MAPPING_ERROR, unmapped_stix_collection))
return {'queries': queries}
else:
self._validate_pattern(data)
antlr_parsing = generate_query(data)
# Extract pattern elements into parsed stix object
parsed_stix_dictionary = parse_stix(
antlr_parsing, options['timerange'])
parsed_stix = parsed_stix_dictionary['parsed_stix']
start_time = parsed_stix_dictionary['start_time']
end_time = parsed_stix_dictionary['end_time']
return {
'parsed_stix': parsed_stix,
'start_time': start_time,
'end_time': end_time
}
elif translate_type == RESULTS:
# Converting data from the datasource to STIX objects
return interface.translate_results(data_source, data, options)
elif translate_type == SUPPORTED_ATTRIBUTES:
# Return mapped STIX attributes supported by the data source
data_model_mapper = self._build_data_mapper(module, options)
mapped_attributes = data_model_mapper.map_data
return {'supported_attributes': mapped_attributes}
else:
raise NotImplementedError('wrong parameter: ' + translate_type)
except Exception as ex:
print('Caught exception: ' + str(ex) + " " + str(type(ex)))
response = dict()
ErrorResponder.fill_error(response,
message_struct={'exception': ex})
return response