def transform_query(self, data, options, mapping=None): query_object = generate_query(data) data_model_mapper = data_mapping.DataMapper(options) query_string = query_constructor.translate_pattern( query_object, data_model_mapper) return query_string
def translate(self, module, translate_type, data_source, data, options={}): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :return: translated results :rtype: str """ dialect = None mod_dia = module.split(':', 1) module = mod_dia[0] if len(mod_dia) > 1: dialect = mod_dia[1] if module not in TRANSLATION_MODULES: raise NotImplementedError translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if dialect is not None: interface = translator_module.Translator(dialect=dialect) else: interface = translator_module.Translator() if translate_type == QUERY: errors = [] # Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP" pattern_match = bool(re.search(start_stop_pattern, data)) if (not pattern_match): errors = run_validator(data) if (errors != []): raise StixValidationException( "The STIX pattern has the following errors: {}".format( errors)) else: # Translating STIX pattern to antlr query object query_object = generate_query(data) # Converting query object to datasource query parsed_stix = parse_stix(query_object) # Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls. # Converting STIX pattern to datasource query queries = interface.transform_query(data, options) return {'queries': queries, 'parsed_stix': parsed_stix} elif translate_type == RESULTS: # Converting data from the datasource to STIX objects return interface.translate_results(data_source, data, options) else: raise NotImplementedError
def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :param recursion_limit: maximum depth of Python interpreter stack :type recursion_limit: int :return: translated results :rtype: str """ dialect = None mod_dia = module.split(':', 1) module = mod_dia[0] if len(mod_dia) > 1: dialect = mod_dia[1] try: if module not in TRANSLATION_MODULES: raise UnsupportedDataSourceException( "{} is an unsupported data source.".format(module)) translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if dialect is not None: interface = translator_module.Translator(dialect=dialect) else: interface = translator_module.Translator() if translate_type == QUERY: # Increase the python recursion limit to allow ANTLR to parse large patterns current_recursion_limit = sys.getrecursionlimit() if current_recursion_limit < recursion_limit: print( "Changing Python recursion limit from {} to {}".format( current_recursion_limit, recursion_limit)) sys.setrecursionlimit(recursion_limit) if 'result_limit' not in options: options['result_limit'] = DEFAULT_LIMIT if 'timerange' not in options: options['timerange'] = DEFAULT_TIMERANGE errors = [] # Temporarily skip validation on patterns with START STOP qualifiers: validator doesn't yet support timestamp format start_stop_pattern = "START\s?t'\d{4}(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z'\sSTOP" pattern_match = re.search(start_stop_pattern, data) if (not pattern_match): errors = run_validator(data) if (errors != []): raise StixValidationException( "The STIX pattern has the following errors: {}".format( errors)) else: # Translating STIX pattern to antlr query object query_object = generate_query(data) # Converting query object to datasource query parsed_stix_dictionary = parse_stix( query_object, options['timerange']) parsed_stix = parsed_stix_dictionary['parsed_stix'] start_time = parsed_stix_dictionary['start_time'] end_time = parsed_stix_dictionary['end_time'] # Todo: pass in the query_object instead of the data so we can remove multiple generate_query calls. # Converting STIX pattern to datasource query queries = interface.transform_query(data, options) return { 'queries': queries, 'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time } elif translate_type == RESULTS: # Converting data from the datasource to STIX objects try: return interface.translate_results(data_source, data, options) except Exception: raise TranslationResultException() else: raise NotImplementedError('wrong parameter: ' + translate_type) except Exception as ex: print('caught exception: ' + str(ex) + " " + str(type(ex))) response = dict() ErrorResponder.fill_error(response, message_struct={'exception': ex}) return response
def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :param recursion_limit: maximum depth of Python interpreter stack :type recursion_limit: int :return: translated results :rtype: str """ dialect = None mod_dia = module.split(':', 1) module = mod_dia[0] if len(mod_dia) > 1: dialect = mod_dia[1] try: if module not in TRANSLATION_MODULES: raise UnsupportedDataSourceException("{} is an unsupported data source.".format(module)) translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if dialect is not None: interface = translator_module.Translator(dialect=dialect) options['dialect'] = dialect else: interface = translator_module.Translator() if translate_type == QUERY or translate_type == PARSE: # Increase the python recursion limit to allow ANTLR to parse large patterns current_recursion_limit = sys.getrecursionlimit() if current_recursion_limit < recursion_limit: print("Changing Python recursion limit from {} to {}".format(current_recursion_limit, recursion_limit)) sys.setrecursionlimit(recursion_limit) if 'result_limit' not in options: options['result_limit'] = DEFAULT_LIMIT if 'timerange' not in options: options['timerange'] = DEFAULT_TIMERANGE if translate_type == QUERY: if 'validate_pattern' in options and options['validate_pattern'] == "true": self._validate_pattern(data) try: data_model = importlib.import_module("stix_shifter.stix_translation.src.modules." + module + ".data_mapping") data_model_mapper = data_model.DataMapper(options) except Exception as ex: print("Data model mapper not found for {} so attempting to use CAR or CIM".format(module)) data_model_mapper = self._cim_or_car_data_mapper(module, options) antlr_parsing = generate_query(data) if data_model_mapper: # Remove unmapped STIX attributes from antlr parsing antlr_parsing = strip_unmapped_attributes(antlr_parsing, data_model_mapper) # Converting STIX pattern to datasource query queries = interface.transform_query(data, antlr_parsing, data_model_mapper, options) return {'queries': queries} else: self._validate_pattern(data) # Translating STIX pattern to antlr query object antlr_parsing = generate_query(data) # Extract pattern elements into parsed stix object parsed_stix_dictionary = parse_stix(antlr_parsing, options['timerange']) parsed_stix = parsed_stix_dictionary['parsed_stix'] start_time = parsed_stix_dictionary['start_time'] end_time = parsed_stix_dictionary['end_time'] return {'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time} elif translate_type == RESULTS: # Converting data from the datasource to STIX objects try: return interface.translate_results(data_source, data, options) except Exception: raise TranslationResultException() else: raise NotImplementedError('wrong parameter: ' + translate_type) except Exception as ex: print('Caught exception: ' + str(ex) + " " + str(type(ex))) response = dict() ErrorResponder.fill_error(response, message_struct={'exception': ex}) return response
def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :param recursion_limit: maximum depth of Python interpreter stack :type recursion_limit: int :return: translated results :rtype: str """ module, dialects = self._collect_dialects(module) try: if module not in TRANSLATION_MODULES: raise UnsupportedDataSourceException( "{} is an unsupported data source.".format(module)) translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if not dialects[0] == DEFAULT_DIALECT: # Todo: This will only work if there is one dialect. # To handle a case such as <MODULE>:<DIALECT_01>:<DIALECT_02> this may need to go in a loop. interface = translator_module.Translator(dialect=dialects[0]) else: interface = translator_module.Translator() if translate_type == QUERY or translate_type == PARSE: # Increase the python recursion limit to allow ANTLR to parse large patterns current_recursion_limit = sys.getrecursionlimit() if current_recursion_limit < recursion_limit: print( "Changing Python recursion limit from {} to {}".format( current_recursion_limit, recursion_limit)) sys.setrecursionlimit(recursion_limit) options['result_limit'] = options.get('resultSizeLimit', DEFAULT_LIMIT) options['timerange'] = options.get('timeRange', DEFAULT_TIMERANGE) if translate_type == QUERY: # Carbon Black combines the mapping files into one JSON using process and binary keys. # The query constructor has some logic around which of the two are used. if options.get('validate_pattern'): self._validate_pattern(data) queries = [] unmapped_stix_collection = [] for dia in dialects: options['dialect'] = dia antlr_parsing = generate_query(data) data_model_mapper = self._build_data_mapper( module, options) if data_model_mapper: stripped_parsing = strip_unmapped_attributes( antlr_parsing, data_model_mapper) antlr_parsing = stripped_parsing.get('parsing') unmapped_stix = stripped_parsing.get( 'unmapped_stix') if unmapped_stix: unmapped_stix_collection.append(unmapped_stix) if not antlr_parsing: continue translated_queries = interface.transform_query( data, antlr_parsing, data_model_mapper, options) if isinstance(translated_queries, str): translated_queries = [translated_queries] for query in translated_queries: queries.append(query) if not queries: raise DataMappingException("{} {}".format( MAPPING_ERROR, unmapped_stix_collection)) return {'queries': queries} else: self._validate_pattern(data) antlr_parsing = generate_query(data) # Extract pattern elements into parsed stix object parsed_stix_dictionary = parse_stix( antlr_parsing, options['timerange']) parsed_stix = parsed_stix_dictionary['parsed_stix'] start_time = parsed_stix_dictionary['start_time'] end_time = parsed_stix_dictionary['end_time'] return { 'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time } elif translate_type == RESULTS: # Converting data from the datasource to STIX objects return interface.translate_results(data_source, data, options) elif translate_type == SUPPORTED_ATTRIBUTES: # Return mapped STIX attributes supported by the data source data_model_mapper = self._build_data_mapper(module, options) mapped_attributes = data_model_mapper.map_data return {'supported_attributes': mapped_attributes} else: raise NotImplementedError('wrong parameter: ' + translate_type) except Exception as ex: print('Caught exception: ' + str(ex) + " " + str(type(ex))) response = dict() ErrorResponder.fill_error(response, message_struct={'exception': ex}) return response