def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :param recursion_limit: maximum depth of Python interpreter stack :type recursion_limit: int :return: translated results :rtype: str """ dialect = None mod_dia = module.split(':', 1) module = mod_dia[0] if len(mod_dia) > 1: dialect = mod_dia[1] try: if module not in TRANSLATION_MODULES: raise UnsupportedDataSourceException("{} is an unsupported data source.".format(module)) translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if dialect is not None: interface = translator_module.Translator(dialect=dialect) options['dialect'] = dialect else: interface = translator_module.Translator() if translate_type == QUERY or translate_type == PARSE: # Increase the python recursion limit to allow ANTLR to parse large patterns current_recursion_limit = sys.getrecursionlimit() if current_recursion_limit < recursion_limit: print("Changing Python recursion limit from {} to {}".format(current_recursion_limit, recursion_limit)) sys.setrecursionlimit(recursion_limit) if 'result_limit' not in options: options['result_limit'] = DEFAULT_LIMIT if 'timerange' not in options: options['timerange'] = DEFAULT_TIMERANGE if translate_type == QUERY: if 'validate_pattern' in options and options['validate_pattern'] == "true": self._validate_pattern(data) try: data_model = importlib.import_module("stix_shifter.stix_translation.src.modules." + module + ".data_mapping") data_model_mapper = data_model.DataMapper(options) except Exception as ex: print("Data model mapper not found for {} so attempting to use CAR or CIM".format(module)) data_model_mapper = self._cim_or_car_data_mapper(module, options) antlr_parsing = generate_query(data) if data_model_mapper: # Remove unmapped STIX attributes from antlr parsing antlr_parsing = strip_unmapped_attributes(antlr_parsing, data_model_mapper) # Converting STIX pattern to datasource query queries = interface.transform_query(data, antlr_parsing, data_model_mapper, options) return {'queries': queries} else: self._validate_pattern(data) # Translating STIX pattern to antlr query object antlr_parsing = generate_query(data) # Extract pattern elements into parsed stix object parsed_stix_dictionary = parse_stix(antlr_parsing, options['timerange']) parsed_stix = parsed_stix_dictionary['parsed_stix'] start_time = parsed_stix_dictionary['start_time'] end_time = parsed_stix_dictionary['end_time'] return {'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time} elif translate_type == RESULTS: # Converting data from the datasource to STIX objects try: return interface.translate_results(data_source, data, options) except Exception: raise TranslationResultException() else: raise NotImplementedError('wrong parameter: ' + translate_type) except Exception as ex: print('Caught exception: ' + str(ex) + " " + str(type(ex))) response = dict() ErrorResponder.fill_error(response, message_struct={'exception': ex}) return response
def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000): """ Translated queries to a specified format :param module: What module to use :type module: one of TRANSLATION_MODULES 'qradar', 'dummy' :param translate_type: translation of a query or result set must be either 'results' or 'query' :type translate_type: str :param data: the data to translate :type data: str :param options: translation options { stix_validator: bool } :type options: dict :param recursion_limit: maximum depth of Python interpreter stack :type recursion_limit: int :return: translated results :rtype: str """ module, dialects = self._collect_dialects(module) try: if module not in TRANSLATION_MODULES: raise UnsupportedDataSourceException( "{} is an unsupported data source.".format(module)) translator_module = importlib.import_module( "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator") if not dialects[0] == DEFAULT_DIALECT: # Todo: This will only work if there is one dialect. # To handle a case such as <MODULE>:<DIALECT_01>:<DIALECT_02> this may need to go in a loop. interface = translator_module.Translator(dialect=dialects[0]) else: interface = translator_module.Translator() if translate_type == QUERY or translate_type == PARSE: # Increase the python recursion limit to allow ANTLR to parse large patterns current_recursion_limit = sys.getrecursionlimit() if current_recursion_limit < recursion_limit: print( "Changing Python recursion limit from {} to {}".format( current_recursion_limit, recursion_limit)) sys.setrecursionlimit(recursion_limit) options['result_limit'] = options.get('resultSizeLimit', DEFAULT_LIMIT) options['timerange'] = options.get('timeRange', DEFAULT_TIMERANGE) if translate_type == QUERY: # Carbon Black combines the mapping files into one JSON using process and binary keys. # The query constructor has some logic around which of the two are used. if options.get('validate_pattern'): self._validate_pattern(data) queries = [] unmapped_stix_collection = [] for dia in dialects: options['dialect'] = dia antlr_parsing = generate_query(data) data_model_mapper = self._build_data_mapper( module, options) if data_model_mapper: stripped_parsing = strip_unmapped_attributes( antlr_parsing, data_model_mapper) antlr_parsing = stripped_parsing.get('parsing') unmapped_stix = stripped_parsing.get( 'unmapped_stix') if unmapped_stix: unmapped_stix_collection.append(unmapped_stix) if not antlr_parsing: continue translated_queries = interface.transform_query( data, antlr_parsing, data_model_mapper, options) if isinstance(translated_queries, str): translated_queries = [translated_queries] for query in translated_queries: queries.append(query) if not queries: raise DataMappingException("{} {}".format( MAPPING_ERROR, unmapped_stix_collection)) return {'queries': queries} else: self._validate_pattern(data) antlr_parsing = generate_query(data) # Extract pattern elements into parsed stix object parsed_stix_dictionary = parse_stix( antlr_parsing, options['timerange']) parsed_stix = parsed_stix_dictionary['parsed_stix'] start_time = parsed_stix_dictionary['start_time'] end_time = parsed_stix_dictionary['end_time'] return { 'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time } elif translate_type == RESULTS: # Converting data from the datasource to STIX objects return interface.translate_results(data_source, data, options) elif translate_type == SUPPORTED_ATTRIBUTES: # Return mapped STIX attributes supported by the data source data_model_mapper = self._build_data_mapper(module, options) mapped_attributes = data_model_mapper.map_data return {'supported_attributes': mapped_attributes} else: raise NotImplementedError('wrong parameter: ' + translate_type) except Exception as ex: print('Caught exception: ' + str(ex) + " " + str(type(ex))) response = dict() ErrorResponder.fill_error(response, message_struct={'exception': ex}) return response