Esempio n. 1
0
    def translate(self, module, translate_type, data_source, data, options={}, recursion_limit=1000):
        """
        Translated queries to a specified format
        :param module: What module to use
        :type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
        :param translate_type: translation of a query or result set must be either 'results' or 'query'
        :type translate_type: str
        :param data: the data to translate
        :type data: str
        :param options: translation options { stix_validator: bool }
        :type options: dict
        :param recursion_limit: maximum depth of Python interpreter stack
        :type recursion_limit: int
        :return: translated results
        :rtype: str
        """
        dialect = None
        mod_dia = module.split(':', 1)
        module = mod_dia[0]
        if len(mod_dia) > 1:
            dialect = mod_dia[1]

        try:
            if module not in TRANSLATION_MODULES:
                raise UnsupportedDataSourceException("{} is an unsupported data source.".format(module))

            translator_module = importlib.import_module(
                "stix_shifter.stix_translation.src.modules." + module + "." + module + "_translator")

            if dialect is not None:
                interface = translator_module.Translator(dialect=dialect)
                options['dialect'] = dialect
            else:
                interface = translator_module.Translator()

            if translate_type == QUERY or translate_type == PARSE:
                # Increase the python recursion limit to allow ANTLR to parse large patterns
                current_recursion_limit = sys.getrecursionlimit()
                if current_recursion_limit < recursion_limit:
                    print("Changing Python recursion limit from {} to {}".format(current_recursion_limit, recursion_limit))
                    sys.setrecursionlimit(recursion_limit)
                if 'result_limit' not in options:
                    options['result_limit'] = DEFAULT_LIMIT
                if 'timerange' not in options:
                    options['timerange'] = DEFAULT_TIMERANGE

                if translate_type == QUERY:
                    if 'validate_pattern' in options and options['validate_pattern'] == "true":
                        self._validate_pattern(data)
                    try:
                        data_model = importlib.import_module("stix_shifter.stix_translation.src.modules." + module + ".data_mapping")
                        data_model_mapper = data_model.DataMapper(options)
                    except Exception as ex:
                        print("Data model mapper not found for {} so attempting to use CAR or CIM".format(module))
                        data_model_mapper = self._cim_or_car_data_mapper(module, options)
                    antlr_parsing = generate_query(data)
                    if data_model_mapper:
                        # Remove unmapped STIX attributes from antlr parsing
                        antlr_parsing = strip_unmapped_attributes(antlr_parsing, data_model_mapper)
                    # Converting STIX pattern to datasource query
                    queries = interface.transform_query(data, antlr_parsing, data_model_mapper, options)
                    return {'queries': queries}
                else:
                    self._validate_pattern(data)
                    # Translating STIX pattern to antlr query object
                    antlr_parsing = generate_query(data)
                    # Extract pattern elements into parsed stix object
                    parsed_stix_dictionary = parse_stix(antlr_parsing, options['timerange'])
                    parsed_stix = parsed_stix_dictionary['parsed_stix']
                    start_time = parsed_stix_dictionary['start_time']
                    end_time = parsed_stix_dictionary['end_time']
                    return {'parsed_stix': parsed_stix, 'start_time': start_time, 'end_time': end_time}

            elif translate_type == RESULTS:
                # Converting data from the datasource to STIX objects
                try:
                    return interface.translate_results(data_source, data, options)
                except Exception:
                    raise TranslationResultException()
            else:
                raise NotImplementedError('wrong parameter: ' + translate_type)
        except Exception as ex:
            print('Caught exception: ' + str(ex) + " " + str(type(ex)))
            response = dict()
            ErrorResponder.fill_error(response, message_struct={'exception': ex})
            return response
    def translate(self,
                  module,
                  translate_type,
                  data_source,
                  data,
                  options={},
                  recursion_limit=1000):
        """
        Translated queries to a specified format
        :param module: What module to use
        :type module: one of TRANSLATION_MODULES 'qradar', 'dummy'
        :param translate_type: translation of a query or result set must be either 'results' or 'query'
        :type translate_type: str
        :param data: the data to translate
        :type data: str
        :param options: translation options { stix_validator: bool }
        :type options: dict
        :param recursion_limit: maximum depth of Python interpreter stack
        :type recursion_limit: int
        :return: translated results
        :rtype: str
        """

        module, dialects = self._collect_dialects(module)

        try:
            if module not in TRANSLATION_MODULES:
                raise UnsupportedDataSourceException(
                    "{} is an unsupported data source.".format(module))

            translator_module = importlib.import_module(
                "stix_shifter.stix_translation.src.modules." + module + "." +
                module + "_translator")

            if not dialects[0] == DEFAULT_DIALECT:
                # Todo: This will only work if there is one dialect.
                # To handle a case such as <MODULE>:<DIALECT_01>:<DIALECT_02> this may need to go in a loop.
                interface = translator_module.Translator(dialect=dialects[0])
            else:
                interface = translator_module.Translator()

            if translate_type == QUERY or translate_type == PARSE:
                # Increase the python recursion limit to allow ANTLR to parse large patterns
                current_recursion_limit = sys.getrecursionlimit()
                if current_recursion_limit < recursion_limit:
                    print(
                        "Changing Python recursion limit from {} to {}".format(
                            current_recursion_limit, recursion_limit))
                    sys.setrecursionlimit(recursion_limit)
                options['result_limit'] = options.get('resultSizeLimit',
                                                      DEFAULT_LIMIT)
                options['timerange'] = options.get('timeRange',
                                                   DEFAULT_TIMERANGE)

                if translate_type == QUERY:
                    # Carbon Black combines the mapping files into one JSON using process and binary keys.
                    # The query constructor has some logic around which of the two are used.
                    if options.get('validate_pattern'):
                        self._validate_pattern(data)
                    queries = []
                    unmapped_stix_collection = []
                    for dia in dialects:
                        options['dialect'] = dia
                        antlr_parsing = generate_query(data)
                        data_model_mapper = self._build_data_mapper(
                            module, options)
                        if data_model_mapper:
                            stripped_parsing = strip_unmapped_attributes(
                                antlr_parsing, data_model_mapper)
                            antlr_parsing = stripped_parsing.get('parsing')
                            unmapped_stix = stripped_parsing.get(
                                'unmapped_stix')
                            if unmapped_stix:
                                unmapped_stix_collection.append(unmapped_stix)
                            if not antlr_parsing:
                                continue
                        translated_queries = interface.transform_query(
                            data, antlr_parsing, data_model_mapper, options)

                        if isinstance(translated_queries, str):
                            translated_queries = [translated_queries]
                        for query in translated_queries:
                            queries.append(query)

                    if not queries:
                        raise DataMappingException("{} {}".format(
                            MAPPING_ERROR, unmapped_stix_collection))

                    return {'queries': queries}
                else:
                    self._validate_pattern(data)
                    antlr_parsing = generate_query(data)
                    # Extract pattern elements into parsed stix object
                    parsed_stix_dictionary = parse_stix(
                        antlr_parsing, options['timerange'])
                    parsed_stix = parsed_stix_dictionary['parsed_stix']
                    start_time = parsed_stix_dictionary['start_time']
                    end_time = parsed_stix_dictionary['end_time']
                    return {
                        'parsed_stix': parsed_stix,
                        'start_time': start_time,
                        'end_time': end_time
                    }

            elif translate_type == RESULTS:
                # Converting data from the datasource to STIX objects
                return interface.translate_results(data_source, data, options)
            elif translate_type == SUPPORTED_ATTRIBUTES:
                # Return mapped STIX attributes supported by the data source
                data_model_mapper = self._build_data_mapper(module, options)
                mapped_attributes = data_model_mapper.map_data
                return {'supported_attributes': mapped_attributes}
            else:
                raise NotImplementedError('wrong parameter: ' + translate_type)
        except Exception as ex:
            print('Caught exception: ' + str(ex) + " " + str(type(ex)))
            response = dict()
            ErrorResponder.fill_error(response,
                                      message_struct={'exception': ex})
            return response