def __init__(self, FileName, ClassName, NameSpace, ClassNameSafe, TokenIDType): self.__file_name = FileName self.class_name = ClassName self.name_space = NameSpace self.class_name_safe = ClassNameSafe self.column_number_type = CodeFragment("size_t") self.line_number_type = CodeFragment("size_t") self.token_id_type = CodeFragment(TokenIDType)
def __start_mode(applicable_mode_name_list, mode_name_list): """If more then one mode is defined, then that requires an explicit definition 'start = mode'. """ assert len(applicable_mode_name_list) != 0 start_mode = lexer_mode.initial_mode.get_pure_code() if start_mode == "": # Choose an applicable mode as start mode start_mode = applicable_mode_name_list[0] lexer_mode.initial_mode = CodeFragment(start_mode) if len(applicable_mode_name_list) > 1: error_msg("No initial mode defined via 'start' while more than one applicable mode exists.\n" + \ "Use for example 'start = %s;' in the quex source file to define an initial mode." \ % start_mode) # This Branch: start mode is applicable and present else: FileName = lexer_mode.initial_mode.filename LineN = lexer_mode.initial_mode.line_n # Start mode present and applicable? verify_word_in_list(start_mode, mode_name_list, "Start mode '%s' is not defined." % start_mode, FileName, LineN) verify_word_in_list( start_mode, applicable_mode_name_list, "Start mode '%s' is inheritable only and cannot be instantiated." % start_mode, FileName, LineN)
def get_code_for_mode(Mode, ModeNameList): # -- some modes only define event handlers that are inherited if not Mode.has_matches(): return "", "" # -- 'end of stream' action if Mode.on_end_of_stream_code_fragments() == []: txt = "self.send(%sTERMINATION);\n" % Setup.input_token_id_prefix txt += "#ifdef __QUEX_OPTION_ANALYSER_RETURN_TYPE_IS_VOID\n" txt += " return /*__QUEX_TOKEN_ID_TERMINATION*/;\n" txt += "#else\n" txt += " return __QUEX_TOKEN_ID_TERMINATION;\n" txt += "#endif\n" Mode.on_end_of_stream = CodeFragment(txt) end_of_stream_action = action_code_formatter.do(Mode, Mode.on_end_of_stream_code_fragments(), "on_end_of_stream", None, EOF_ActionF=True) # -- 'default' action (nothing matched) if Mode.on_failure_code_fragments() == []: txt = "self.send(%sTERMINATION);\n" % Setup.input_token_id_prefix txt += "#ifdef __QUEX_OPTION_ANALYSER_RETURN_TYPE_IS_VOID\n" txt += " return /*__QUEX_TOKEN_ID_TERMINATION*/;\n" txt += "#else\n" txt += " return __QUEX_TOKEN_ID_TERMINATION;\n" txt += "#endif\n" Mode.on_failure = CodeFragment(txt) default_action = action_code_formatter.do(Mode, Mode.on_failure_code_fragments(), "on_failure", None, Default_ActionF=True) # -- adapt pattern-action pair information so that it can be treated # by the code generator. inheritance_info_str, pattern_action_pair_list = get_generator_input(Mode) analyzer_code = generator.do(pattern_action_pair_list, DefaultAction = PatternActionInfo(None, default_action), EndOfStreamAction = PatternActionInfo(None, end_of_stream_action), PrintStateMachineF = True, StateMachineName = Mode.name, AnalyserStateClassName = Setup.output_engine_name, StandAloneAnalyserF = False, QuexEngineHeaderDefinitionFile = Setup.output_file_stem, ModeNameList = ModeNameList) return analyzer_code, inheritance_info_str
def __prepare_on_failure_action(Mode): if not Mode.has_code_fragment_list("on_failure"): txt = "QUEX_ERROR_EXIT(\"\\n Match failure in mode '%s'.\\n\"\n" % Mode.name txt += " \" No 'on_failure' section provided for this mode.\\n\"\n" txt += " \" Proposal: Define 'on_failure' and analyze 'Lexeme'.\\n\");\n" Mode.set_code_fragment_list("on_failure", CodeFragment(txt)) # RETURNS: on_failure_action, db return action_code_formatter.do(Mode, Mode.get_code_fragment_list("on_failure"), "on_failure", None, Default_ActionF=True)
def __init__(self, Core=None): if Core == None: self._file_name = Setup.output_token_class_file self._file_name_implementation = Setup.output_token_class_file_implementation if Setup.token_class_name.find("::") != -1: Setup.token_class_name, \ Setup.token_class_name_space, \ Setup.token_class_name_safe = \ read_namespaced_name(Setup.token_class_name, "token class (options --token-class, --tc)") self.class_name = Setup.token_class_name self.class_name_safe = Setup.token_class_name_safe self.name_space = Setup.token_class_name_space self.open_for_derivation_f = False self.token_contains_token_id_f = True self.token_id_type = CodeFragment("size_t") self.column_number_type = CodeFragment("size_t") self.line_number_type = CodeFragment("size_t") self.distinct_db = {} self.union_db = {} for name in token_type_code_fragment_db.keys(): self.__dict__[name] = CodeFragment("") else: self._file_name = Core._file_name self._file_name_implementation = Core._file_name_implementation self.class_name = Core.class_name self.class_name_safe = Core.class_name_safe self.name_space = Core.name_space self.open_for_derivation_f = Core.open_for_derivation_f self.token_contains_token_id_f = Core.token_contains_token_id_f self.token_id_type = Core.token_id_type self.column_number_type = Core.column_number_type self.line_number_type = Core.line_number_type self.distinct_db = Core.distinct_db self.union_db = Core.union_db for name in token_type_code_fragment_db.keys(): self.__dict__[name] = Core.__dict__[name]
def __prepare_end_of_stream_action(Mode, IndentationSupportF): if not Mode.has_code_fragment_list("on_end_of_stream"): # We cannot make any assumptions about the token class, i.e. whether # it can take a lexeme or not. Thus, no passing of lexeme here. txt = "self_send(__QUEX_SETTING_TOKEN_ID_TERMINATION);\n" txt += "RETURN;\n" Mode.set_code_fragment_list("on_end_of_stream", CodeFragment(txt)) if IndentationSupportF: if Mode.default_indentation_handler_sufficient(): code = "QUEX_NAME(on_indentation)(me, /*Indentation*/0, LexemeNull);\n" else: code = "QUEX_NAME(%s_on_indentation)(me, /*Indentation*/0, LexemeNull);\n" % Mode.name code_fragment = CodeFragment(code) Mode.insert_code_fragment_at_front("on_end_of_stream", code_fragment) # RETURNS: end_of_stream_action, db return action_code_formatter.do(Mode, Mode.get_code_fragment_list("on_end_of_stream"), "on_end_of_stream", None, EOF_ActionF=True)
def do(Modes): """If consistency check fails due to a fatal error, then this functions exits back to the system with error code -1. Otherwise, in 'not so perfect' cases there might be only a warning being printed to standard output. """ if len(Modes) == 0: error_msg("No single mode defined - bailing out", Prefix="consistency check") # -- is there a mode that is applicable? for mode in Modes.values(): if mode.options["inheritable"] != "only": break else: error_msg("There is no mode that can be applied---all existing modes are 'inheritable only'.\n" + \ "modes are = " + repr(map(lambda m: m.name, Modes.values()))[1:-1], Prefix="consistency check") # -- is the initial mode defined if lexer_mode.initial_mode.get_code() == "": # find first mode that can actually be applied for mode in Modes.values(): if mode.options["inheritable"] != "only": selected_mode = mode.name break lexer_mode.initial_mode = CodeFragment(selected_mode) error_msg("no initial mode defined via 'start'\n" + \ "using mode '%s' as initial mode" % selected_mode, DontExitF=True, Prefix="warning") # -- is the start mode applicable? if Modes.has_key(lexer_mode.initial_mode.get_code()) == False: error_msg( "Start mode '%s' has not been defined anywhere." % lexer_mode.initial_mode.get_code(), lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n) if Modes[lexer_mode.initial_mode.get_code( )].options["inheritable"] == "only": error_msg( "Start mode '%s' is inheritable only and cannot be instantiated." % lexer_mode.initial_mode.get_code(), lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n) # -- check for circular inheritance check_circular_inheritance(Modes) # -- mode specific checks for mode in Modes.values(): mode.consistency_check()
class TokenTypeDescriptorCore: """Object used during the generation of the TokenTypeDescriptor.""" def __init__(self, Core=None): if Core == None: self._file_name = Setup.output_token_class_file self._file_name_implementation = Setup.output_token_class_file_implementation if Setup.token_class_name.find("::") != -1: Setup.token_class_name, \ Setup.token_class_name_space, \ Setup.token_class_name_safe = \ read_namespaced_name(Setup.token_class_name, "token class (options --token-class, --tc)") self.class_name = Setup.token_class_name self.class_name_safe = Setup.token_class_name_safe self.name_space = Setup.token_class_name_space self.open_for_derivation_f = False self.token_contains_token_id_f = True self.token_id_type = CodeFragment("size_t") self.column_number_type = CodeFragment("size_t") self.line_number_type = CodeFragment("size_t") self.distinct_db = {} self.union_db = {} for name in token_type_code_fragment_db.keys(): self.__dict__[name] = CodeFragment("") else: self._file_name = Core._file_name self._file_name_implementation = Core._file_name_implementation self.class_name = Core.class_name self.class_name_safe = Core.class_name_safe self.name_space = Core.name_space self.open_for_derivation_f = Core.open_for_derivation_f self.token_contains_token_id_f = Core.token_contains_token_id_f self.token_id_type = Core.token_id_type self.column_number_type = Core.column_number_type self.line_number_type = Core.line_number_type self.distinct_db = Core.distinct_db self.union_db = Core.union_db for name in token_type_code_fragment_db.keys(): self.__dict__[name] = Core.__dict__[name] def set_file_name(self, FileName): self._file_name = FileName ext = Setup.language_db[Setup.language].extension_db[ Setup.output_file_naming_scheme][HEADER_IMPLEMTATION] self._file_name_implementation = FileName + ext def __repr__(self): txt = "" if self._file_name != "": txt += "file name: '%s'\n" % self._file_name txt += "class: '%s'\n" % self.class_name if self.open_for_derivation_f: txt += " (with virtual destructor)\n" if self.token_contains_token_id_f == False: txt += " (token id not part of token object)\n" txt += "namespace: '%s'\n" % repr(self.name_space)[1:-1] txt += "type(token_id) = %s\n" % self.token_id_type.get_pure_code( ) txt += "type(column_number) = %s\n" % self.column_number_type.get_pure_code( ) txt += "type(line_number) = %s\n" % self.line_number_type.get_pure_code( ) txt += "distinct members {\n" # '0' to make sure, that it works on an empty sequence too. L = self.distinct_members_type_name_length_max() for name, type_code in self.distinct_db.items(): txt += " %s%s %s\n" % (type_code.get_pure_code(), " " * (L - len(type_code.get_pure_code())), name) txt += "}\n" txt += "union members {\n" # '0' to make sure, that it works on an empty sequence too. L = self.union_members_type_name_length_max() for name, type_descr in self.union_db.items(): if type(type_descr) == dict: txt += " {\n" for sub_name, sub_type in type_descr.items(): txt += " %s%s %s\n" % \ (sub_type.get_pure_code(), " " * (L - len(sub_type.get_pure_code())-4), sub_name) txt += " }\n" else: txt += " %s%s %s\n" % \ (type_descr.get_pure_code(), " " * (L - len(type_descr.get_pure_code())), name) txt += "}\n" # constructor / copy / destructor if self.constructor.get_pure_code() != "": txt += "constructor {\n" txt += self.constructor.get_code() txt += "}" if self.copy.get_pure_code() != "": txt += "copy {\n" txt += self.copy.get_code() txt += "}" if self.destructor.get_pure_code() != "": txt += "destructor {\n" txt += self.destructor.get_code() txt += "}" if self.body.get_pure_code() != "": txt += "body {\n" txt += self.body.get_code() txt += "}" return txt def manually_written(self): return False
class TokenTypeDescriptorCore: """Object used during the generation of the TokenTypeDescriptor.""" def __init__(self, Core=None): if Core == None: self._file_name = Setup.output_token_class_file self._file_name_implementation = Setup.output_token_class_file_implementation if Setup.token_class_name.find("::") != -1: Setup.token_class_name, \ Setup.token_class_name_space, \ Setup.token_class_name_safe = \ read_namespaced_name(Setup.token_class_name, "token class (options --token-class, --tc)") self.class_name = Setup.token_class_name self.class_name_safe = Setup.token_class_name_safe self.name_space = Setup.token_class_name_space self.open_for_derivation_f = False self.token_contains_token_id_f = True self.token_id_type = CodeFragment("size_t") self.column_number_type = CodeFragment("size_t") self.line_number_type = CodeFragment("size_t") self.distinct_db = {} self.union_db = {} for name in token_type_code_fragment_db.keys(): self.__dict__[name] = CodeFragment("") else: self._file_name = Core._file_name self._file_name_implementation = Core._file_name_implementation self.class_name = Core.class_name self.class_name_safe = Core.class_name_safe self.name_space = Core.name_space self.open_for_derivation_f = Core.open_for_derivation_f self.token_contains_token_id_f = Core.token_contains_token_id_f self.token_id_type = Core.token_id_type self.column_number_type = Core.column_number_type self.line_number_type = Core.line_number_type self.distinct_db = Core.distinct_db self.union_db = Core.union_db for name in token_type_code_fragment_db.keys(): self.__dict__[name] = Core.__dict__[name] def set_file_name(self, FileName): self._file_name = FileName ext = Setup.language_db[Setup.language].extension_db[Setup.output_file_naming_scheme][HEADER_IMPLEMTATION] self._file_name_implementation = FileName + ext def __repr__(self): txt = "" if self._file_name != "": txt += "file name: '%s'\n" % self._file_name txt += "class: '%s'\n" % self.class_name if self.open_for_derivation_f: txt += " (with virtual destructor)\n" if self.token_contains_token_id_f == False: txt += " (token id not part of token object)\n" txt += "namespace: '%s'\n" % repr(self.name_space)[1:-1] txt += "type(token_id) = %s\n" % self.token_id_type.get_pure_code() txt += "type(column_number) = %s\n" % self.column_number_type.get_pure_code() txt += "type(line_number) = %s\n" % self.line_number_type.get_pure_code() txt += "distinct members {\n" # '0' to make sure, that it works on an empty sequence too. L = self.distinct_members_type_name_length_max() for name, type_code in self.distinct_db.items(): txt += " %s%s %s\n" % (type_code.get_pure_code(), " " * (L - len(type_code.get_pure_code())), name) txt += "}\n" txt += "union members {\n" # '0' to make sure, that it works on an empty sequence too. L = self.union_members_type_name_length_max() for name, type_descr in self.union_db.items(): if type(type_descr) == dict: txt += " {\n" for sub_name, sub_type in type_descr.items(): txt += " %s%s %s\n" % \ (sub_type.get_pure_code(), " " * (L - len(sub_type.get_pure_code())-4), sub_name) txt += " }\n" else: txt += " %s%s %s\n" % \ (type_descr.get_pure_code(), " " * (L - len(type_descr.get_pure_code())), name) txt += "}\n" # constructor / copy / destructor if self.constructor.get_pure_code() != "": txt += "constructor {\n" txt += self.constructor.get_code() txt += "}" if self.copy.get_pure_code() != "": txt += "copy {\n" txt += self.copy.get_code() txt += "}" if self.destructor.get_pure_code() != "": txt += "destructor {\n" txt += self.destructor.get_code() txt += "}" if self.body.get_pure_code() != "": txt += "body {\n" txt += self.body.get_code() txt += "}" return txt def manually_written(self): return False
def parse_mode_option(fh, new_mode): skip_whitespace(fh) # (*) base modes if fh.read(1) != "<": return False skip_whitespace(fh) identifier = read_identifier(fh).strip() if identifier == "": error_msg("missing identifer after start of mode option '<'", fh) skip_whitespace(fh) if fh.read(1) != ":": error_msg("missing ':' after option name '%s'" % identifier, fh) skip_whitespace(fh) if identifier == "skip": # A skipper 'eats' characters at the beginning of a pattern that belong # to a specified set of characters. A useful application is most probably # the whitespace skipper '[ \t\n]'. The skipper definition allows quex to # implement a very effective way to skip these regions. pattern_str, trigger_set = regular_expression.parse_character_set( fh, PatternStringF=True) skip_whitespace(fh) if fh.read(1) != ">": error_msg("missing closing '>' for mode option '%s'." % identifier, fh) if trigger_set.is_empty(): error_msg("Empty trigger set for skipper." % identifier, fh) # TriggerSet skipping is implemented the following way: As soon as one element of the # trigger set appears, the state machine enters the 'trigger set skipper section'. opener_sm = StateMachine() opener_sm.add_transition(opener_sm.init_state_index, trigger_set, AcceptanceF=True) action = CodeFragment(create_skip_code(trigger_set)) # Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action. new_mode.add_match(pattern_str, action, opener_sm) return True elif identifier == "skip_range": # A non-nesting skipper can contain a full fledged regular expression as opener, # since it only effects the trigger. Not so the nested range skipper-see below. # -- opener skip_whitespace(fh) opener_str, opener_sm = regular_expression.parse(fh) skip_whitespace(fh) # -- closer if fh.read(1) != "\"": error_msg( "closing pattern for skip_range can only be a string and must start with a quote like \".", fh) closer_sequence = snap_character_string.get_character_code_sequence(fh) skip_whitespace(fh) if fh.read(1) != ">": error_msg("missing closing '>' for mode option '%s'" % identifier, fh) action = CodeFragment(create_skip_range_code(closer_sequence)) # Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action. new_mode.add_match(opener_str, action, opener_sm) return True elif identifier == "skip_nesting_range": error_msg("skip_nesting_range is not yet supported.", fh) else: value, i = read_until_letter(fh, [">"], Verbose=1) if i != 0: error_msg("missing closing '>' for mode option '%s'" % identifier, fh) value = value.strip() # Does the specified option actually exist? if not lexer_mode.mode_option_info_db.has_key(identifier): error_msg("tried to set option '%s' which does not exist!\n" % identifier + \ "options are %s" % repr(lexer_mode.mode_option_info_db.keys()), fh) # Is the option of the appropriate value? option_info = lexer_mode.mode_option_info_db[identifier] if option_info.type != "list" and value not in option_info.domain: error_msg("Tried to set value '%s' for option '%s'. " % (Value, Option) + \ "Though, possible \n" + \ "for this option are %s" % repr(oi.domain), fh) # Finally, set the option new_mode.add_option(identifier, value) return True