def __init__(self, FileName, ClassName, NameSpace, ClassNameSafe,
TokenIDType):
self.__file_name = FileName
self.class_name = ClassName
self.name_space = NameSpace
self.class_name_safe = ClassNameSafe
self.column_number_type = CodeFragment("size_t")
self.line_number_type = CodeFragment("size_t")
self.token_id_type = CodeFragment(TokenIDType)
def __start_mode(applicable_mode_name_list, mode_name_list):
"""If more then one mode is defined, then that requires an explicit
definition 'start = mode'.
"""
assert len(applicable_mode_name_list) != 0
start_mode = lexer_mode.initial_mode.get_pure_code()
if start_mode == "":
# Choose an applicable mode as start mode
start_mode = applicable_mode_name_list[0]
lexer_mode.initial_mode = CodeFragment(start_mode)
if len(applicable_mode_name_list) > 1:
error_msg("No initial mode defined via 'start' while more than one applicable mode exists.\n" + \
"Use for example 'start = %s;' in the quex source file to define an initial mode." \
% start_mode)
# This Branch: start mode is applicable and present
else:
FileName = lexer_mode.initial_mode.filename
LineN = lexer_mode.initial_mode.line_n
# Start mode present and applicable?
verify_word_in_list(start_mode, mode_name_list,
"Start mode '%s' is not defined." % start_mode,
FileName, LineN)
verify_word_in_list(
start_mode, applicable_mode_name_list,
"Start mode '%s' is inheritable only and cannot be instantiated." %
start_mode, FileName, LineN)
def get_code_for_mode(Mode, ModeNameList):
# -- some modes only define event handlers that are inherited
if not Mode.has_matches(): return "", ""
# -- 'end of stream' action
if Mode.on_end_of_stream_code_fragments() == []:
txt = "self.send(%sTERMINATION);\n" % Setup.input_token_id_prefix
txt += "#ifdef __QUEX_OPTION_ANALYSER_RETURN_TYPE_IS_VOID\n"
txt += " return /*__QUEX_TOKEN_ID_TERMINATION*/;\n"
txt += "#else\n"
txt += " return __QUEX_TOKEN_ID_TERMINATION;\n"
txt += "#endif\n"
Mode.on_end_of_stream = CodeFragment(txt)
end_of_stream_action = action_code_formatter.do(Mode, Mode.on_end_of_stream_code_fragments(),
"on_end_of_stream", None, EOF_ActionF=True)
# -- 'default' action (nothing matched)
if Mode.on_failure_code_fragments() == []:
txt = "self.send(%sTERMINATION);\n" % Setup.input_token_id_prefix
txt += "#ifdef __QUEX_OPTION_ANALYSER_RETURN_TYPE_IS_VOID\n"
txt += " return /*__QUEX_TOKEN_ID_TERMINATION*/;\n"
txt += "#else\n"
txt += " return __QUEX_TOKEN_ID_TERMINATION;\n"
txt += "#endif\n"
Mode.on_failure = CodeFragment(txt)
default_action = action_code_formatter.do(Mode, Mode.on_failure_code_fragments(),
"on_failure", None, Default_ActionF=True)
# -- adapt pattern-action pair information so that it can be treated
# by the code generator.
inheritance_info_str, pattern_action_pair_list = get_generator_input(Mode)
analyzer_code = generator.do(pattern_action_pair_list,
DefaultAction = PatternActionInfo(None, default_action),
EndOfStreamAction = PatternActionInfo(None, end_of_stream_action),
PrintStateMachineF = True,
StateMachineName = Mode.name,
AnalyserStateClassName = Setup.output_engine_name,
StandAloneAnalyserF = False,
QuexEngineHeaderDefinitionFile = Setup.output_file_stem,
ModeNameList = ModeNameList)
return analyzer_code, inheritance_info_str
def __prepare_on_failure_action(Mode):
if not Mode.has_code_fragment_list("on_failure"):
txt = "QUEX_ERROR_EXIT(\"\\n Match failure in mode '%s'.\\n\"\n" % Mode.name
txt += " \" No 'on_failure' section provided for this mode.\\n\"\n"
txt += " \" Proposal: Define 'on_failure' and analyze 'Lexeme'.\\n\");\n"
Mode.set_code_fragment_list("on_failure", CodeFragment(txt))
# RETURNS: on_failure_action, db
return action_code_formatter.do(Mode, Mode.get_code_fragment_list("on_failure"),
"on_failure", None, Default_ActionF=True)
def __init__(self, Core=None):
if Core == None:
self._file_name = Setup.output_token_class_file
self._file_name_implementation = Setup.output_token_class_file_implementation
if Setup.token_class_name.find("::") != -1:
Setup.token_class_name, \
Setup.token_class_name_space, \
Setup.token_class_name_safe = \
read_namespaced_name(Setup.token_class_name,
"token class (options --token-class, --tc)")
self.class_name = Setup.token_class_name
self.class_name_safe = Setup.token_class_name_safe
self.name_space = Setup.token_class_name_space
self.open_for_derivation_f = False
self.token_contains_token_id_f = True
self.token_id_type = CodeFragment("size_t")
self.column_number_type = CodeFragment("size_t")
self.line_number_type = CodeFragment("size_t")
self.distinct_db = {}
self.union_db = {}
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = CodeFragment("")
else:
self._file_name = Core._file_name
self._file_name_implementation = Core._file_name_implementation
self.class_name = Core.class_name
self.class_name_safe = Core.class_name_safe
self.name_space = Core.name_space
self.open_for_derivation_f = Core.open_for_derivation_f
self.token_contains_token_id_f = Core.token_contains_token_id_f
self.token_id_type = Core.token_id_type
self.column_number_type = Core.column_number_type
self.line_number_type = Core.line_number_type
self.distinct_db = Core.distinct_db
self.union_db = Core.union_db
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = Core.__dict__[name]
def __prepare_end_of_stream_action(Mode, IndentationSupportF):
if not Mode.has_code_fragment_list("on_end_of_stream"):
# We cannot make any assumptions about the token class, i.e. whether
# it can take a lexeme or not. Thus, no passing of lexeme here.
txt = "self_send(__QUEX_SETTING_TOKEN_ID_TERMINATION);\n"
txt += "RETURN;\n"
Mode.set_code_fragment_list("on_end_of_stream", CodeFragment(txt))
if IndentationSupportF:
if Mode.default_indentation_handler_sufficient():
code = "QUEX_NAME(on_indentation)(me, /*Indentation*/0, LexemeNull);\n"
else:
code = "QUEX_NAME(%s_on_indentation)(me, /*Indentation*/0, LexemeNull);\n" % Mode.name
code_fragment = CodeFragment(code)
Mode.insert_code_fragment_at_front("on_end_of_stream", code_fragment)
# RETURNS: end_of_stream_action, db
return action_code_formatter.do(Mode, Mode.get_code_fragment_list("on_end_of_stream"),
"on_end_of_stream", None, EOF_ActionF=True)
def do(Modes):
"""If consistency check fails due to a fatal error, then this functions
exits back to the system with error code -1. Otherwise, in 'not so
perfect' cases there might be only a warning being printed to standard
output.
"""
if len(Modes) == 0:
error_msg("No single mode defined - bailing out",
Prefix="consistency check")
# -- is there a mode that is applicable?
for mode in Modes.values():
if mode.options["inheritable"] != "only": break
else:
error_msg("There is no mode that can be applied---all existing modes are 'inheritable only'.\n" + \
"modes are = " + repr(map(lambda m: m.name, Modes.values()))[1:-1],
Prefix="consistency check")
# -- is the initial mode defined
if lexer_mode.initial_mode.get_code() == "":
# find first mode that can actually be applied
for mode in Modes.values():
if mode.options["inheritable"] != "only":
selected_mode = mode.name
break
lexer_mode.initial_mode = CodeFragment(selected_mode)
error_msg("no initial mode defined via 'start'\n" + \
"using mode '%s' as initial mode" % selected_mode, DontExitF=True,
Prefix="warning")
# -- is the start mode applicable?
if Modes.has_key(lexer_mode.initial_mode.get_code()) == False:
error_msg(
"Start mode '%s' has not been defined anywhere." %
lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
if Modes[lexer_mode.initial_mode.get_code(
)].options["inheritable"] == "only":
error_msg(
"Start mode '%s' is inheritable only and cannot be instantiated." %
lexer_mode.initial_mode.get_code(),
lexer_mode.initial_mode.filename, lexer_mode.initial_mode.line_n)
# -- check for circular inheritance
check_circular_inheritance(Modes)
# -- mode specific checks
for mode in Modes.values():
mode.consistency_check()
def __init__(self, Core=None):
if Core == None:
self._file_name = Setup.output_token_class_file
self._file_name_implementation = Setup.output_token_class_file_implementation
if Setup.token_class_name.find("::") != -1:
Setup.token_class_name, \
Setup.token_class_name_space, \
Setup.token_class_name_safe = \
read_namespaced_name(Setup.token_class_name,
"token class (options --token-class, --tc)")
self.class_name = Setup.token_class_name
self.class_name_safe = Setup.token_class_name_safe
self.name_space = Setup.token_class_name_space
self.open_for_derivation_f = False
self.token_contains_token_id_f = True
self.token_id_type = CodeFragment("size_t")
self.column_number_type = CodeFragment("size_t")
self.line_number_type = CodeFragment("size_t")
self.distinct_db = {}
self.union_db = {}
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = CodeFragment("")
else:
self._file_name = Core._file_name
self._file_name_implementation = Core._file_name_implementation
self.class_name = Core.class_name
self.class_name_safe = Core.class_name_safe
self.name_space = Core.name_space
self.open_for_derivation_f = Core.open_for_derivation_f
self.token_contains_token_id_f = Core.token_contains_token_id_f
self.token_id_type = Core.token_id_type
self.column_number_type = Core.column_number_type
self.line_number_type = Core.line_number_type
self.distinct_db = Core.distinct_db
self.union_db = Core.union_db
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = Core.__dict__[name]
class TokenTypeDescriptorCore:
"""Object used during the generation of the TokenTypeDescriptor."""
def __init__(self, Core=None):
if Core == None:
self._file_name = Setup.output_token_class_file
self._file_name_implementation = Setup.output_token_class_file_implementation
if Setup.token_class_name.find("::") != -1:
Setup.token_class_name, \
Setup.token_class_name_space, \
Setup.token_class_name_safe = \
read_namespaced_name(Setup.token_class_name,
"token class (options --token-class, --tc)")
self.class_name = Setup.token_class_name
self.class_name_safe = Setup.token_class_name_safe
self.name_space = Setup.token_class_name_space
self.open_for_derivation_f = False
self.token_contains_token_id_f = True
self.token_id_type = CodeFragment("size_t")
self.column_number_type = CodeFragment("size_t")
self.line_number_type = CodeFragment("size_t")
self.distinct_db = {}
self.union_db = {}
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = CodeFragment("")
else:
self._file_name = Core._file_name
self._file_name_implementation = Core._file_name_implementation
self.class_name = Core.class_name
self.class_name_safe = Core.class_name_safe
self.name_space = Core.name_space
self.open_for_derivation_f = Core.open_for_derivation_f
self.token_contains_token_id_f = Core.token_contains_token_id_f
self.token_id_type = Core.token_id_type
self.column_number_type = Core.column_number_type
self.line_number_type = Core.line_number_type
self.distinct_db = Core.distinct_db
self.union_db = Core.union_db
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = Core.__dict__[name]
def set_file_name(self, FileName):
self._file_name = FileName
ext = Setup.language_db[Setup.language].extension_db[
Setup.output_file_naming_scheme][HEADER_IMPLEMTATION]
self._file_name_implementation = FileName + ext
def __repr__(self):
txt = ""
if self._file_name != "":
txt += "file name: '%s'\n" % self._file_name
txt += "class: '%s'\n" % self.class_name
if self.open_for_derivation_f:
txt += " (with virtual destructor)\n"
if self.token_contains_token_id_f == False:
txt += " (token id not part of token object)\n"
txt += "namespace: '%s'\n" % repr(self.name_space)[1:-1]
txt += "type(token_id) = %s\n" % self.token_id_type.get_pure_code(
)
txt += "type(column_number) = %s\n" % self.column_number_type.get_pure_code(
)
txt += "type(line_number) = %s\n" % self.line_number_type.get_pure_code(
)
txt += "distinct members {\n"
# '0' to make sure, that it works on an empty sequence too.
L = self.distinct_members_type_name_length_max()
for name, type_code in self.distinct_db.items():
txt += " %s%s %s\n" % (type_code.get_pure_code(), " " *
(L - len(type_code.get_pure_code())),
name)
txt += "}\n"
txt += "union members {\n"
# '0' to make sure, that it works on an empty sequence too.
L = self.union_members_type_name_length_max()
for name, type_descr in self.union_db.items():
if type(type_descr) == dict:
txt += " {\n"
for sub_name, sub_type in type_descr.items():
txt += " %s%s %s\n" % \
(sub_type.get_pure_code(),
" " * (L - len(sub_type.get_pure_code())-4),
sub_name)
txt += " }\n"
else:
txt += " %s%s %s\n" % \
(type_descr.get_pure_code(),
" " * (L - len(type_descr.get_pure_code())),
name)
txt += "}\n"
# constructor / copy / destructor
if self.constructor.get_pure_code() != "":
txt += "constructor {\n"
txt += self.constructor.get_code()
txt += "}"
if self.copy.get_pure_code() != "":
txt += "copy {\n"
txt += self.copy.get_code()
txt += "}"
if self.destructor.get_pure_code() != "":
txt += "destructor {\n"
txt += self.destructor.get_code()
txt += "}"
if self.body.get_pure_code() != "":
txt += "body {\n"
txt += self.body.get_code()
txt += "}"
return txt
def manually_written(self):
return False
class TokenTypeDescriptorCore:
"""Object used during the generation of the TokenTypeDescriptor."""
def __init__(self, Core=None):
if Core == None:
self._file_name = Setup.output_token_class_file
self._file_name_implementation = Setup.output_token_class_file_implementation
if Setup.token_class_name.find("::") != -1:
Setup.token_class_name, \
Setup.token_class_name_space, \
Setup.token_class_name_safe = \
read_namespaced_name(Setup.token_class_name,
"token class (options --token-class, --tc)")
self.class_name = Setup.token_class_name
self.class_name_safe = Setup.token_class_name_safe
self.name_space = Setup.token_class_name_space
self.open_for_derivation_f = False
self.token_contains_token_id_f = True
self.token_id_type = CodeFragment("size_t")
self.column_number_type = CodeFragment("size_t")
self.line_number_type = CodeFragment("size_t")
self.distinct_db = {}
self.union_db = {}
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = CodeFragment("")
else:
self._file_name = Core._file_name
self._file_name_implementation = Core._file_name_implementation
self.class_name = Core.class_name
self.class_name_safe = Core.class_name_safe
self.name_space = Core.name_space
self.open_for_derivation_f = Core.open_for_derivation_f
self.token_contains_token_id_f = Core.token_contains_token_id_f
self.token_id_type = Core.token_id_type
self.column_number_type = Core.column_number_type
self.line_number_type = Core.line_number_type
self.distinct_db = Core.distinct_db
self.union_db = Core.union_db
for name in token_type_code_fragment_db.keys():
self.__dict__[name] = Core.__dict__[name]
def set_file_name(self, FileName):
self._file_name = FileName
ext = Setup.language_db[Setup.language].extension_db[Setup.output_file_naming_scheme][HEADER_IMPLEMTATION]
self._file_name_implementation = FileName + ext
def __repr__(self):
txt = ""
if self._file_name != "":
txt += "file name: '%s'\n" % self._file_name
txt += "class: '%s'\n" % self.class_name
if self.open_for_derivation_f:
txt += " (with virtual destructor)\n"
if self.token_contains_token_id_f == False:
txt += " (token id not part of token object)\n"
txt += "namespace: '%s'\n" % repr(self.name_space)[1:-1]
txt += "type(token_id) = %s\n" % self.token_id_type.get_pure_code()
txt += "type(column_number) = %s\n" % self.column_number_type.get_pure_code()
txt += "type(line_number) = %s\n" % self.line_number_type.get_pure_code()
txt += "distinct members {\n"
# '0' to make sure, that it works on an empty sequence too.
L = self.distinct_members_type_name_length_max()
for name, type_code in self.distinct_db.items():
txt += " %s%s %s\n" % (type_code.get_pure_code(), " " * (L - len(type_code.get_pure_code())), name)
txt += "}\n"
txt += "union members {\n"
# '0' to make sure, that it works on an empty sequence too.
L = self.union_members_type_name_length_max()
for name, type_descr in self.union_db.items():
if type(type_descr) == dict:
txt += " {\n"
for sub_name, sub_type in type_descr.items():
txt += " %s%s %s\n" % \
(sub_type.get_pure_code(),
" " * (L - len(sub_type.get_pure_code())-4),
sub_name)
txt += " }\n"
else:
txt += " %s%s %s\n" % \
(type_descr.get_pure_code(),
" " * (L - len(type_descr.get_pure_code())),
name)
txt += "}\n"
# constructor / copy / destructor
if self.constructor.get_pure_code() != "":
txt += "constructor {\n"
txt += self.constructor.get_code()
txt += "}"
if self.copy.get_pure_code() != "":
txt += "copy {\n"
txt += self.copy.get_code()
txt += "}"
if self.destructor.get_pure_code() != "":
txt += "destructor {\n"
txt += self.destructor.get_code()
txt += "}"
if self.body.get_pure_code() != "":
txt += "body {\n"
txt += self.body.get_code()
txt += "}"
return txt
def manually_written(self):
return False
def parse_mode_option(fh, new_mode):
skip_whitespace(fh)
# (*) base modes
if fh.read(1) != "<": return False
skip_whitespace(fh)
identifier = read_identifier(fh).strip()
if identifier == "":
error_msg("missing identifer after start of mode option '<'", fh)
skip_whitespace(fh)
if fh.read(1) != ":":
error_msg("missing ':' after option name '%s'" % identifier, fh)
skip_whitespace(fh)
if identifier == "skip":
# A skipper 'eats' characters at the beginning of a pattern that belong
# to a specified set of characters. A useful application is most probably
# the whitespace skipper '[ \t\n]'. The skipper definition allows quex to
# implement a very effective way to skip these regions.
pattern_str, trigger_set = regular_expression.parse_character_set(
fh, PatternStringF=True)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'." % identifier,
fh)
if trigger_set.is_empty():
error_msg("Empty trigger set for skipper." % identifier, fh)
# TriggerSet skipping is implemented the following way: As soon as one element of the
# trigger set appears, the state machine enters the 'trigger set skipper section'.
opener_sm = StateMachine()
opener_sm.add_transition(opener_sm.init_state_index,
trigger_set,
AcceptanceF=True)
action = CodeFragment(create_skip_code(trigger_set))
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
new_mode.add_match(pattern_str, action, opener_sm)
return True
elif identifier == "skip_range":
# A non-nesting skipper can contain a full fledged regular expression as opener,
# since it only effects the trigger. Not so the nested range skipper-see below.
# -- opener
skip_whitespace(fh)
opener_str, opener_sm = regular_expression.parse(fh)
skip_whitespace(fh)
# -- closer
if fh.read(1) != "\"":
error_msg(
"closing pattern for skip_range can only be a string and must start with a quote like \".",
fh)
closer_sequence = snap_character_string.get_character_code_sequence(fh)
skip_whitespace(fh)
if fh.read(1) != ">":
error_msg("missing closing '>' for mode option '%s'" % identifier,
fh)
action = CodeFragment(create_skip_range_code(closer_sequence))
# Enter the skipper as if the opener pattern was a normal pattern and the 'skipper' is the action.
new_mode.add_match(opener_str, action, opener_sm)
return True
elif identifier == "skip_nesting_range":
error_msg("skip_nesting_range is not yet supported.", fh)
else:
value, i = read_until_letter(fh, [">"], Verbose=1)
if i != 0:
error_msg("missing closing '>' for mode option '%s'" % identifier,
fh)
value = value.strip()
# Does the specified option actually exist?
if not lexer_mode.mode_option_info_db.has_key(identifier):
error_msg("tried to set option '%s' which does not exist!\n" % identifier + \
"options are %s" % repr(lexer_mode.mode_option_info_db.keys()), fh)
# Is the option of the appropriate value?
option_info = lexer_mode.mode_option_info_db[identifier]
if option_info.type != "list" and value not in option_info.domain:
error_msg("Tried to set value '%s' for option '%s'. " % (Value, Option) + \
"Though, possible \n" + \
"for this option are %s" % repr(oi.domain), fh)
# Finally, set the option
new_mode.add_option(identifier, value)
return True
