def do(Modes):
LexerClassName = Setup.analyzer_class_name
DerivedClassName = Setup.analyzer_derived_class_name
DerivedClassHeaderFileName = Setup.analyzer_derived_class_file
if DerivedClassHeaderFileName != "": txt = "#include <" + Setup.get_file_reference(DerivedClassHeaderFileName) +">\n"
else: txt = "#include \"" + Setup.get_file_reference(Setup.output_header_file) +"\"\n"
txt += "#include <quex/code_base/analyzer/C-adaptions.h>\n"
# -- mode class member function definitions (on_entry, on_exit, has_base, ...)
mode_class_member_functions_txt = write_member_functions(Modes.values())
mode_objects_txt = ""
for mode_name, mode in Modes.items():
if mode.options["inheritable"] == "only": continue
mode_objects_txt += "/* Global */QUEX_NAME(Mode) QUEX_NAME(%s);\n" % mode_name
txt += "QUEX_NAMESPACE_MAIN_OPEN\n"
txt += mode_objects_txt
txt += mode_class_member_functions_txt
txt += "QUEX_NAMESPACE_MAIN_CLOSE\n"
txt = blue_print(txt, [["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_DERIVED_CLASS_NAME$$", DerivedClassName]])
return txt
def ENGINE_TEXT_EPILOG(self):
if Setup.analyzer_derived_class_file:
header = "<" + Setup.get_file_reference(
Setup.analyzer_derived_class_file) + ">\n"
else:
header = "\"" + Setup.get_file_reference(
Setup.output_header_file) + "\"\n"
return cpp_include_Multi_i_str.replace("$$HEADER$$", header)
def do(setup):
"""________________________________________________________________________
(1) Error Check
(2) Generates a file containing:
-- token id definitions (if they are not done in '--foreign-token-id-file').
-- const string& TokenClass::map_id_to_name(), i.e. a function which can
convert token ids into strings.
________________________________________________________________________
"""
global file_str
# At this point, assume that the token type has been generated.
assert blackboard.token_type_definition is not None
# (1) Error Check
#
__warn_implicit_token_definitions()
if len(Setup.token_id_foreign_definition_file) == 0:
__autogenerate_token_id_numbers()
__warn_on_double_definition()
# If a mandatory token id is missing, this means that Quex did not
# properly do implicit token definitions. Program error-abort.
__error_on_mandatory_token_id_missing(AssertF=True)
else:
__error_on_mandatory_token_id_missing()
__error_on_no_specific_token_ids()
# (2) Generate token id file (if not specified outside)
#
if len(Setup.token_id_foreign_definition_file) != 0:
# Content of file = inclusion of 'Setup.token_id_foreign_definition_file'.
token_id_txt = [
"#include \"%s\"\n" %
Setup.get_file_reference(Setup.token_id_foreign_definition_file)
]
else:
token_id_txt = __get_token_id_definition_txt()
include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())
content = blue_print(file_str, [
["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
[
"$$TOKEN_CLASS_DEFINITION_FILE$$",
Setup.get_file_reference(
blackboard.token_type_definition.get_file_name())
],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", include_guard_ext],
])
return content
def do(setup):
"""________________________________________________________________________
(1) Error Check
(2) Generates a file containing:
-- token id definitions (if they are not done in '--foreign-token-id-file').
-- const string& TokenClass::map_id_to_name(), i.e. a function which can
convert token ids into strings.
________________________________________________________________________
"""
global file_str
# At this point, assume that the token type has been generated.
assert blackboard.token_type_definition is not None
# (1) Error Check
#
__warn_implicit_token_definitions()
if len(Setup.token_id_foreign_definition_file) == 0:
__autogenerate_token_id_numbers()
__warn_on_double_definition()
# If a mandatory token id is missing, this means that Quex did not
# properly do implicit token definitions. Program error-abort.
__error_on_mandatory_token_id_missing(AssertF=True)
else:
__error_on_mandatory_token_id_missing()
__error_on_no_specific_token_ids()
# (2) Generate token id file (if not specified outside)
#
if len(Setup.token_id_foreign_definition_file) != 0:
# Content of file = inclusion of 'Setup.token_id_foreign_definition_file'.
token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(Setup.token_id_foreign_definition_file)]
else:
token_id_txt = __get_token_id_definition_txt()
include_guard_ext = get_include_guard_extension(Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", Setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", include_guard_ext],
])
return content
def UserCodeFragment_straighten_open_line_pragmas(filename, Language):
if Language not in UserCodeFragment_OpenLinePragma.keys():
return
fh = open_file_or_die(filename)
norm_filename = Setup.get_file_reference(filename)
new_content = []
line_n = 0
LinePragmaInfoList = UserCodeFragment_OpenLinePragma[Language]
for line in fh.readlines():
line_n += 1
if Language == "C":
for info in LinePragmaInfoList:
if line.find(info[0]) == -1: continue
line = info[1]
# Since by some definition, line number pragmas < 32768; let us avoid
# compiler warnings by setting line_n = min(line_n, 32768)
line = line.replace("NUMBER", repr(int(min(line_n + 1,
32767))))
# Even under Windows (tm), the '/' is accepted. Thus do not rely on 'normpath'
line = line.replace("FILENAME", norm_filename)
if len(line) == 0 or line[-1] != "\n":
line = line + "\n"
new_content.append(line)
fh.close()
write_safely_and_close(filename, "".join(new_content))
def do_implementation(ModeDB):
FileTemplate = os.path.normpath(QUEX_PATH + Lng["$code_base"] +
"/analyzer/TXT-Cpp.i")
func_txt = get_file_content_or_die(FileTemplate)
func_txt = blue_print(func_txt, [
[
"$$CONSTRUCTOR_EXTENSTION$$",
Lng.SOURCE_REFERENCED(blackboard.class_constructor_extension)
],
[
"$$CONVERTER_HELPER_I$$",
Setup.get_file_reference(Setup.output_buffer_codec_header_i)
],
[
"$$CONSTRUCTOR_MODE_DB_INITIALIZATION_CODE$$",
get_constructor_code(ModeDB.values())
],
[
"$$MEMENTO_EXTENSIONS_PACK$$",
Lng.SOURCE_REFERENCED(blackboard.memento_pack_extension)
],
[
"$$MEMENTO_EXTENSIONS_UNPACK$$",
Lng.SOURCE_REFERENCED(blackboard.memento_unpack_extension)
],
])
return func_txt
def UserCodeFragment_straighten_open_line_pragmas(filename, Language):
if Language not in UserCodeFragment_OpenLinePragma.keys():
return
fh = open_file_or_die(filename)
norm_filename = Setup.get_file_reference(filename)
new_content = []
line_n = 0
LinePragmaInfoList = UserCodeFragment_OpenLinePragma[Language]
for line in fh.readlines():
line_n += 1
if Language == "C":
for info in LinePragmaInfoList:
if line.find(info[0]) == -1: continue
line = info[1]
# Since by some definition, line number pragmas < 32768; let us avoid
# compiler warnings by setting line_n = min(line_n, 32768)
line = line.replace("NUMBER", repr(int(min(line_n + 1, 32767))))
# Even under Windows (tm), the '/' is accepted. Thus do not rely on 'normpath'
line = line.replace("FILENAME", norm_filename)
if len(line) == 0 or line[-1] != "\n":
line = line + "\n"
new_content.append(line)
fh.close()
write_safely_and_close(filename, "".join(new_content))
def adorn_with_source_reference(self, Code, ReturnToSourceF=True):
if len(Code.strip()) == 0: return ""
# Even under Windows (tm), the '/' is accepted. Thus do not rely on 'normpath'
norm_filename = Setup.get_file_reference(self.filename)
txt = '\n# line %i "%s"\n' % (self.line_n, norm_filename)
txt += Code
if ReturnToSourceF:
if txt[-1] != "\n": txt = txt + "\n"
txt += get_return_to_source_reference()
return txt
def adorn_with_source_reference(self, Code, ReturnToSourceF=True):
if len(Code.strip()) == 0: return ""
# Even under Windows (tm), the '/' is accepted. Thus do not rely on 'normpath'
norm_filename = Setup.get_file_reference(self.filename)
txt = '\n# line %i "%s"\n' % (self.line_n, norm_filename)
txt += Code
if ReturnToSourceF:
if txt[-1] != "\n": txt = txt + "\n"
txt += get_return_to_source_reference()
return txt
def lexeme_null_implementation():
namespace_open, namespace_close = __namespace_brackets()
return "".join([
"#include \"%s\"\n" %
Setup.get_file_reference(Setup.output_token_class_file),
namespace_open,
"\n",
"%s %s = (%s)0;\n" %
(Setup.buffer_element_type, common_lexeme_null_str(),
Setup.buffer_element_type),
"\n",
namespace_close,
"\n",
])
def _SOURCE_REFERENCE_BEGIN(self, SourceReference):
"""Return a code fragment that returns a source reference pragma. If
the source reference is void, no pragma is required.
"""
if not SourceReference.is_void():
norm_file_name = Setup.get_file_reference(SourceReference.file_name)
line_n = SourceReference.line_n
if line_n <= 0:
line_n = 1
elif line_n >= 2**15:
line_n = 2**15 - 1 # ISO 89: line number <= 32767
return '\n# line %i "%s" /* ISO C89: line number <= 32767 */\n' % (line_n, norm_file_name)
return '\n# line %i "%s"\n' % (line_n, norm_file_name)
else:
return ""
def do_implementation(ModeDB):
FileTemplate = os.path.normpath(QUEX_PATH
+ Lng["$code_base"]
+ "/analyzer/TXT-Cpp.i")
func_txt = get_file_content_or_die(FileTemplate)
func_txt = blue_print(func_txt,
[
["$$CONSTRUCTOR_EXTENSTION$$", Lng.SOURCE_REFERENCED(blackboard.class_constructor_extension)],
["$$CONVERTER_HELPER_I$$", Setup.get_file_reference(Setup.output_buffer_codec_header_i)],
["$$CONSTRUCTOR_MODE_DB_INITIALIZATION_CODE$$", get_constructor_code(ModeDB.values())],
["$$MEMENTO_EXTENSIONS_PACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_pack_extension)],
["$$MEMENTO_EXTENSIONS_UNPACK$$", Lng.SOURCE_REFERENCED(blackboard.memento_unpack_extension)],
])
return func_txt
def _SOURCE_REFERENCE_BEGIN(self, SourceReference):
"""Return a code fragment that returns a source reference pragma. If
the source reference is void, no pragma is required.
"""
if not SourceReference.is_void():
norm_file_name = Setup.get_file_reference(SourceReference.file_name)
line_n = SourceReference.line_n
if line_n <= 0:
line_n = 1
elif line_n >= 2**15:
line_n = 2**15 - 1 # ISO 89: line number <= 32767
return '\n# line %i "%s" /* ISO C89: line number <= 32767 */\n' % (line_n, norm_file_name)
return '\n# line %i "%s"\n' % (line_n, norm_file_name)
else:
return ""
def straighten_open_line_pragmas(self, FileName):
norm_filename = Setup.get_file_reference(FileName)
line_pragma_txt = self._SOURCE_REFERENCE_END().strip()
new_content = []
line_n = 1 # NOT: 0!
fh = open_file_or_die(FileName)
while 1 + 1 == 2:
line = fh.readline()
line_n += 1
if not line:
break
elif line.strip() != line_pragma_txt:
new_content.append(line)
else:
line_n += 1
new_content.append(self._SOURCE_REFERENCE_BEGIN(SourceRef(norm_filename, line_n)))
fh.close()
write_safely_and_close(FileName, "".join(new_content))
def straighten_open_line_pragmas(self, FileName):
norm_filename = Setup.get_file_reference(FileName)
line_pragma_txt = self._SOURCE_REFERENCE_END().strip()
new_content = []
line_n = 1 # NOT: 0!
fh = open_file_or_die(FileName)
while 1 + 1 == 2:
line = fh.readline()
line_n += 1
if not line:
break
elif line.strip() != line_pragma_txt:
new_content.append(line)
else:
line_n += 1
new_content.append(self._SOURCE_REFERENCE_BEGIN(SourceRef(norm_filename, line_n)))
fh.close()
write_safely_and_close(FileName, "".join(new_content))
def _do(UnicodeTrafoInfo):
"""
PURPOSE: Writes converters for conversion towards UTF8/UTF16/UCS2/UCS4.
UnicodeTrafoInfo:
Provides the information about the relation of character codes in a particular
coding to unicode character codes. It is provided in the following form:
# Codec Values Unicode Values
[ (Source0_Begin, Source0_End, TargetInterval0_Begin),
(Source1_Begin, Source1_End, TargetInterval1_Begin),
(Source2_Begin, Source2_End, TargetInterval2_Begin),
...
]
"""
codec_name = make_safe_identifier(UnicodeTrafoInfo.name).lower()
utf8_epilog, utf8_function_body = ConverterWriterUTF8().do(UnicodeTrafoInfo)
utf16_prolog, utf16_function_body = ConverterWriterUTF16().do(UnicodeTrafoInfo)
dummy, utf32_function_body = ConverterWriterUTF32().do(UnicodeTrafoInfo)
# Provide only the constant which are necessary
FileName = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ "/converter_helper/TXT-from-codec-buffer.i")
codec_header = Setup.get_file_reference(Setup.output_buffer_codec_header)
txt_i = blue_print(get_file_content_or_die(FileName),
[["$$CODEC$$", codec_name],
["$$EPILOG$$", utf8_epilog],
["$$CODEC_HEADER$$", codec_header],
["$$BODY_UTF8$$", utf8_function_body],
["$$BODY_UTF16$$", utf16_function_body],
["$$BODY_UTF32$$", utf32_function_body]])
# A separate declaration header is required
FileName = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ "/converter_helper/TXT-from-codec-buffer")
template_h_txt = get_file_content_or_die(FileName)
txt_h = template_h_txt.replace("$$CODEC$$", codec_name)
return txt_h, txt_i
def _do(UnicodeTrafoInfo):
"""
PURPOSE: Writes converters for conversion towards UTF8/UTF16/UCS2/UCS4.
UnicodeTrafoInfo:
Provides the information about the relation of character codes in a particular
coding to unicode character codes. It is provided in the following form:
# Codec Values Unicode Values
[ (Source0_Begin, Source0_End, TargetInterval0_Begin),
(Source1_Begin, Source1_End, TargetInterval1_Begin),
(Source2_Begin, Source2_End, TargetInterval2_Begin),
...
]
"""
codec_name = make_safe_identifier(UnicodeTrafoInfo.name).lower()
utf8_epilog, utf8_function_body = ConverterWriterUTF8().do(
UnicodeTrafoInfo)
utf16_prolog, utf16_function_body = ConverterWriterUTF16().do(
UnicodeTrafoInfo)
dummy, utf32_function_body = ConverterWriterUTF32().do(UnicodeTrafoInfo)
# Provide only the constant which are necessary
FileName = os.path.normpath(QUEX_PATH + Lng["$code_base"] +
"/converter_helper/TXT-from-codec-buffer.i")
codec_header = Setup.get_file_reference(Setup.output_buffer_codec_header)
txt_i = blue_print(get_file_content_or_die(FileName),
[["$$CODEC$$", codec_name], ["$$EPILOG$$", utf8_epilog],
["$$CODEC_HEADER$$", codec_header],
["$$BODY_UTF8$$", utf8_function_body],
["$$BODY_UTF16$$", utf16_function_body],
["$$BODY_UTF32$$", utf32_function_body]])
# A separate declaration header is required
FileName = os.path.normpath(QUEX_PATH + Lng["$code_base"] +
"/converter_helper/TXT-from-codec-buffer")
template_h_txt = get_file_content_or_die(FileName)
txt_h = template_h_txt.replace("$$CODEC$$", codec_name)
return txt_h, txt_i
def get_helper_definitions():
namespace_open, namespace_close = __namespace_brackets(DefineF=True)
token_descr = blackboard.token_type_definition
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
ln_namespace_open = Lng.NAMESPACE_OPEN(Setup.lexeme_null_namespace).replace("\n", "\\\n")
ln_namespace_close = Lng.NAMESPACE_CLOSE(Setup.lexeme_null_namespace).replace("\n", "\\\n")
return helper_definitions \
% (token_descr.class_name,
namespace_open,
namespace_close,
ln_namespace_open,
ln_namespace_close,
token_descr.class_name_safe,
Setup.token_id_type,
Setup.get_file_reference(Setup.output_token_class_file),
token_id_definition_file)
def get_helper_definitions():
namespace_open, namespace_close = __namespace_brackets(DefineF=True)
token_descr = blackboard.token_type_definition
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
ln_namespace_open = Lng.NAMESPACE_OPEN(
Setup.lexeme_null_namespace).replace("\n", "\\\n")
ln_namespace_close = Lng.NAMESPACE_CLOSE(
Setup.lexeme_null_namespace).replace("\n", "\\\n")
return helper_definitions \
% (token_descr.class_name,
namespace_open,
namespace_close,
ln_namespace_open,
ln_namespace_close,
token_descr.class_name_safe,
Setup.token_id_type,
Setup.get_file_reference(Setup.output_token_class_file),
token_id_definition_file)
def _do(Descr):
# The following things must be ensured before the function is called
assert Descr is not None
assert Descr.__class__.__name__ == "TokenTypeDescriptor"
## ALLOW: Descr.get_member_db().keys() == empty
TemplateFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_file"]
TemplateIFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_i_file"]
template_str = open_file_or_die(TemplateFile, Mode="rb").read()
template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read()
virtual_destructor_str = ""
if Descr.open_for_derivation_f: virtual_destructor_str = "virtual "
if Descr.copy is None:
# Default copy operation: Plain Copy of token memory
copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n"
else:
copy_str = Lng.SOURCE_REFERENCED(Descr.copy)
if Descr.take_text is None:
take_text_str = "return true;\n"
else:
take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text)
include_guard_extension_str = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Descr.name_space)
+ "__" + Descr.class_name)
# In case of plain 'C' the class name must incorporate the namespace (list)
token_class_name = Descr.class_name
if Setup.language == "C":
token_class_name = Setup.token_class_name_safe
converter_declaration_include, \
converter_implementation_include, \
converter_string, \
converter_wstring = __get_converter_configuration(include_guard_extension_str)
extra_at_begin_str = lexeme_null_declaration()
extra_at_end_str = ""
if Setup.token_class_only_f:
extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \
+ extra_at_begin_str
extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \
+ extra_at_end_str
namespace_open, namespace_close = __namespace_brackets()
helper_variable_replacements = [
["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include],
["$INCLUDE_CONVERTER_IMPLEMENTATION", converter_implementation_include],
["$CONVERTER_STRING", converter_string],
["$CONVERTER_WSTRING", converter_wstring],
["$NAMESPACE_CLOSE", namespace_close],
["$NAMESPACE_OPEN", namespace_open],
["$TOKEN_CLASS", token_class_name],
]
txt = blue_print(template_str,
[
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt = blue_print(txt,
[
["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)],
["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$DISTINCT_MEMBERS$$", get_distinct_members(Descr)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$QUICK_SETTERS$$", get_quick_setters(Descr)],
["$$SETTERS_GETTERS$$", get_setter_getter(Descr)],
["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)],
["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)],
["$$UNION_MEMBERS$$", get_union_members(Descr)],
["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt = blue_print(txt, helper_variable_replacements)
if Setup.language.upper() != "C++" and Setup.token_class_only_f:
extra_at_begin_str += local_strlen_str % (Descr.class_name_safe, Setup.buffer_element_type, Setup.buffer_element_type)
txt_i = blue_print(template_i_str,
[
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt_i = blue_print(txt_i,
[
["$$CONSTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$", Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$TOKEN_CLASS_HEADER$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_OPEN$$", Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$NAMESPACE_CLOSE$$", Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$TOKEN_REPETITION_N_GET$$", Lng.SOURCE_REFERENCED(Descr.repetition_get)],
["$$TOKEN_REPETITION_N_SET$$", Lng.SOURCE_REFERENCED(Descr.repetition_set)],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt_i = blue_print(txt_i, helper_variable_replacements)
return txt, txt_i
def do(ModeDB):
assert blackboard.token_type_definition is not None
QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ Lng["$analyzer_template_file"]).replace("//","/")
LexerClassName = Setup.analyzer_class_name
quex_converter_coding_name_str = Setup.converter_ucs_coding_name
mode_id_definition_str = ""
# NOTE: First mode-id needs to be '1' for compatibility with flex generated engines
for i, info in enumerate(ModeDB.items()):
name = info[0]
mode = info[1]
if mode.abstract_f(): continue
mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i)
if mode_id_definition_str != "":
mode_id_definition_str = mode_id_definition_str[:-2]
# -- instances of mode classes as members of the lexer
mode_object_members_txt, \
mode_specific_functions_txt, \
friend_txt = get_mode_class_related_code_fragments(ModeDB.values())
# -- define a pointer that directly has the type of the derived class
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
derived_class_type_declaration = ""
token_class_file_name = blackboard.token_type_definition.get_file_name()
token_class_name = blackboard.token_type_definition.class_name
token_class_name_safe = blackboard.token_type_definition.class_name_safe
template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate)
include_guard_ext = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)
+ "__" + Setup.analyzer_class_name)
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space))
txt = blue_print(template_code_txt,
[
["$$___SPACE___$$", " " * (len(LexerClassName) + 1)],
["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)],
["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_NAME_SPACE$$", lexer_name_space_safe],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)],
["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str],
["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)],
["$$MODE_CLASS_FRIENDS$$", friend_txt],
["$$MODE_OBJECTS$$", mode_object_members_txt],
["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt],
["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)],
["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)],
["$$TOKEN_CLASS$$", token_class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe],
["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)],
["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str],
["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"],
])
return txt
def do(ModeDB):
assert blackboard.token_type_definition is not None
QuexClassHeaderFileTemplate = os.path.normpath( QUEX_PATH
+ Lng["$code_base"]
+ Lng["$analyzer_template_file"]).replace("//","/")
LexerClassName = Setup.analyzer_class_name
quex_converter_coding_name_str = Setup.converter_ucs_coding_name
mode_id_definition_str = ""
# NOTE: First mode-id needs to be '1' for compatibility with flex generated engines
for i, info in enumerate(ModeDB.items()):
name = info[0]
mode = info[1]
if mode.abstract_f(): continue
mode_id_definition_str += " QUEX_NAME(ModeID_%s) = %i,\n" % (name, i)
if mode_id_definition_str != "":
mode_id_definition_str = mode_id_definition_str[:-2]
# -- instances of mode classes as members of the lexer
mode_object_members_txt, \
mode_specific_functions_txt, \
friend_txt = get_mode_class_related_code_fragments(ModeDB.values())
# -- define a pointer that directly has the type of the derived class
if Setup.analyzer_derived_class_name != "":
analyzer_derived_class_name = Setup.analyzer_derived_class_name
derived_class_type_declaration = "class %s;" % Setup.analyzer_derived_class_name
else:
analyzer_derived_class_name = Setup.analyzer_class_name
derived_class_type_declaration = ""
token_class_file_name = blackboard.token_type_definition.get_file_name()
token_class_name = blackboard.token_type_definition.class_name
token_class_name_safe = blackboard.token_type_definition.class_name_safe
template_code_txt = get_file_content_or_die(QuexClassHeaderFileTemplate)
include_guard_ext = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space)
+ "__" + Setup.analyzer_class_name)
if len(Setup.token_id_foreign_definition_file) != 0:
token_id_definition_file = Setup.token_id_foreign_definition_file
else:
token_id_definition_file = Setup.output_token_id_file
lexer_name_space_safe = get_include_guard_extension(Lng.NAMESPACE_REFERENCE(Setup.analyzer_name_space))
txt = blue_print(template_code_txt,
[
["$$___SPACE___$$", " " * (len(LexerClassName) + 1)],
["$$CLASS_BODY_EXTENSION$$", Lng.SOURCE_REFERENCED(blackboard.class_body_extension)],
["$$CONVERTER_HELPER$$", Setup.get_file_reference(Setup.output_buffer_codec_header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_ext],
["$$LEXER_CLASS_NAME$$", LexerClassName],
["$$LEXER_NAME_SPACE$$", lexer_name_space_safe],
["$$LEXER_CLASS_NAME_SAFE$$", Setup.analyzer_name_safe],
["$$LEXER_CONFIG_FILE$$", Setup.get_file_reference(Setup.output_configuration_file)],
["$$LEXER_DERIVED_CLASS_DECL$$", derived_class_type_declaration],
["$$LEXER_DERIVED_CLASS_NAME$$", analyzer_derived_class_name],
["$$QUEX_MODE_ID_DEFINITIONS$$", mode_id_definition_str],
["$$MEMENTO_EXTENSIONS$$", Lng.SOURCE_REFERENCED(blackboard.memento_class_extension)],
["$$MODE_CLASS_FRIENDS$$", friend_txt],
["$$MODE_OBJECTS$$", mode_object_members_txt],
["$$MODE_SPECIFIC_ANALYSER_FUNCTIONS$$", mode_specific_functions_txt],
["$$PRETTY_INDENTATION$$", " " + " " * (len(LexerClassName)*2 + 2)],
["$$QUEX_TEMPLATE_DIR$$", QUEX_PATH + Lng["$code_base"]],
["$$QUEX_VERSION$$", QUEX_VERSION],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(token_class_file_name)],
["$$TOKEN_CLASS$$", token_class_name],
["$$TOKEN_CLASS_NAME_SAFE$$", token_class_name_safe],
["$$TOKEN_ID_DEFINITION_FILE$$", Setup.get_file_reference(token_id_definition_file)],
["$$CORE_ENGINE_CHARACTER_CODING$$", quex_converter_coding_name_str],
["$$USER_DEFINED_HEADER$$", Lng.SOURCE_REFERENCED(blackboard.header) + "\n"],
])
return txt
def ENGINE_TEXT_EPILOG(self):
if Setup.analyzer_derived_class_file:
header = "<" + Setup.get_file_reference(Setup.analyzer_derived_class_file) +">\n"
else:
header = "\"" + Setup.get_file_reference(Setup.output_header_file) +"\"\n"
return cpp_include_Multi_i_str.replace("$$HEADER$$", header)
def _do(Descr):
# The following things must be ensured before the function is called
assert Descr is not None
assert Descr.__class__.__name__ == "TokenTypeDescriptor"
## ALLOW: Descr.get_member_db().keys() == empty
TemplateFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_file"]
TemplateIFile = QUEX_PATH \
+ Lng["$code_base"] \
+ Lng["$token_template_i_file"]
template_str = open_file_or_die(TemplateFile, Mode="rb").read()
template_i_str = open_file_or_die(TemplateIFile, Mode="rb").read()
virtual_destructor_str = ""
if Descr.open_for_derivation_f: virtual_destructor_str = "virtual "
if Descr.copy is None:
# Default copy operation: Plain Copy of token memory
copy_str = "__QUEX_STD_memcpy((void*)__this, (void*)__That, sizeof(QUEX_TYPE_TOKEN));\n"
else:
copy_str = Lng.SOURCE_REFERENCED(Descr.copy)
if Descr.take_text is None:
take_text_str = "return true;\n"
else:
take_text_str = Lng.SOURCE_REFERENCED(Descr.take_text)
include_guard_extension_str = get_include_guard_extension(
Lng.NAMESPACE_REFERENCE(Descr.name_space) + "__" + Descr.class_name)
# In case of plain 'C' the class name must incorporate the namespace (list)
token_class_name = Descr.class_name
if Setup.language == "C":
token_class_name = Setup.token_class_name_safe
converter_declaration_include, \
converter_implementation_include, \
converter_string, \
converter_wstring = __get_converter_configuration(include_guard_extension_str)
extra_at_begin_str = lexeme_null_declaration()
extra_at_end_str = ""
if Setup.token_class_only_f:
extra_at_begin_str = QUEX_NAME_TOKEN_define_str % include_guard_extension_str \
+ extra_at_begin_str
extra_at_end_str = QUEX_NAME_TOKEN_undef_str % include_guard_extension_str \
+ extra_at_end_str
namespace_open, namespace_close = __namespace_brackets()
helper_variable_replacements = [
["$INCLUDE_CONVERTER_DECLARATION", converter_declaration_include],
[
"$INCLUDE_CONVERTER_IMPLEMENTATION",
converter_implementation_include
],
["$CONVERTER_STRING", converter_string],
["$CONVERTER_WSTRING", converter_wstring],
["$NAMESPACE_CLOSE", namespace_close],
["$NAMESPACE_OPEN", namespace_open],
["$TOKEN_CLASS", token_class_name],
]
txt = blue_print(template_str, [
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt = blue_print(txt, [
["$$BODY$$", Lng.SOURCE_REFERENCED(Descr.body)],
["$$CONSTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$DISTINCT_MEMBERS$$",
get_distinct_members(Descr)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
["$$HEADER$$", Lng.SOURCE_REFERENCED(Descr.header)],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_CLOSE$$",
Lng.NAMESPACE_CLOSE(Descr.name_space)],
["$$NAMESPACE_OPEN$$",
Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$QUICK_SETTERS$$", get_quick_setters(Descr)],
["$$SETTERS_GETTERS$$",
get_setter_getter(Descr)],
[
"$$TOKEN_REPETITION_N_GET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_get)
],
[
"$$TOKEN_REPETITION_N_SET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_set)
],
["$$UNION_MEMBERS$$", get_union_members(Descr)],
["$$VIRTUAL_DESTRUCTOR$$", virtual_destructor_str],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt = blue_print(txt, helper_variable_replacements)
if Setup.language.upper() != "C++" and Setup.token_class_only_f:
extra_at_begin_str += local_strlen_str % (Descr.class_name_safe,
Setup.buffer_element_type,
Setup.buffer_element_type)
txt_i = blue_print(template_i_str, [
["$$EXTRA_AT_BEGIN$$", extra_at_begin_str],
["$$EXTRA_AT_END$$", extra_at_end_str],
])
txt_i = blue_print(txt_i, [
["$$CONSTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.constructor)],
["$$COPY$$", copy_str],
["$$DESTRUCTOR$$",
Lng.SOURCE_REFERENCED(Descr.destructor)],
["$$FOOTER$$", Lng.SOURCE_REFERENCED(Descr.footer)],
["$$FUNC_TAKE_TEXT$$", take_text_str],
[
"$$TOKEN_CLASS_HEADER$$",
Setup.get_file_reference(
blackboard.token_type_definition.get_file_name())
],
["$$INCLUDE_GUARD_EXTENSION$$", include_guard_extension_str],
["$$NAMESPACE_OPEN$$",
Lng.NAMESPACE_OPEN(Descr.name_space)],
["$$NAMESPACE_CLOSE$$",
Lng.NAMESPACE_CLOSE(Descr.name_space)],
[
"$$TOKEN_REPETITION_N_GET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_get)
],
[
"$$TOKEN_REPETITION_N_SET$$",
Lng.SOURCE_REFERENCED(Descr.repetition_set)
],
["$$TOKEN_CLASS_NAME_SAFE$$", Descr.class_name_safe],
])
txt_i = blue_print(txt_i, helper_variable_replacements)
return txt, txt_i
def __get_converter_configuration(IncludeGuardExtension):
token_descr = blackboard.token_type_definition
if not Setup.converter_helper_required_f:
declaration_include = "#include <quex/code_base/converter_helper/identity>"
implementation_include = "#include <quex/code_base/converter_helper/identity.i>"
from_codec = "identical"
elif Setup.buffer_codec.name in ["utf8", "utf16", "utf32"]:
declaration_include = "#include <quex/code_base/converter_helper/from-%s>" % Setup.buffer_codec.name
implementation_include = "#include <quex/code_base/converter_helper/from-%s.i>" % Setup.buffer_codec.name
from_codec = Setup.buffer_codec.name
elif Setup.buffer_codec.name == "unicode":
declaration_include = "#include <quex/code_base/converter_helper/from-unicode-buffer>"
implementation_include = "#include <quex/code_base/converter_helper/from-unicode-buffer.i>"
from_codec = Setup.buffer_codec.name
else:
declaration_include = "#include \"%s\"" % \
Setup.get_file_reference(Setup.output_buffer_codec_header)
implementation_include = "#include \"%s\"" % \
Setup.get_file_reference(Setup.output_buffer_codec_header_i)
from_codec = Setup.buffer_codec.name
if not Setup.token_class_only_f:
string = "QUEX_CONVERTER_STRING(%s,char)" % from_codec
wstring = "QUEX_CONVERTER_STRING(%s,wchar)" % from_codec
return declaration_include, implementation_include, string, wstring
# From Here One: 'Sharable Token Class Generation'
if Setup.language.upper() == "C++":
function_prefix = Lng.NAMESPACE_REFERENCE(token_descr.name_space)
function_def_prefix = ""
function_def_prefix_0 = ""
namespace_token_open = Lng.NAMESPACE_OPEN(
token_descr.name_space).replace("\n", " ")
namespace_token_close = Lng.NAMESPACE_CLOSE(
token_descr.name_space).replace("\n", " ")
else:
function_prefix = token_descr.class_name_safe + " ##"
function_def_prefix = token_descr.class_name_safe + " ##"
function_def_prefix_0 = token_descr.class_name_safe
namespace_token_open = ""
namespace_token_close = ""
before = frame_begin \
% (IncludeGuardExtension, \
function_def_prefix, function_def_prefix, \
function_prefix, function_prefix, \
namespace_token_open, namespace_token_close,\
Setup.buffer_element_type)
after = frame_end \
% IncludeGuardExtension
declaration_include = "%s%s\n%s" \
% (before, declaration_include, after)
implementation_include = "%s%s\n%s" \
% (before, implementation_include, after)
# In C: Function call and def prefix is the same
# In C++: We are in the same namespace, no prefix, function_def_prefix is empty anyway.
string = "%s%s_to_char" % (function_def_prefix_0, from_codec)
wstring = "%s%s_to_wchar" % (function_def_prefix_0, from_codec)
return declaration_include, implementation_include, string, wstring
def do(setup):
"""Creates a file of token-ids from a given set of names.
Creates also a function:
const string& $$token$$::map_id_to_name().
"""
global file_str
LanguageDB = Setup.language_db
__propose_implicit_token_definitions()
for standard_token_id in standard_token_id_list:
assert token_id_db.has_key(standard_token_id)
assert blackboard.token_type_definition is not None, \
"Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \
"handle this."
# (*) Token ID File ________________________________________________________________
#
# The token id file can either be specified as database of
# token-id names, or as a file that directly assigns the token-ids
# to variables. If the flag '--user-token-id-file' is defined, then
# then the token-id file is provided by the user. Otherwise, the
# token id file is created by the token-id maker.
#
# The token id maker considers the file passed by the option '-t'
# as the database file and creates a C++ file with the output filestem
# plus the suffix "--token-ids". Note, that the token id file is a
# header file.
#
if len(token_id_db.keys()) == len(standard_token_id_list):
token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \
(setup.token_id_prefix_plain, setup.token_id_prefix_plain)
# TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done.
error_msg("Only token ids %s are defined.\n" % token_id_str + \
"Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \
"specify at least one other token id.")
#______________________________________________________________________________________
L = max(map(lambda name: len(name), token_id_db.keys()))
def space(Name):
return " " * (L - len(Name))
# -- define values for the token ids
def define_this(txt, token):
if setup.language == "C":
txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
else:
txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
if setup.token_id_foreign_definition_file != "":
token_id_txt = [
"#include \"%s\"\n" %
Setup.get_file_reference(setup.token_id_foreign_definition_file)
]
else:
if setup.language == "C":
prolog = ""
epilog = ""
else:
prolog = LanguageDB.NAMESPACE_OPEN(
setup.token_id_prefix_name_space)
epilog = LanguageDB.NAMESPACE_CLOSE(
setup.token_id_prefix_name_space)
token_id_txt = [prolog]
# Assign values to tokens with no numeric identifier
# NOTE: This has not to happen if token's are defined by the user's provided file.
i = setup.token_id_counter_offset
# Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name'
# may contain '--' to indicate a unicode value, so do not use it as name.
for dummy_name, token in sorted(token_id_db.items()):
if token.number is None:
while __is_token_id_occupied(i):
i += 1
token.number = i
define_this(token_id_txt, token)
# Double check that no token id appears twice
# Again, this can only happen, if quex itself produced the numeric values for the token
token_list = token_id_db.values()
for i, x in enumerate(token_list):
for y in token_list[i + 1:]:
if x.number != y.number: continue
error_msg("Token id '%s'" % x.name,
x.file_name,
x.line_n,
DontExitF=True)
error_msg("and token id '%s' have same numeric value '%s'." \
% (y.name, x.number), y.file_name, y.line_n, DontExitF=True)
token_id_txt.append(epilog)
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \
Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())],
])
return content
def __get_converter_configuration(IncludeGuardExtension):
token_descr = blackboard.token_type_definition
if not Setup.converter_helper_required_f:
declaration_include = "#include <quex/code_base/converter_helper/identity>"
implementation_include = "#include <quex/code_base/converter_helper/identity.i>"
from_codec = "identical"
elif Setup.buffer_codec.name in ["utf8", "utf16", "utf32"]:
declaration_include = "#include <quex/code_base/converter_helper/from-%s>" % Setup.buffer_codec.name
implementation_include = "#include <quex/code_base/converter_helper/from-%s.i>" % Setup.buffer_codec.name
from_codec = Setup.buffer_codec.name
elif Setup.buffer_codec.name == "unicode":
declaration_include = "#include <quex/code_base/converter_helper/from-unicode-buffer>"
implementation_include = "#include <quex/code_base/converter_helper/from-unicode-buffer.i>"
from_codec = Setup.buffer_codec.name
else:
declaration_include = "#include \"%s\"" % \
Setup.get_file_reference(Setup.output_buffer_codec_header)
implementation_include = "#include \"%s\"" % \
Setup.get_file_reference(Setup.output_buffer_codec_header_i)
from_codec = Setup.buffer_codec.name
if not Setup.token_class_only_f:
string = "QUEX_CONVERTER_STRING(%s,char)" % from_codec
wstring = "QUEX_CONVERTER_STRING(%s,wchar)" % from_codec
return declaration_include, implementation_include, string, wstring
# From Here One: 'Sharable Token Class Generation'
if Setup.language.upper() == "C++":
function_prefix = Lng.NAMESPACE_REFERENCE(token_descr.name_space)
function_def_prefix = ""
function_def_prefix_0 = ""
namespace_token_open = Lng.NAMESPACE_OPEN(token_descr.name_space).replace("\n", " ")
namespace_token_close = Lng.NAMESPACE_CLOSE(token_descr.name_space).replace("\n", " ")
else:
function_prefix = token_descr.class_name_safe + " ##"
function_def_prefix = token_descr.class_name_safe + " ##"
function_def_prefix_0 = token_descr.class_name_safe
namespace_token_open = ""
namespace_token_close = ""
before = frame_begin \
% (IncludeGuardExtension, \
function_def_prefix, function_def_prefix, \
function_prefix, function_prefix, \
namespace_token_open, namespace_token_close,\
Setup.buffer_element_type)
after = frame_end \
% IncludeGuardExtension
declaration_include = "%s%s\n%s" \
% (before, declaration_include, after)
implementation_include = "%s%s\n%s" \
% (before, implementation_include, after)
# In C: Function call and def prefix is the same
# In C++: We are in the same namespace, no prefix, function_def_prefix is empty anyway.
string = "%s%s_to_char" % (function_def_prefix_0, from_codec)
wstring = "%s%s_to_wchar" % (function_def_prefix_0, from_codec)
return declaration_include, implementation_include, string, wstring
def do(setup):
"""Creates a file of token-ids from a given set of names.
Creates also a function:
const string& $$token$$::map_id_to_name().
"""
global file_str
LanguageDB = Setup.language_db
__propose_implicit_token_definitions()
for standard_token_id in standard_token_id_list:
assert token_id_db.has_key(standard_token_id)
assert blackboard.token_type_definition is not None, \
"Token type has not been defined yet, see $QUEX_PATH/quex/core.py how to\n" + \
"handle this."
# (*) Token ID File ________________________________________________________________
#
# The token id file can either be specified as database of
# token-id names, or as a file that directly assigns the token-ids
# to variables. If the flag '--user-token-id-file' is defined, then
# then the token-id file is provided by the user. Otherwise, the
# token id file is created by the token-id maker.
#
# The token id maker considers the file passed by the option '-t'
# as the database file and creates a C++ file with the output filestem
# plus the suffix "--token-ids". Note, that the token id file is a
# header file.
#
if len(token_id_db.keys()) == len(standard_token_id_list):
token_id_str = "%sTERMINATION and %sUNINITIALIZED" % \
(setup.token_id_prefix_plain, setup.token_id_prefix_plain)
# TERMINATION + UNINITIALIZED = 2 token ids. If they are the only ones nothing can be done.
error_msg("Only token ids %s are defined.\n" % token_id_str + \
"Quex refuses to proceed. Please, use the 'token { ... }' section to\n" + \
"specify at least one other token id.")
#______________________________________________________________________________________
L = max(map(lambda name: len(name), token_id_db.keys()))
def space(Name):
return " " * (L - len(Name))
# -- define values for the token ids
def define_this(txt, token):
if setup.language == "C":
txt.append("#define %s%s %s((QUEX_TYPE_TOKEN_ID)%i)\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
else:
txt.append("const QUEX_TYPE_TOKEN_ID %s%s%s = ((QUEX_TYPE_TOKEN_ID)%i);\n" \
% (setup.token_id_prefix_plain, token.name, space(token.name), token.number))
if setup.token_id_foreign_definition_file != "":
token_id_txt = ["#include \"%s\"\n" % Setup.get_file_reference(setup.token_id_foreign_definition_file)]
else:
if setup.language == "C":
prolog = ""
epilog = ""
else:
prolog = LanguageDB.NAMESPACE_OPEN(setup.token_id_prefix_name_space)
epilog = LanguageDB.NAMESPACE_CLOSE(setup.token_id_prefix_name_space)
token_id_txt = [prolog]
# Assign values to tokens with no numeric identifier
# NOTE: This has not to happen if token's are defined by the user's provided file.
i = setup.token_id_counter_offset
# Take the 'dummy_name' only to have the list sorted by name. The key 'dummy_name'
# may contain '--' to indicate a unicode value, so do not use it as name.
for dummy_name, token in sorted(token_id_db.items()):
if token.number is None:
while __is_token_id_occupied(i):
i += 1
token.number = i;
define_this(token_id_txt, token)
# Double check that no token id appears twice
# Again, this can only happen, if quex itself produced the numeric values for the token
token_list = token_id_db.values()
for i, x in enumerate(token_list):
for y in token_list[i+1:]:
if x.number != y.number: continue
error_msg("Token id '%s'" % x.name, x.file_name, x.line_n, DontExitF=True)
error_msg("and token id '%s' have same numeric value '%s'." \
% (y.name, x.number), y.file_name, y.line_n, DontExitF=True)
token_id_txt.append(epilog)
content = blue_print(file_str,
[["$$TOKEN_ID_DEFINITIONS$$", "".join(token_id_txt)],
["$$DATE$$", time.asctime()],
["$$TOKEN_CLASS_DEFINITION_FILE$$", Setup.get_file_reference(blackboard.token_type_definition.get_file_name())],
["$$TOKEN_PREFIX$$", setup.token_id_prefix],
["$$INCLUDE_GUARD_EXT$$", get_include_guard_extension( \
Setup.analyzer_name_safe.upper() \
+ "__" \
+ Setup.token_class_name_safe.upper())],
])
return content