def backward_lexing(OriginList):
"""Backward Lexing:
-- Using an inverse state machine from 'real' current start position backwards
until a drop out occurs.
-- During backward lexing, there is no 'winner' so all origins that indicate
acceptance need to be considered. They raise there flag 'pre-condition fulfilled'.
"""
LanguageDB = Setup.language_db
# There should be nothing, but unconditional acceptances or no-acceptance
# origins in the list of origins.
inadmissible_origin_list = filter(lambda origin:
origin.pre_context_begin_of_line_f() or
origin.pre_context_id() != -1L or
origin.post_context_id() != -1L,
OriginList)
assert inadmissible_origin_list == [], \
"Inadmissible origins for inverse state machine."
#___________________________________________________________________________________________
txt = []
for origin in OriginList:
if not origin.store_input_position_f(): continue
assert origin.is_acceptance()
variable = "pre_context_%s_fulfilled_f" % __nice(origin.state_machine_id)
txt.append(" " + LanguageDB["$assignment"](variable, 1))
txt.append(" __quex_debug(\"pre context %s fulfilled\");\n" % __nice(origin.state_machine_id));
txt.append("\n")
return txt
def backward_lexing(OriginList):
"""Backward Lexing:
-- Using an inverse state machine from 'real' current start position backwards
until a drop out occurs.
-- During backward lexing, there is no 'winner' so all origins that indicate
acceptance need to be considered. They raise there flag 'pre-condition fulfilled'.
"""
LanguageDB = Setup.language_db
# There should be nothing, but unconditional acceptances or no-acceptance
# origins in the list of origins.
inadmissible_origin_list = filter(
lambda origin: origin.pre_context_begin_of_line_f() or origin.
pre_context_id() != -1L or origin.post_context_id() != -1L, OriginList)
assert inadmissible_origin_list == [], \
"Inadmissible origins for inverse state machine."
#___________________________________________________________________________________________
txt = []
for origin in OriginList:
if not origin.store_input_position_f(): continue
assert origin.is_acceptance()
variable = "pre_context_%s_fulfilled_f" % __nice(
origin.state_machine_id)
txt.append(" " + LanguageDB["$assignment"](variable, 1))
txt.append(" __quex_debug(\"pre context %s fulfilled\");\n" %
__nice(origin.state_machine_id))
txt.append("\n")
return txt
def __handle_post_conditioned_core_patterns(OriginList, SMD):
"""Creates code to store the input position for post conditioned
core patterns. That means, that the input pointer must return
to this place in case that the post condition is fulfilled.
"""
global LanguageDB
# (1) Set the post context registers (if appropriate)
# (also determine the list of acceptance origins)
txtl = []
for origin in OriginList:
if origin.is_end_of_post_contexted_core_pattern():
# Assumption about origin based on assumption about single pattern state machine:
#
# According to 'setup_post_context.py' (currently line 121) no acceptance
# state of a post context can store the input position. This has been made
# impossible! Otherwise, the post context would have zero length.
#
# Post-conditions via backward search, i.e. pseudo ambigous post conditions,
# are a different ball-game.
#
assert origin.is_acceptance() == False
# Store current input position, to be restored when post condition really matches
post_context_index = SMD.get_post_context_index(
origin.state_machine_id)
txtl += [
" __quex_debug(\"post context %s: mark position\");\n" % __nice(origin.state_machine_id),
" " + LanguageDB["$input/tell_position"]("post_context_start_position[%i]" % \
post_context_index),
"\n"
]
return txtl
def __handle_post_conditioned_core_patterns(OriginList, SMD):
"""Creates code to store the input position for post conditioned
core patterns. That means, that the input pointer must return
to this place in case that the post condition is fulfilled.
"""
global LanguageDB
# (1) Set the post context registers (if appropriate)
# (also determine the list of acceptance origins)
txtl = []
for origin in OriginList:
if origin.is_end_of_post_contexted_core_pattern():
# Assumption about origin based on assumption about single pattern state machine:
#
# According to 'setup_post_context.py' (currently line 121) no acceptance
# state of a post context can store the input position. This has been made
# impossible! Otherwise, the post context would have zero length.
#
# Post-conditions via backward search, i.e. pseudo ambigous post conditions,
# are a different ball-game.
#
assert origin.is_acceptance() == False
# Store current input position, to be restored when post condition really matches
post_context_index = SMD.get_post_context_index(origin.state_machine_id)
txtl += [
" __quex_debug(\"post context %s: mark position\");\n" % __nice(origin.state_machine_id),
" " + LanguageDB["$input/tell_position"]("post_context_start_position[%i]" % \
post_context_index),
"\n"
]
return txtl
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, \
delimiter_length_str, \
delimiter_comment_str \
= get_character_sequence(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](" Delimiter: "
+ delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i-1) + "\n"
txt += " " + LanguageDB["$if !="]("Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " goto %s;" % get_label("$entry", skipper_index, U=True)
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label("$terminal-direct", IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(template_str,
[["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index, U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
])
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)],
["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]])
if reference_p_f:
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = \
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
return code_str, local_variable_db
def get_skipper(EndSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert type(EndSequence) == list
assert len(EndSequence) >= 1
assert map(type, EndSequence) == [int] * len(EndSequence)
local_variable_db = {}
global template_str
LanguageDB = Setup.language_db
# Name the $$SKIPPER$$
skipper_index = sm_index.get()
# Determine the $$DELIMITER$$
delimiter_str, delimiter_length_str, delimiter_comment_str = get_character_sequence(EndSequence)
delimiter_comment_str = LanguageDB["$comment"](" Delimiter: " + delimiter_comment_str)
# Determine the check for the tail of the delimiter
delimiter_remainder_test_str = ""
if len(EndSequence) != 1:
txt = ""
i = 0
for letter in EndSequence[1:]:
i += 1
txt += " " + LanguageDB["$input/get-offset"](i - 1) + "\n"
txt += " " + LanguageDB["$if !="]("Skipper$$SKIPPER_INDEX$$[%i]" % i)
txt += " goto %s;" % get_label("$entry", skipper_index, U=True)
txt += " " + LanguageDB["$endif"]
delimiter_remainder_test_str = txt
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, EndSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label(
"$terminal-direct", IndentationCounterTerminalID, U=True
)
if OnSkipRangeOpenStr != "":
on_skip_range_open_str = OnSkipRangeOpenStr
else:
on_skip_range_open_str = get_on_skip_range_open(Mode, EndSequence)
# The main part
code_str = blue_print(
template_str,
[
["$$DELIMITER$$", delimiter_str],
["$$DELIMITER_LENGTH$$", delimiter_length_str],
["$$DELIMITER_COMMENT$$", delimiter_comment_str],
["$$WHILE_1_PLUS_1_EQUAL_2$$", LanguageDB["$loop-start-endless"]],
["$$END_WHILE$$", LanguageDB["$loop-end"]],
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index, U=True)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$DELIMITER_REMAINDER_TEST$$", delimiter_remainder_test_str],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
],
)
# Line and column number counting
code_str, reference_p_f = __lc_counting_replacements(code_str, EndSequence)
# The finishing touch
code_str = blue_print(
code_str,
[["$$SKIPPER_INDEX$$", __nice(skipper_index)], ["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)]],
)
if reference_p_f:
local_variable_db["QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p"] = Variable(
"reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING",
)
return code_str, local_variable_db
def get_skipper(OpenerSequence, CloserSequence, Mode=None, IndentationCounterTerminalID=None, OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_length_str, opener_comment_str = get_character_sequence(OpenerSequence)
closer_str, closer_length_str, closer_comment_str = get_character_sequence(CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(Mode, CloserSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start", U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label("$terminal-direct",
IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {
"counter": Variable("counter", "size_t", None, "0"),
"QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p":
Variable("reference_p",
"QUEX_TYPE_CHARACTER_POSITION",
None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
}
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER$$", opener_str],
["$$OPENER_LENGTH$$", opener_length_str],
["$$OPENER_COMMENT$$", opener_comment_str],
["$$CLOSER$$", closer_str],
["$$CLOSER_LENGTH$$", closer_length_str],
["$$CLOSER_COMMENT$$", closer_comment_str],
#
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
["$$IF_INPUT_EQUAL_DELIMITER_0$$", LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$", get_label("$reload", skipper_index)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$", get_label("$entry", skipper_index)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
def get_skipper(OpenerSequence,
CloserSequence,
Mode=None,
IndentationCounterTerminalID=None,
OnSkipRangeOpenStr=""):
assert OpenerSequence.__class__ == list
assert len(OpenerSequence) >= 1
assert map(type, OpenerSequence) == [int] * len(OpenerSequence)
assert CloserSequence.__class__ == list
assert len(CloserSequence) >= 1
assert map(type, CloserSequence) == [int] * len(CloserSequence)
assert OpenerSequence != CloserSequence
LanguageDB = Setup.language_db
skipper_index = sm_index.get()
opener_str, opener_length_str, opener_comment_str = get_character_sequence(
OpenerSequence)
closer_str, closer_length_str, closer_comment_str = get_character_sequence(
CloserSequence)
if not end_delimiter_is_subset_of_indentation_counter_newline(
Mode, CloserSequence):
goto_after_end_of_skipping_str = "goto %s;" % get_label("$start",
U=True)
else:
# If there is indentation counting involved, then the counter's terminal id must
# be determined at this place.
assert IndentationCounterTerminalID != None
# If the ending delimiter is a subset of what the 'newline' pattern triggers
# in indentation counting => move on to the indentation counter.
goto_after_end_of_skipping_str = "goto %s;" % get_label(
"$terminal-direct", IndentationCounterTerminalID, U=True)
if OnSkipRangeOpenStr != "": on_skip_range_open_str = OnSkipRangeOpenStr
else: on_skip_range_open_str = get_on_skip_range_open(Mode, CloserSequence)
local_variable_db = {
"counter":
Variable("counter", "size_t", None, "0"),
"QUEX_OPTION_COLUMN_NUMBER_COUNTING/reference_p":
Variable("reference_p", "QUEX_TYPE_CHARACTER_POSITION", None,
"(QUEX_TYPE_CHARACTER_POSITION)0x0",
"QUEX_OPTION_COLUMN_NUMBER_COUNTING")
}
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
reference_p_def = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
before_reload = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
after_reload = " __QUEX_IF_COUNT_COLUMNS(reference_p = QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer));\n"
if CloserSequence[-1] == ord('\n'):
end_procedure = " __QUEX_IF_COUNT_LINES_ADD((size_t)1);\n"
end_procedure += " __QUEX_IF_COUNT_COLUMNS_SET((size_t)1);\n"
else:
end_procedure = " __QUEX_IF_COUNT_COLUMNS_ADD((size_t)(QUEX_NAME(Buffer_tell_memory_adr)(&me->buffer)\n" + \
" - reference_p));\n"
code_str = blue_print(
template_str,
[
["$$SKIPPER_INDEX$$", __nice(skipper_index)],
#
["$$OPENER$$", opener_str],
["$$OPENER_LENGTH$$", opener_length_str],
["$$OPENER_COMMENT$$", opener_comment_str],
["$$CLOSER$$", closer_str],
["$$CLOSER_LENGTH$$", closer_length_str],
["$$CLOSER_COMMENT$$", closer_comment_str],
#
["$$INPUT_P_INCREMENT$$", LanguageDB["$input/increment"]],
["$$INPUT_P_DECREMENT$$", LanguageDB["$input/decrement"]],
["$$INPUT_GET$$", LanguageDB["$input/get"]],
[
"$$IF_INPUT_EQUAL_DELIMITER_0$$",
LanguageDB["$if =="]("Skipper$$SKIPPER_INDEX$$[0]")
],
["$$ENDIF$$", LanguageDB["$endif"]],
["$$ENTRY$$", get_label("$entry", skipper_index)],
["$$RELOAD$$", get_label("$reload", skipper_index)],
["$$GOTO_AFTER_END_OF_SKIPPING$$", goto_after_end_of_skipping_str],
["$$GOTO_RELOAD$$",
get_label("$reload", skipper_index)],
# When things were skipped, no change to acceptance flags or modes has
# happend. One can jump immediately to the start without re-entry preparation.
["$$GOTO_ENTRY$$",
get_label("$entry", skipper_index)],
["$$MARK_LEXEME_START$$", LanguageDB["$mark-lexeme-start"]],
["$$ON_SKIP_RANGE_OPEN$$", on_skip_range_open_str],
#
["$$LC_COUNT_COLUMN_N_POINTER_DEFINITION$$", reference_p_def],
["$$LC_COUNT_IN_LOOP$$", line_column_counter_in_loop],
["$$LC_COUNT_END_PROCEDURE$$", end_procedure],
["$$LC_COUNT_BEFORE_RELOAD$$", before_reload],
["$$LC_COUNT_AFTER_RELOAD$$", after_reload],
])
return code_str, local_variable_db
