def __parse_brief_token_sender(fh, ContinueF):
# shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); }
LanguageDB = Setup.language_db
position = fh.tell()
line_n = get_current_line_info_number(fh) + 1
try:
skip_whitespace(fh)
position = fh.tell()
code = __parse_token_id_specification_by_character_code(fh)
if code != -1:
code = __create_token_sender_by_character_code(fh, code)
else:
skip_whitespace(fh)
identifier = __read_token_identifier(fh)
skip_whitespace(fh)
if identifier in ["GOTO", "GOSUB", "GOUP"]:
code = __create_mode_transition_and_token_sender(
fh, identifier)
else:
code = __create_token_sender_by_token_name(fh, identifier)
check_or_die(fh, ";")
if code != "":
if ContinueF:
code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n"
return UserCodeFragment(code, fh.name, line_n, LanguageDB)
else:
return None
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing token shortcut.", fh)
def __parse_brief_token_sender(fh, ContinueF):
# shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); }
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code = __parse_token_id_specification_by_character_code(fh)
if code != -1:
code = __create_token_sender_by_character_code(fh, code)
else:
skip_whitespace(fh)
identifier = __read_token_identifier(fh)
skip_whitespace(fh)
if identifier in ["GOTO", "GOSUB", "GOUP"]:
code = __create_mode_transition_and_token_sender(
fh, identifier)
else:
code = __create_token_sender_by_token_name(fh, identifier)
check_or_die(fh, ";")
if len(code) != 0:
if ContinueF:
code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n"
return CodeUser(code, SourceRef.from_FileHandle(fh))
else:
return None
except EndOfStreamException:
fh.seek(position)
error_eof("token", fh)
def __parse_section(fh, descriptor, already_defined_list):
global token_type_code_fragment_db
assert type(already_defined_list) == list
SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \
+ token_type_code_fragment_db.keys()
position = fh.tell()
skip_whitespace(fh)
word = read_identifier(fh)
if word == "":
fh.seek(position)
if check(fh, "}"):
fh.seek(position)
return False
error_msg("Missing token_type section ('standard', 'distinct', or 'union').", fh)
verify_word_in_list(word, SubsectionList,
"Subsection '%s' not allowed in token_type section." % word, fh)
if word == "name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'name' specification.", fh)
descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(fh, "token_type")
if not check(fh, ";"):
error_msg("Missing terminating ';' in token_type 'name' specification.", fh)
elif word == "inheritable":
descriptor.open_for_derivation_f = True
check_or_die(fh, ";")
elif word == "noid":
descriptor.token_contains_token_id_f = False;
check_or_die(fh, ";")
elif word == "file_name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'file_name' specification.", fh)
descriptor.set_file_name(read_until_letter(fh, ";"))
if not check(fh, ";"):
error_msg("Missing terminating ';' in token_type 'file_name' specification.", fh)
elif word in ["standard", "distinct", "union"]:
if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list)
elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list)
elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg("Missing closing '}' at end of token_type section '%s'." % word, fh);
elif word in token_type_code_fragment_db.keys():
fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False)
descriptor.__dict__[word] = fragment
else:
assert False, "This code section section should not be reachable because 'word'\n" + \
"was checked to fit in one of the 'elif' cases."
return True
def __parse_brief_token_sender(fh, ContinueF):
# shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); }
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code = __parse_token_id_specification_by_character_code(fh)
if code != -1:
code = __create_token_sender_by_character_code(fh, code)
else:
skip_whitespace(fh)
identifier = __read_token_identifier(fh)
skip_whitespace(fh)
if identifier in ["GOTO", "GOSUB", "GOUP"]:
code = __create_mode_transition_and_token_sender(fh, identifier)
else:
code = __create_token_sender_by_token_name(fh, identifier)
check_or_die(fh, ";")
if len(code) != 0:
if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n"
return CodeUser(code, SourceRef.from_FileHandle(fh))
else:
return None
except EndOfStreamException:
fh.seek(position)
error.error_eof("token", fh)
def __parse_brief_token_sender(fh, ContinueF):
# shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); }
LanguageDB = Setup.language_db
position = fh.tell()
line_n = get_current_line_info_number(fh) + 1
try:
skip_whitespace(fh)
position = fh.tell()
code = __parse_token_id_specification_by_character_code(fh)
if code != -1:
code = __create_token_sender_by_character_code(fh, code)
else:
skip_whitespace(fh)
identifier = __read_token_identifier(fh)
skip_whitespace(fh)
if identifier in ["GOTO", "GOSUB", "GOUP"]:
code = __create_mode_transition_and_token_sender(fh, identifier)
else:
code = __create_token_sender_by_token_name(fh, identifier)
check_or_die(fh, ";")
if code != "":
if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n"
return UserCodeFragment(code, fh.name, line_n, LanguageDB)
else:
return None
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing token shortcut.", fh)
def __parse_token_id_specification_by_character_code(fh):
## pos = fh.tell(); print "##input:", fh.read(3); fh.seek(pos)
character_code = read_character_code(fh)
## print "##cc:", character_code
if character_code == -1: return -1
check_or_die(fh, ";")
return character_code
def __parse_token_id_specification_by_character_code(fh):
## pos = fh.tell(); print "##input:", fh.read(3); fh.seek(pos)
character_code = read_character_code(fh)
## print "##cc:", character_code
if character_code == -1: return -1
check_or_die(fh, ";")
return character_code
def __parse_keyword_list(new_mode, fh):
"""ADAPTS: new_mode.pattern_action_list where new pattern action pairs
are entered.
RETURNS: True, in case of success.
EXITS: in case of syntax errors.
"""
flags = optional_flags(
fh, "keyword_list", "", {
"u": "make correspondent token identifiers uppercase.",
"l": "make correspondent token identifiers lowercase.",
"N": "pass LexemeNull to token contructor.",
"L": "pass Lexeme to token constructor.",
"i": "implicit token identifier definition."
}, ["ul", "NL"])
lexeme_null_f = "N" in flags
lexeme_f = "L" in flags
implicit_tid_f = "i" in flags
lowercase_f = "l" in flags
uppercase_f = "u" in flags
skip_whitespace(fh)
prefix = read_identifier(fh)
skip_whitespace(fh)
check_or_die(fh, "{", "Opening bracket required after 'keyword_list'.")
while not check(fh, "}"):
skip_whitespace(fh)
position = fh.tell()
identifier = read_identifier(fh)
pattern = regular_expression.parse(StringIO("%s " % identifier))
check_or_die(fh, ";",
"Semincolon required after keyword '%s'." % identifier)
if not identifier: continue
if uppercase_f: identifier = identifier.upper()
elif lowercase_f: identifier = identifier.lower()
identifier = "%s%s" % (prefix, identifier)
if implicit_tid_f: token_id_db_enter(fh, identifier)
code = code_fragment.get_CodeUser_for_token_sending(
fh,
identifier,
position,
LexemeNullF=lexeme_null_f,
LexemeF=lexeme_f)
new_mode.add_pattern_action_pair(pattern, code, fh)
return True
def __parse_brief(new_mode, fh):
"""ADAPTS: new_mode.pattern_action_list where new pattern action pairs
are entered.
RETURNS: True, in case of success.
EXITS: in case of syntax errors.
"""
flags = optional_flags(
fh, "brief pattern action pair list", "", {
"N": "pass LexemeNull to token contructor.",
"L": "pass Lexeme to token constructor.",
"i": "implicit token identifier definition."
}, ["NL"])
skip_whitespace(fh)
prefix = read_identifier(fh)
skip_whitespace(fh)
lexeme_null_f = "N" in flags
lexeme_f = "L" in flags
implicit_tid_f = "i" in flags
check_or_die(fh, "{", "Opening bracket required after 'brief'.")
while not check(fh, "}"):
skip_whitespace(fh)
pattern = regular_expression.parse(fh)
skip_whitespace(fh)
position = fh.tell()
identifier = read_identifier(fh)
if not identifier:
error.log("Missing identifier after regular expression.", fh)
identifier = "%s%s" % (prefix, identifier)
check_or_die(
fh, ";", "Semincolon required after brief token identifier '%s'." %
identifier)
if implicit_tid_f: token_id_db_enter(fh, identifier)
code = code_fragment.get_CodeUser_for_token_sending(
fh,
identifier,
position,
LexemeNullF=lexeme_null_f,
LexemeF=lexeme_f)
new_mode.add_pattern_action_pair(pattern, code, fh)
return True
def __parse_action(new_mode, fh, pattern_str, pattern):
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code_obj = code_fragment.parse(fh,
"regular expression",
ErrorOnFailureF=False)
if code_obj is not None:
new_mode.add_match(pattern_str, code_obj, pattern)
return
fh.seek(position)
word = read_until_letter(fh, [";"])
if word == "PRIORITY-MARK":
# This mark 'lowers' the priority of a pattern to the priority of the current
# pattern index (important for inherited patterns, that have higher precedence).
# The parser already constructed a state machine for the pattern that is to
# be assigned a new priority. Since, this machine is not used, let us just
# use its id.
fh.seek(-1, 1)
check_or_die(fh, ";",
". Since quex version 0.33.5 this is required.")
new_mode.add_match_priority(pattern_str, pattern,
pattern.sm.get_id(), fh.name,
get_current_line_info_number(fh))
elif word == "DELETION":
# This mark deletes any pattern that was inherited with the same 'name'
fh.seek(-1, 1)
check_or_die(fh, ";",
". Since quex version 0.33.5 this is required.")
new_mode.add_match_deletion(pattern_str, pattern, fh.name,
get_current_line_info_number(fh))
else:
error_msg("Missing token '{', 'PRIORITY-MARK', 'DELETION', or '=>' after '%s'.\n" % pattern_str + \
"found: '%s'. Note, that since quex version 0.33.5 it is required to add a ';'\n" % word + \
"to the commands PRIORITY-MARK and DELETION.", fh)
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing action code for pattern.",
fh)
def _parse_definition_head(fh, IdentifierList):
if check(fh, "\\default"):
error.log("'\\default' has been replaced by keyword '\\else' since quex 0.64.9!", fh)
elif check(fh, "\\else"):
pattern = None
else:
pattern = regular_expression.parse(fh)
skip_whitespace(fh)
check_or_die(fh, "=>", " after character set definition.")
skip_whitespace(fh)
identifier = read_identifier(fh, OnMissingStr="Missing identifier following '=>'.")
error.verify_word_in_list(identifier, IdentifierList,
"Unrecognized specifier '%s'." % identifier, fh)
skip_whitespace(fh)
return pattern, identifier, SourceRef.from_FileHandle(fh)
def __parse_action(new_mode, fh, pattern_str, pattern):
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code_obj = code_fragment.parse(fh, "regular expression", ErrorOnFailureF=False)
if code_obj is not None:
new_mode.add_match(pattern_str, code_obj, pattern)
return
fh.seek(position)
word = read_until_letter(fh, [";"])
if word == "PRIORITY-MARK":
# This mark 'lowers' the priority of a pattern to the priority of the current
# pattern index (important for inherited patterns, that have higher precedence).
# The parser already constructed a state machine for the pattern that is to
# be assigned a new priority. Since, this machine is not used, let us just
# use its id.
fh.seek(-1, 1)
check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.")
new_mode.add_match_priority(pattern_str, pattern, pattern.sm.get_id(),
fh.name, get_current_line_info_number(fh))
elif word == "DELETION":
# This mark deletes any pattern that was inherited with the same 'name'
fh.seek(-1, 1)
check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.")
new_mode.add_match_deletion(pattern_str, pattern, fh.name, get_current_line_info_number(fh))
else:
error_msg("Missing token '{', 'PRIORITY-MARK', 'DELETION', or '=>' after '%s'.\n" % pattern_str + \
"found: '%s'. Note, that since quex version 0.33.5 it is required to add a ';'\n" % word + \
"to the commands PRIORITY-MARK and DELETION.", fh)
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing action code for pattern.", fh)
def __parse_brief_token_sender(fh):
# shorthand for { self.send(TKN_SOMETHING); RETURN; }
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code = __parse_token_id_specification_by_character_code(fh)
if code != -1:
code = __create_token_sender_by_character_code(fh, code)
else:
skip_whitespace(fh)
identifier = __read_token_identifier(fh)
skip_whitespace(fh)
if identifier in ["GOTO", "GOSUB", "GOUP"]:
code = __create_mode_transition_and_token_sender(
fh, identifier)
else:
code = __create_token_sender_by_token_name(fh, identifier)
check_or_die(fh, ";")
if code:
# IMPORTANT: For handlers 'on_end_of_stream' and 'on_failure',
# => CONTINUE would be desastrous!
# -- When a termination token is sent, no other token shall follow.
# Return MUST be enforced => Do not allow CONTINUE!
# -- When an 'on_failure' is detected allow immediate action of the
# receiver. => Do not allow CONTINUE!
code += "\n%s\n" % Lng.PURE_RETURN # Immediate RETURN after token sending
return CodeUser(code,
SourceRef.from_FileHandle(fh, BeginPos=position))
else:
return None
except EndOfStreamException:
fh.seek(position)
error.error_eof("token", fh)
def __parse_section(fh, descriptor, already_defined_list):
global token_type_code_fragment_db
assert type(already_defined_list) == list
SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \
+ token_type_code_fragment_db.keys()
position = fh.tell()
skip_whitespace(fh)
word = read_identifier(fh)
if word == "":
fh.seek(position)
if check(fh, "}"):
fh.seek(position)
return False
error_msg(
"Missing token_type section ('standard', 'distinct', or 'union').",
fh)
verify_word_in_list(
word, SubsectionList,
"Subsection '%s' not allowed in token_type section." % word, fh)
if word == "name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'name' specification.", fh)
descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(
fh, "token_type")
if not check(fh, ";"):
error_msg(
"Missing terminating ';' in token_type 'name' specification.",
fh)
elif word == "inheritable":
descriptor.open_for_derivation_f = True
check_or_die(fh, ";")
elif word == "noid":
descriptor.token_contains_token_id_f = False
check_or_die(fh, ";")
elif word == "file_name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'file_name' specification.",
fh)
descriptor.set_file_name(read_until_letter(fh, ";"))
if not check(fh, ";"):
error_msg(
"Missing terminating ';' in token_type 'file_name' specification.",
fh)
elif word in ["standard", "distinct", "union"]:
if word == "standard":
parse_standard_members(fh, word, descriptor, already_defined_list)
elif word == "distinct":
parse_distinct_members(fh, word, descriptor, already_defined_list)
elif word == "union":
parse_union_members(fh, word, descriptor, already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg(
"Missing closing '}' at end of token_type section '%s'." %
word, fh)
elif word in token_type_code_fragment_db.keys():
fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False)
descriptor.__dict__[word] = fragment
else:
assert False, "This code section section should not be reachable because 'word'\n" + \
"was checked to fit in one of the 'elif' cases."
return True
def parse_state_statistics(fh):
skip_whitespace(fh)
check_or_die(fh, "{")
check_or_die(fh, "mode:")
skip_whitespace(fh)
mode_name = read_until_letter(fh, ";")
check_or_die(fh, "state:")
skip_whitespace(fh)
state_index = read_integer(fh)
check_or_die(fh, ";")
check_or_die(fh, "{")
skip_whitespace(fh)
boundary_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
counter_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
check_or_die(fh, "}")
check_or_die(fh, "}")
def help(X):
map(lambda x: int(x), X.strip().split())
return StateStatistics(mode_name, state_index, help(boundary_list_str),
help(counter_list_str))
def __parse_token_id_specification_by_character_code(fh):
character_code = read_character_code(fh)
if character_code == -1: return -1
check_or_die(fh, ";")
return character_code
def parse_state_statistics(fh):
skip_whitespace(fh)
check_or_die(fh, "{")
check_or_die(fh, "mode:")
skip_whitespace(fh)
mode_name = read_until_letter(fh, ";")
check_or_die(fh, "state:")
skip_whitespace(fh)
state_index = read_integer(fh)
check_or_die(fh, ";")
check_or_die(fh, "{")
skip_whitespace(fh)
boundary_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
counter_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
check_or_die(fh, "}")
check_or_die(fh, "}")
def help(X):
map(lambda x: int(x), X.strip().split())
return StateStatistics(mode_name, state_index, help(boundary_list_str), help(counter_list_str))
def __parse_token_id_specification_by_character_code(fh):
character_code = read_character_code(fh)
if character_code == -1: return -1
check_or_die(fh, ";")
return character_code
