def parse_state_statistics(fh):
skip_whitespace(fh)
check_or_die(fh, "{")
check_or_die(fh, "mode:")
skip_whitespace(fh)
mode_name = read_until_letter(fh, ";")
check_or_die(fh, "state:")
skip_whitespace(fh)
state_index = read_integer(fh)
check_or_die(fh, ";")
check_or_die(fh, "{")
skip_whitespace(fh)
boundary_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
counter_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
check_or_die(fh, "}")
check_or_die(fh, "}")
def help(X):
map(lambda x: int(x), X.strip().split())
return StateStatistics(mode_name, state_index, help(boundary_list_str), help(counter_list_str))
def parse_state_statistics(fh):
skip_whitespace(fh)
check_or_die(fh, "{")
check_or_die(fh, "mode:")
skip_whitespace(fh)
mode_name = read_until_letter(fh, ";")
check_or_die(fh, "state:")
skip_whitespace(fh)
state_index = read_integer(fh)
check_or_die(fh, ";")
check_or_die(fh, "{")
skip_whitespace(fh)
boundary_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
counter_list_str = read_until_letter(fh, ";")
skip_whitespace(fh)
check_or_die(fh, "}")
check_or_die(fh, "}")
def help(X):
map(lambda x: int(x), X.strip().split())
return StateStatistics(mode_name, state_index, help(boundary_list_str),
help(counter_list_str))
def __parse_section(fh, descriptor, already_defined_list):
global token_type_code_fragment_db
assert type(already_defined_list) == list
SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \
+ token_type_code_fragment_db.keys()
position = fh.tell()
skip_whitespace(fh)
word = read_identifier(fh)
if word == "":
fh.seek(position)
if check(fh, "}"):
fh.seek(position)
return False
error_msg("Missing token_type section ('standard', 'distinct', or 'union').", fh)
verify_word_in_list(word, SubsectionList,
"Subsection '%s' not allowed in token_type section." % word, fh)
if word == "name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'name' specification.", fh)
descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(fh, "token_type")
if not check(fh, ";"):
error_msg("Missing terminating ';' in token_type 'name' specification.", fh)
elif word == "inheritable":
descriptor.open_for_derivation_f = True
check_or_die(fh, ";")
elif word == "noid":
descriptor.token_contains_token_id_f = False;
check_or_die(fh, ";")
elif word == "file_name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'file_name' specification.", fh)
descriptor.set_file_name(read_until_letter(fh, ";"))
if not check(fh, ";"):
error_msg("Missing terminating ';' in token_type 'file_name' specification.", fh)
elif word in ["standard", "distinct", "union"]:
if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list)
elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list)
elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg("Missing closing '}' at end of token_type section '%s'." % word, fh);
elif word in token_type_code_fragment_db.keys():
fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False)
descriptor.__dict__[word] = fragment
else:
assert False, "This code section section should not be reachable because 'word'\n" + \
"was checked to fit in one of the 'elif' cases."
return True
def __snap_word(stream):
try:
the_word = read_until_letter(stream, ["("])
except:
raise RegularExpressionException("Missing opening bracket.")
stream.seek(-1, 1)
return the_word.strip()
def __parse_action(new_mode, fh, pattern_str, pattern):
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code_obj = code_fragment.parse(fh,
"regular expression",
ErrorOnFailureF=False)
if code_obj is not None:
new_mode.add_match(pattern_str, code_obj, pattern)
return
fh.seek(position)
word = read_until_letter(fh, [";"])
if word == "PRIORITY-MARK":
# This mark 'lowers' the priority of a pattern to the priority of the current
# pattern index (important for inherited patterns, that have higher precedence).
# The parser already constructed a state machine for the pattern that is to
# be assigned a new priority. Since, this machine is not used, let us just
# use its id.
fh.seek(-1, 1)
check_or_die(fh, ";",
". Since quex version 0.33.5 this is required.")
new_mode.add_match_priority(pattern_str, pattern,
pattern.sm.get_id(), fh.name,
get_current_line_info_number(fh))
elif word == "DELETION":
# This mark deletes any pattern that was inherited with the same 'name'
fh.seek(-1, 1)
check_or_die(fh, ";",
". Since quex version 0.33.5 this is required.")
new_mode.add_match_deletion(pattern_str, pattern, fh.name,
get_current_line_info_number(fh))
else:
error_msg("Missing token '{', 'PRIORITY-MARK', 'DELETION', or '=>' after '%s'.\n" % pattern_str + \
"found: '%s'. Note, that since quex version 0.33.5 it is required to add a ';'\n" % word + \
"to the commands PRIORITY-MARK and DELETION.", fh)
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing action code for pattern.",
fh)
def __parse_action(new_mode, fh, pattern_str, pattern):
position = fh.tell()
try:
skip_whitespace(fh)
position = fh.tell()
code_obj = code_fragment.parse(fh, "regular expression", ErrorOnFailureF=False)
if code_obj is not None:
new_mode.add_match(pattern_str, code_obj, pattern)
return
fh.seek(position)
word = read_until_letter(fh, [";"])
if word == "PRIORITY-MARK":
# This mark 'lowers' the priority of a pattern to the priority of the current
# pattern index (important for inherited patterns, that have higher precedence).
# The parser already constructed a state machine for the pattern that is to
# be assigned a new priority. Since, this machine is not used, let us just
# use its id.
fh.seek(-1, 1)
check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.")
new_mode.add_match_priority(pattern_str, pattern, pattern.sm.get_id(),
fh.name, get_current_line_info_number(fh))
elif word == "DELETION":
# This mark deletes any pattern that was inherited with the same 'name'
fh.seek(-1, 1)
check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.")
new_mode.add_match_deletion(pattern_str, pattern, fh.name, get_current_line_info_number(fh))
else:
error_msg("Missing token '{', 'PRIORITY-MARK', 'DELETION', or '=>' after '%s'.\n" % pattern_str + \
"found: '%s'. Note, that since quex version 0.33.5 it is required to add a ';'\n" % word + \
"to the commands PRIORITY-MARK and DELETION.", fh)
except EndOfStreamException:
fh.seek(position)
error_msg("End of file reached while parsing action code for pattern.", fh)
def parse_variable_definition(fh, GroupF=False, already_defined_list=[]):
"""PURPOSE: Parsing of a variable definition consisting of 'type' and 'name.
Members can be mentioned together in a group, which means that
they can appear simultaneously. Possible expresions are
(1) single variables:
name0 : type;
name1 : type[32];
name2 : type*;
(2) combined variables
{
sub_name0 : type0;
sub_name1 : type[64];
sub_name2 : type1*;
}
ARGUMENTS:
'GroupF' allows to have 'nested variable groups' in curly brackets
'already_defined_list' informs about variable names that have been already
chosen. It is only used for groups.
RETURNS:
None on failure to pass a variable definition.
array when a single variable definition was found.
array[0] = UserCodeFragment containing the type.
array[1] = name of the variable.
dictionary if it was a combined variable definition. The dictionary
maps: (variable name) ---> (UserCodeFragment with type)
"""
position = fh.tell()
skip_whitespace(fh)
name_str = read_identifier(fh)
if name_str == "":
if not GroupF or not check(fh, "{"):
fh.seek(position)
return None
sub_db = parse_variable_definition_list(fh,
"Concurrent union variables",
already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg(
"Missing closing '}' after concurrent variable definition.",
fh)
return [sub_db]
else:
name_str = name_str.strip()
if not check(fh, ":"):
error_msg("Missing ':' after identifier '%s'." % name_str, fh)
if fh.read(1).isspace() == False:
error_msg("Missing whitespace after ':' after identifier '%s'.\n" % name_str \
+ "The notation has to be: variable-name ':' type ';'.", fh)
type_str, i = read_until_letter(fh, ";", Verbose=True)
if i == -1: error_msg("missing ';'", fh)
type_str = type_str.strip()
return [CodeUser(type_str, SourceRef.from_FileHandle(fh)), name_str]
def __parse_section(fh, descriptor, already_defined_list):
global token_type_code_fragment_db
assert type(already_defined_list) == list
SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \
+ token_type_code_fragment_db.keys()
position = fh.tell()
skip_whitespace(fh)
word = read_identifier(fh)
if word == "":
fh.seek(position)
if check(fh, "}"):
fh.seek(position)
return False
error_msg(
"Missing token_type section ('standard', 'distinct', or 'union').",
fh)
verify_word_in_list(
word, SubsectionList,
"Subsection '%s' not allowed in token_type section." % word, fh)
if word == "name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'name' specification.", fh)
descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(
fh, "token_type")
if not check(fh, ";"):
error_msg(
"Missing terminating ';' in token_type 'name' specification.",
fh)
elif word == "inheritable":
descriptor.open_for_derivation_f = True
check_or_die(fh, ";")
elif word == "noid":
descriptor.token_contains_token_id_f = False
check_or_die(fh, ";")
elif word == "file_name":
if not check(fh, "="):
error_msg("Missing '=' in token_type 'file_name' specification.",
fh)
descriptor.set_file_name(read_until_letter(fh, ";"))
if not check(fh, ";"):
error_msg(
"Missing terminating ';' in token_type 'file_name' specification.",
fh)
elif word in ["standard", "distinct", "union"]:
if word == "standard":
parse_standard_members(fh, word, descriptor, already_defined_list)
elif word == "distinct":
parse_distinct_members(fh, word, descriptor, already_defined_list)
elif word == "union":
parse_union_members(fh, word, descriptor, already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg(
"Missing closing '}' at end of token_type section '%s'." %
word, fh)
elif word in token_type_code_fragment_db.keys():
fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False)
descriptor.__dict__[word] = fragment
else:
assert False, "This code section section should not be reachable because 'word'\n" + \
"was checked to fit in one of the 'elif' cases."
return True
def __snap_word(stream):
try: the_word = read_until_letter(stream, ["("])
except:
raise RegularExpressionException("Missing opening bracket.")
stream.seek(-1,1)
return the_word.strip()
def parse_variable_definition(fh, GroupF=False, already_defined_list=[]):
"""PURPOSE: Parsing of a variable definition consisting of 'type' and 'name.
Members can be mentioned together in a group, which means that
they can appear simultaneously. Possible expresions are
(1) single variables:
name0 : type;
name1 : type[32];
name2 : type*;
(2) combined variables
{
sub_name0 : type0;
sub_name1 : type[64];
sub_name2 : type1*;
}
ARGUMENTS:
'GroupF' allows to have 'nested variable groups' in curly brackets
'already_defined_list' informs about variable names that have been already
chosen. It is only used for groups.
RETURNS:
None on failure to pass a variable definition.
array when a single variable definition was found.
array[0] = UserCodeFragment containing the type.
array[1] = name of the variable.
dictionary if it was a combined variable definition. The dictionary
maps: (variable name) ---> (UserCodeFragment with type)
"""
position = fh.tell()
skip_whitespace(fh)
name_str = read_identifier(fh)
if name_str == "":
if not GroupF or not check(fh, "{"):
fh.seek(position);
return None
sub_db = parse_variable_definition_list(fh, "Concurrent union variables", already_defined_list)
if not check(fh, "}"):
fh.seek(position)
error_msg("Missing closing '}' after concurrent variable definition.", fh)
return [ sub_db ]
else:
name_str = name_str.strip()
if not check(fh, ":"): error_msg("Missing ':' after identifier '%s'." % name_str, fh)
if fh.read(1).isspace() == False:
error_msg("Missing whitespace after ':' after identifier '%s'.\n" % name_str \
+ "The notation has to be: variable-name ':' type ';'.", fh)
type_str, i = read_until_letter(fh, ";", Verbose=True)
if i == -1: error_msg("missing ';'", fh)
type_str = type_str.strip()
return [ CodeUser(type_str, SourceRef.from_FileHandle(fh)), name_str ]