def __parse_brief_token_sender(fh, ContinueF): # shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); } LanguageDB = Setup.language_db position = fh.tell() line_n = get_current_line_info_number(fh) + 1 try: skip_whitespace(fh) position = fh.tell() code = __parse_token_id_specification_by_character_code(fh) if code != -1: code = __create_token_sender_by_character_code(fh, code) else: skip_whitespace(fh) identifier = __read_token_identifier(fh) skip_whitespace(fh) if identifier in ["GOTO", "GOSUB", "GOUP"]: code = __create_mode_transition_and_token_sender( fh, identifier) else: code = __create_token_sender_by_token_name(fh, identifier) check_or_die(fh, ";") if code != "": if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n" return UserCodeFragment(code, fh.name, line_n, LanguageDB) else: return None except EndOfStreamException: fh.seek(position) error_msg("End of file reached while parsing token shortcut.", fh)
def __parse_brief_token_sender(fh, ContinueF): # shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); } position = fh.tell() try: skip_whitespace(fh) position = fh.tell() code = __parse_token_id_specification_by_character_code(fh) if code != -1: code = __create_token_sender_by_character_code(fh, code) else: skip_whitespace(fh) identifier = __read_token_identifier(fh) skip_whitespace(fh) if identifier in ["GOTO", "GOSUB", "GOUP"]: code = __create_mode_transition_and_token_sender( fh, identifier) else: code = __create_token_sender_by_token_name(fh, identifier) check_or_die(fh, ";") if len(code) != 0: if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n" return CodeUser(code, SourceRef.from_FileHandle(fh)) else: return None except EndOfStreamException: fh.seek(position) error_eof("token", fh)
def __parse_section(fh, descriptor, already_defined_list): global token_type_code_fragment_db assert type(already_defined_list) == list SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \ + token_type_code_fragment_db.keys() position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": fh.seek(position) if check(fh, "}"): fh.seek(position) return False error_msg("Missing token_type section ('standard', 'distinct', or 'union').", fh) verify_word_in_list(word, SubsectionList, "Subsection '%s' not allowed in token_type section." % word, fh) if word == "name": if not check(fh, "="): error_msg("Missing '=' in token_type 'name' specification.", fh) descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(fh, "token_type") if not check(fh, ";"): error_msg("Missing terminating ';' in token_type 'name' specification.", fh) elif word == "inheritable": descriptor.open_for_derivation_f = True check_or_die(fh, ";") elif word == "noid": descriptor.token_contains_token_id_f = False; check_or_die(fh, ";") elif word == "file_name": if not check(fh, "="): error_msg("Missing '=' in token_type 'file_name' specification.", fh) descriptor.set_file_name(read_until_letter(fh, ";")) if not check(fh, ";"): error_msg("Missing terminating ';' in token_type 'file_name' specification.", fh) elif word in ["standard", "distinct", "union"]: if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list) elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list) elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg("Missing closing '}' at end of token_type section '%s'." % word, fh); elif word in token_type_code_fragment_db.keys(): fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) descriptor.__dict__[word] = fragment else: assert False, "This code section section should not be reachable because 'word'\n" + \ "was checked to fit in one of the 'elif' cases." return True
def __parse_brief_token_sender(fh, ContinueF): # shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); } position = fh.tell() try: skip_whitespace(fh) position = fh.tell() code = __parse_token_id_specification_by_character_code(fh) if code != -1: code = __create_token_sender_by_character_code(fh, code) else: skip_whitespace(fh) identifier = __read_token_identifier(fh) skip_whitespace(fh) if identifier in ["GOTO", "GOSUB", "GOUP"]: code = __create_mode_transition_and_token_sender(fh, identifier) else: code = __create_token_sender_by_token_name(fh, identifier) check_or_die(fh, ";") if len(code) != 0: if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n" return CodeUser(code, SourceRef.from_FileHandle(fh)) else: return None except EndOfStreamException: fh.seek(position) error.error_eof("token", fh)
def __parse_brief_token_sender(fh, ContinueF): # shorthand for { self.send(TKN_SOMETHING); QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN(); } LanguageDB = Setup.language_db position = fh.tell() line_n = get_current_line_info_number(fh) + 1 try: skip_whitespace(fh) position = fh.tell() code = __parse_token_id_specification_by_character_code(fh) if code != -1: code = __create_token_sender_by_character_code(fh, code) else: skip_whitespace(fh) identifier = __read_token_identifier(fh) skip_whitespace(fh) if identifier in ["GOTO", "GOSUB", "GOUP"]: code = __create_mode_transition_and_token_sender(fh, identifier) else: code = __create_token_sender_by_token_name(fh, identifier) check_or_die(fh, ";") if code != "": if ContinueF: code += "QUEX_SETTING_AFTER_SEND_CONTINUE_OR_RETURN();\n" return UserCodeFragment(code, fh.name, line_n, LanguageDB) else: return None except EndOfStreamException: fh.seek(position) error_msg("End of file reached while parsing token shortcut.", fh)
def __parse_token_id_specification_by_character_code(fh): ## pos = fh.tell(); print "##input:", fh.read(3); fh.seek(pos) character_code = read_character_code(fh) ## print "##cc:", character_code if character_code == -1: return -1 check_or_die(fh, ";") return character_code
def __parse_keyword_list(new_mode, fh): """ADAPTS: new_mode.pattern_action_list where new pattern action pairs are entered. RETURNS: True, in case of success. EXITS: in case of syntax errors. """ flags = optional_flags( fh, "keyword_list", "", { "u": "make correspondent token identifiers uppercase.", "l": "make correspondent token identifiers lowercase.", "N": "pass LexemeNull to token contructor.", "L": "pass Lexeme to token constructor.", "i": "implicit token identifier definition." }, ["ul", "NL"]) lexeme_null_f = "N" in flags lexeme_f = "L" in flags implicit_tid_f = "i" in flags lowercase_f = "l" in flags uppercase_f = "u" in flags skip_whitespace(fh) prefix = read_identifier(fh) skip_whitespace(fh) check_or_die(fh, "{", "Opening bracket required after 'keyword_list'.") while not check(fh, "}"): skip_whitespace(fh) position = fh.tell() identifier = read_identifier(fh) pattern = regular_expression.parse(StringIO("%s " % identifier)) check_or_die(fh, ";", "Semincolon required after keyword '%s'." % identifier) if not identifier: continue if uppercase_f: identifier = identifier.upper() elif lowercase_f: identifier = identifier.lower() identifier = "%s%s" % (prefix, identifier) if implicit_tid_f: token_id_db_enter(fh, identifier) code = code_fragment.get_CodeUser_for_token_sending( fh, identifier, position, LexemeNullF=lexeme_null_f, LexemeF=lexeme_f) new_mode.add_pattern_action_pair(pattern, code, fh) return True
def __parse_brief(new_mode, fh): """ADAPTS: new_mode.pattern_action_list where new pattern action pairs are entered. RETURNS: True, in case of success. EXITS: in case of syntax errors. """ flags = optional_flags( fh, "brief pattern action pair list", "", { "N": "pass LexemeNull to token contructor.", "L": "pass Lexeme to token constructor.", "i": "implicit token identifier definition." }, ["NL"]) skip_whitespace(fh) prefix = read_identifier(fh) skip_whitespace(fh) lexeme_null_f = "N" in flags lexeme_f = "L" in flags implicit_tid_f = "i" in flags check_or_die(fh, "{", "Opening bracket required after 'brief'.") while not check(fh, "}"): skip_whitespace(fh) pattern = regular_expression.parse(fh) skip_whitespace(fh) position = fh.tell() identifier = read_identifier(fh) if not identifier: error.log("Missing identifier after regular expression.", fh) identifier = "%s%s" % (prefix, identifier) check_or_die( fh, ";", "Semincolon required after brief token identifier '%s'." % identifier) if implicit_tid_f: token_id_db_enter(fh, identifier) code = code_fragment.get_CodeUser_for_token_sending( fh, identifier, position, LexemeNullF=lexeme_null_f, LexemeF=lexeme_f) new_mode.add_pattern_action_pair(pattern, code, fh) return True
def __parse_action(new_mode, fh, pattern_str, pattern): position = fh.tell() try: skip_whitespace(fh) position = fh.tell() code_obj = code_fragment.parse(fh, "regular expression", ErrorOnFailureF=False) if code_obj is not None: new_mode.add_match(pattern_str, code_obj, pattern) return fh.seek(position) word = read_until_letter(fh, [";"]) if word == "PRIORITY-MARK": # This mark 'lowers' the priority of a pattern to the priority of the current # pattern index (important for inherited patterns, that have higher precedence). # The parser already constructed a state machine for the pattern that is to # be assigned a new priority. Since, this machine is not used, let us just # use its id. fh.seek(-1, 1) check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.") new_mode.add_match_priority(pattern_str, pattern, pattern.sm.get_id(), fh.name, get_current_line_info_number(fh)) elif word == "DELETION": # This mark deletes any pattern that was inherited with the same 'name' fh.seek(-1, 1) check_or_die(fh, ";", ". Since quex version 0.33.5 this is required.") new_mode.add_match_deletion(pattern_str, pattern, fh.name, get_current_line_info_number(fh)) else: error_msg("Missing token '{', 'PRIORITY-MARK', 'DELETION', or '=>' after '%s'.\n" % pattern_str + \ "found: '%s'. Note, that since quex version 0.33.5 it is required to add a ';'\n" % word + \ "to the commands PRIORITY-MARK and DELETION.", fh) except EndOfStreamException: fh.seek(position) error_msg("End of file reached while parsing action code for pattern.", fh)
def _parse_definition_head(fh, IdentifierList): if check(fh, "\\default"): error.log("'\\default' has been replaced by keyword '\\else' since quex 0.64.9!", fh) elif check(fh, "\\else"): pattern = None else: pattern = regular_expression.parse(fh) skip_whitespace(fh) check_or_die(fh, "=>", " after character set definition.") skip_whitespace(fh) identifier = read_identifier(fh, OnMissingStr="Missing identifier following '=>'.") error.verify_word_in_list(identifier, IdentifierList, "Unrecognized specifier '%s'." % identifier, fh) skip_whitespace(fh) return pattern, identifier, SourceRef.from_FileHandle(fh)
def __parse_brief_token_sender(fh): # shorthand for { self.send(TKN_SOMETHING); RETURN; } position = fh.tell() try: skip_whitespace(fh) position = fh.tell() code = __parse_token_id_specification_by_character_code(fh) if code != -1: code = __create_token_sender_by_character_code(fh, code) else: skip_whitespace(fh) identifier = __read_token_identifier(fh) skip_whitespace(fh) if identifier in ["GOTO", "GOSUB", "GOUP"]: code = __create_mode_transition_and_token_sender( fh, identifier) else: code = __create_token_sender_by_token_name(fh, identifier) check_or_die(fh, ";") if code: # IMPORTANT: For handlers 'on_end_of_stream' and 'on_failure', # => CONTINUE would be desastrous! # -- When a termination token is sent, no other token shall follow. # Return MUST be enforced => Do not allow CONTINUE! # -- When an 'on_failure' is detected allow immediate action of the # receiver. => Do not allow CONTINUE! code += "\n%s\n" % Lng.PURE_RETURN # Immediate RETURN after token sending return CodeUser(code, SourceRef.from_FileHandle(fh, BeginPos=position)) else: return None except EndOfStreamException: fh.seek(position) error.error_eof("token", fh)
def __parse_section(fh, descriptor, already_defined_list): global token_type_code_fragment_db assert type(already_defined_list) == list SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \ + token_type_code_fragment_db.keys() position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": fh.seek(position) if check(fh, "}"): fh.seek(position) return False error_msg( "Missing token_type section ('standard', 'distinct', or 'union').", fh) verify_word_in_list( word, SubsectionList, "Subsection '%s' not allowed in token_type section." % word, fh) if word == "name": if not check(fh, "="): error_msg("Missing '=' in token_type 'name' specification.", fh) descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name( fh, "token_type") if not check(fh, ";"): error_msg( "Missing terminating ';' in token_type 'name' specification.", fh) elif word == "inheritable": descriptor.open_for_derivation_f = True check_or_die(fh, ";") elif word == "noid": descriptor.token_contains_token_id_f = False check_or_die(fh, ";") elif word == "file_name": if not check(fh, "="): error_msg("Missing '=' in token_type 'file_name' specification.", fh) descriptor.set_file_name(read_until_letter(fh, ";")) if not check(fh, ";"): error_msg( "Missing terminating ';' in token_type 'file_name' specification.", fh) elif word in ["standard", "distinct", "union"]: if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list) elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list) elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg( "Missing closing '}' at end of token_type section '%s'." % word, fh) elif word in token_type_code_fragment_db.keys(): fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) descriptor.__dict__[word] = fragment else: assert False, "This code section section should not be reachable because 'word'\n" + \ "was checked to fit in one of the 'elif' cases." return True
def parse_state_statistics(fh): skip_whitespace(fh) check_or_die(fh, "{") check_or_die(fh, "mode:") skip_whitespace(fh) mode_name = read_until_letter(fh, ";") check_or_die(fh, "state:") skip_whitespace(fh) state_index = read_integer(fh) check_or_die(fh, ";") check_or_die(fh, "{") skip_whitespace(fh) boundary_list_str = read_until_letter(fh, ";") skip_whitespace(fh) counter_list_str = read_until_letter(fh, ";") skip_whitespace(fh) check_or_die(fh, "}") check_or_die(fh, "}") def help(X): map(lambda x: int(x), X.strip().split()) return StateStatistics(mode_name, state_index, help(boundary_list_str), help(counter_list_str))
def __parse_token_id_specification_by_character_code(fh): character_code = read_character_code(fh) if character_code == -1: return -1 check_or_die(fh, ";") return character_code