def __parse_base_mode_list(fh, new_mode): new_mode.base_modes = [] trailing_comma_f = False while 1 + 1 == 2: if check(fh, "{"): fh.seek(-1, 1); break elif check(fh, "<"): fh.seek(-1, 1); break skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": break new_mode.base_modes.append(identifier) trailing_comma_f = False if not check(fh, ","): break trailing_comma_f = True if trailing_comma_f: error_msg("Trailing ',' after base mode '%s'." % new_mode.base_modes[-1], fh, DontExitF=True, WarningF=True) elif len(new_mode.base_modes) != 0: # This check is a 'service' -- for those who follow the old convention pos = fh.tell() skip_whitespace(fh) dummy_identifier = read_identifier(fh) if dummy_identifier != "": error_msg("Missing separating ',' between base modes '%s' and '%s'.\n" \ % (new_mode.base_modes[-1], dummy_identifier) + \ "(The comma separator is mandatory since quex 0.53.1)", fh) fh.seek(pos)
def __parse_base_mode_list(fh, new_mode): new_mode.derived_from_list = [] trailing_comma_f = False while 1 + 1 == 2: if check(fh, "{"): fh.seek(-1, 1) break elif check(fh, "<"): fh.seek(-1, 1) break skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": break new_mode.derived_from_list.append(identifier) trailing_comma_f = False if not check(fh, ","): break trailing_comma_f = True if trailing_comma_f: error.warning( "Trailing ',' after base mode '%s'." % new_mode.derived_from_list[-1], fh) elif len(new_mode.derived_from_list) != 0: # This check is a 'service' -- for those who follow the old convention pos = fh.tell() skip_whitespace(fh) dummy_identifier = read_identifier(fh) if dummy_identifier != "": error.log("Missing separating ',' between base modes '%s' and '%s'.\n" \ % (new_mode.derived_from_list[-1], dummy_identifier) + \ "(The comma separator is mandatory since quex 0.53.1)", fh) fh.seek(pos)
def __parse_keyword_list(new_mode, fh): """ADAPTS: new_mode.pattern_action_list where new pattern action pairs are entered. RETURNS: True, in case of success. EXITS: in case of syntax errors. """ flags = optional_flags( fh, "keyword_list", "", { "u": "make correspondent token identifiers uppercase.", "l": "make correspondent token identifiers lowercase.", "N": "pass LexemeNull to token contructor.", "L": "pass Lexeme to token constructor.", "i": "implicit token identifier definition." }, ["ul", "NL"]) lexeme_null_f = "N" in flags lexeme_f = "L" in flags implicit_tid_f = "i" in flags lowercase_f = "l" in flags uppercase_f = "u" in flags skip_whitespace(fh) prefix = read_identifier(fh) skip_whitespace(fh) check_or_die(fh, "{", "Opening bracket required after 'keyword_list'.") while not check(fh, "}"): skip_whitespace(fh) position = fh.tell() identifier = read_identifier(fh) pattern = regular_expression.parse(StringIO("%s " % identifier)) check_or_die(fh, ";", "Semincolon required after keyword '%s'." % identifier) if not identifier: continue if uppercase_f: identifier = identifier.upper() elif lowercase_f: identifier = identifier.lower() identifier = "%s%s" % (prefix, identifier) if implicit_tid_f: token_id_db_enter(fh, identifier) code = code_fragment.get_CodeUser_for_token_sending( fh, identifier, position, LexemeNullF=lexeme_null_f, LexemeF=lexeme_f) new_mode.add_pattern_action_pair(pattern, code, fh) return True
def __parse_brief(new_mode, fh): """ADAPTS: new_mode.pattern_action_list where new pattern action pairs are entered. RETURNS: True, in case of success. EXITS: in case of syntax errors. """ flags = optional_flags( fh, "brief pattern action pair list", "", { "N": "pass LexemeNull to token contructor.", "L": "pass Lexeme to token constructor.", "i": "implicit token identifier definition." }, ["NL"]) skip_whitespace(fh) prefix = read_identifier(fh) skip_whitespace(fh) lexeme_null_f = "N" in flags lexeme_f = "L" in flags implicit_tid_f = "i" in flags check_or_die(fh, "{", "Opening bracket required after 'brief'.") while not check(fh, "}"): skip_whitespace(fh) pattern = regular_expression.parse(fh) skip_whitespace(fh) position = fh.tell() identifier = read_identifier(fh) if not identifier: error.log("Missing identifier after regular expression.", fh) identifier = "%s%s" % (prefix, identifier) check_or_die( fh, ";", "Semincolon required after brief token identifier '%s'." % identifier) if implicit_tid_f: token_id_db_enter(fh, identifier) code = code_fragment.get_CodeUser_for_token_sending( fh, identifier, position, LexemeNullF=lexeme_null_f, LexemeF=lexeme_f) new_mode.add_pattern_action_pair(pattern, code, fh) return True
def __parse_section(fh, descriptor, already_defined_list): global token_type_code_fragment_db assert type(already_defined_list) == list SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \ + token_type_code_fragment_db.keys() position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": fh.seek(position) if check(fh, "}"): fh.seek(position) return False error_msg("Missing token_type section ('standard', 'distinct', or 'union').", fh) verify_word_in_list(word, SubsectionList, "Subsection '%s' not allowed in token_type section." % word, fh) if word == "name": if not check(fh, "="): error_msg("Missing '=' in token_type 'name' specification.", fh) descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name(fh, "token_type") if not check(fh, ";"): error_msg("Missing terminating ';' in token_type 'name' specification.", fh) elif word == "inheritable": descriptor.open_for_derivation_f = True check_or_die(fh, ";") elif word == "noid": descriptor.token_contains_token_id_f = False; check_or_die(fh, ";") elif word == "file_name": if not check(fh, "="): error_msg("Missing '=' in token_type 'file_name' specification.", fh) descriptor.set_file_name(read_until_letter(fh, ";")) if not check(fh, ";"): error_msg("Missing terminating ';' in token_type 'file_name' specification.", fh) elif word in ["standard", "distinct", "union"]: if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list) elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list) elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg("Missing closing '}' at end of token_type section '%s'." % word, fh); elif word in token_type_code_fragment_db.keys(): fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) descriptor.__dict__[word] = fragment else: assert False, "This code section section should not be reachable because 'word'\n" + \ "was checked to fit in one of the 'elif' cases." return True
def parse_token_id_definitions(fh, NamesOnlyF=False): # NOTE: Catching of EOF happens in caller: parse_section(...) # token_prefix = Setup.token_id_prefix token_prefix_plain = Setup.token_id_prefix_plain # i.e. without name space included if NamesOnlyF: db = {} else: db = blackboard.token_id_db skip_whitespace(fh) if not check(fh, "{"): error_msg("missing opening '{' for after 'token' section identifier.\n", fh) while check(fh, "}") == False: skip_whitespace(fh) candidate = read_identifier(fh, TolerantF=True) if candidate == "": error_msg("Missing valid token identifier." % candidate, fh) # -- check the name, if it starts with the token prefix paste a warning if candidate.find(token_prefix) == 0: error_msg("Token identifier '%s' starts with token prefix '%s'.\n" % (candidate, token_prefix) + \ "Token prefix is mounted automatically. This token id appears in the source\n" + \ "code as '%s%s'." % (token_prefix, candidate), \ fh, DontExitF=True) elif candidate.find(token_prefix_plain) == 0: error_msg("Token identifier '%s' starts with token prefix '%s'.\n" % (candidate, token_prefix) + \ "Token prefix is mounted automatically. This token id appears in the source\n" + \ "code as '%s%s'." % (token_prefix, candidate), \ fh, DontExitF=True) skip_whitespace(fh) if NamesOnlyF: db[token_prefix + candidate] = True if check(fh, ";") == False: error_msg("Missing ';' after definition of token identifier '%s'.\n" % candidate + \ "This is mandatory since Quex version 0.50.1.", fh) continue # Parse a possible numeric value after '=' numeric_value = None if check(fh, "="): skip_whitespace(fh) numeric_value = read_integer(fh) if numeric_value is None: error_msg("Missing number after '=' for token identifier '%s'." % candidate, fh) if check(fh, ";") == False: error_msg("Missing ';' after definition of token identifier '%s'.\n" % candidate + \ "This is mandatory since Quex version 0.50.1.", fh) db[candidate] = TokenInfo(candidate, numeric_value, Filename=fh.name, LineN=get_current_line_info_number(fh)) if NamesOnlyF: result = db.keys() result.sort() return result
def parse(fh): """This function parses a mode description and enters it into the 'blackboard.mode_description_db'. Once all modes are parsed they can be translated into 'real' modes and are located in 'blackboard.mode_db'. """ # NOTE: Catching of EOF happens in caller: parse_section(...) skip_whitespace(fh) mode_name = read_identifier( fh, OnMissingStr="Missing identifier at beginning of mode definition.") # NOTE: constructor does register this mode in the mode_db new_mode = ModeDescription(mode_name, SourceRef.from_FileHandle(fh)) # (*) inherited modes / option_db skip_whitespace(fh) dummy = fh.read(1) if dummy not in [":", "{"]: error.log("missing ':' or '{' after mode '%s'" % mode_name, fh) if dummy == ":": __parse_option_list(new_mode, fh) # (*) read in pattern-action pairs and events while __parse_element(new_mode, fh): pass
def parse(fh, mode_prep_prep_db): """This function parses a mode description and enters it into the 'blackboard.mode_prep_prep_db'. Modes are represented by Mode_PrepPrep objects. """ # NOTE: Catching of EOF happens in caller: parse_section(...) skip_whitespace(fh) mode_name = read_identifier( fh, OnMissingStr="Missing identifier at beginning of mode definition.") error.insight("Mode '%s'" % mode_name) # NOTE: constructor does register this mode in the mode_db new_mode = Mode_PrepPrep(mode_name, SourceRef.from_FileHandle(fh)) if new_mode.name in mode_prep_prep_db: error.log("Mode '%s' has been defined twice.\n" % new_mode.name, new_mode.sr, DontExitF=True) error.log("Earlier definition here.", mode_prep_prep_db[new_mode.name].sr) mode_prep_prep_db[new_mode.name] = new_mode # (*) inherited modes / option_db skip_whitespace(fh) dummy = fh.read(1) if dummy not in [":", "{"]: error.log("missing ':' or '{' after mode '%s'" % mode_name, fh) if dummy == ":": __parse_option_list(new_mode, fh) # (*) read in pattern-action pairs and events while __parse_element(new_mode, fh): pass
def parse(fh): """This function parses a mode description and enters it into the 'mode_description_db'. Once all modes are parsed they can be translated into 'real' modes and are located in 'blackboard.mode_db'. """ # NOTE: Catching of EOF happens in caller: parse_section(...) skip_whitespace(fh) mode_name = read_identifier(fh) if mode_name == "": error_msg("missing identifier at beginning of mode definition.", fh) # NOTE: constructor does register this mode in the mode_db new_mode = ModeDescription(mode_name, fh.name, get_current_line_info_number(fh)) # (*) inherited modes / options skip_whitespace(fh) dummy = fh.read(1) if dummy not in [":", "{"]: error_msg("missing ':' or '{' after mode '%s'" % mode_name, fh) if dummy == ":": __parse_option_list(new_mode, fh) # (*) read in pattern-action pairs and events while __parse_element(new_mode, fh): pass # (*) check for modes w/o pattern definitions if not new_mode.has_event_handler() and not new_mode.has_own_matches(): if new_mode.options["inheritable"] != "only": new_mode.options["inheritable"] = "only" error_msg("Mode without pattern and event handlers needs to be 'inheritable only'.\n" + \ "<inheritable: only> has been added automatically.", fh, DontExitF=True)
def parse(fh): """This function parses a mode description and enters it into the 'blackboard.mode_description_db'. Once all modes are parsed they can be translated into 'real' modes and are located in 'blackboard.mode_db'. """ # NOTE: Catching of EOF happens in caller: parse_section(...) skip_whitespace(fh) mode_name = read_identifier(fh, OnMissingStr="Missing identifier at beginning of mode definition.") # NOTE: constructor does register this mode in the mode_db new_mode = ModeDescription(mode_name, SourceRef.from_FileHandle(fh)) # (*) inherited modes / option_db skip_whitespace(fh) dummy = fh.read(1) if dummy not in [":", "{"]: error_msg("missing ':' or '{' after mode '%s'" % mode_name, fh) if dummy == ":": __parse_option_list(new_mode, fh) # (*) read in pattern-action pairs and events while __parse_element(new_mode, fh): pass
def _parse_pattern(fh): name = read_identifier(fh, OnMissingStr="Missing identifier for pattern definition.") if blackboard.shorthand_db.has_key(name): error.log("Second definition of pattern '%s'.\n" % name + \ "Pattern names must be unique.", fh) skip_whitespace(fh) if check(fh, "}"): error.log("Missing regular expression for pattern definition '%s'." % \ name, fh) # No encoding transformation, here. Transformation happens after # expansion in a mode. pattern = regular_expression.parse(fh, AllowNothingIsFineF = True) if pattern.has_pre_or_post_context(): error.log("Pattern definition with pre- and/or post-context.\n" + \ "Pre- and Post-Contexts can only be defined inside mode definitions.", fh) state_machine = pattern.extract_sm() value = PatternShorthand(name, state_machine, SourceRef.from_FileHandle(fh), pattern.pattern_string()) return name, value
def snap_replacement(stream, PatternDict, StateMachineF=True): """Snaps a predefined pattern from the input string and returns the resulting state machine. """ skip_whitespace(stream) pattern_name = read_identifier(stream) if pattern_name == "": raise RegularExpressionException( "Pattern replacement expression misses identifier after '{'.") skip_whitespace(stream) if not check(stream, "}"): raise RegularExpressionException("Pattern replacement expression misses closing '}' after '%s'." \ % pattern_name) error.verify_word_in_list( pattern_name, PatternDict.keys(), "Specifier '%s' not found in any preceeding 'define { ... }' section." % pattern_name, stream) reference = PatternDict[pattern_name] assert reference.__class__ == PatternShorthand # The replacement may be a state machine or a number set if StateMachineF: # Get a cloned version of state machine state_machine = reference.get_state_machine() assert isinstance(state_machine, DFA) # It is essential that state machines defined as patterns do not # have origins. Otherwise, the optimization of patterns that # contain pattern replacements might get confused and can # not find all optimizations. assert not state_machine.has_specific_acceptance_id() # A state machine, that contains pre- or post- conditions cannot be part # of a replacement. The addition of new post-contexts would mess up the pattern. ## if state_machine.has_pre_or_post_context(): ## error.log("Pre- or post-conditioned pattern was used in replacement.\n" + \ ## "Quex's regular expression grammar does not allow this.", stream) return state_machine else: # Get a cloned version of character set character_set = reference.get_character_set() if character_set is None: error.log( "Replacement in character set expression must be a character set.\n" "Specifier '%s' relates to a pattern state machine." % pattern_name, stream) if character_set.is_empty(): error.log( "Referenced character set '%s' is empty.\nAborted." % pattern_name, stream) return character_set
def snap_set_term(stream, PatternDict): global special_character_set_db __debug_entry("set_term", stream) operation_list = ["union", "intersection", "difference", "inverse"] character_set_list = special_character_set_db.keys() skip_whitespace(stream) position = stream.tell() # if there is no following '(', then enter the 'snap_expression' block below word = read_identifier(stream) if word in operation_list: set_list = snap_set_list(stream, word, PatternDict) # if an error occurs during set_list parsing, an exception is thrown about syntax error L = len(set_list) result = set_list[0] if word == "inverse": # The inverse of multiple sets, is to be the inverse of the union of these sets. if L > 1: for character_set in set_list[1:]: result.unite_with(character_set) return __debug_exit( result.get_complement(Setup.buffer_codec.source_set), stream) if L < 2: raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \ "two sets to operate on them.") if word == "union": for set in set_list[1:]: result.unite_with(set) elif word == "intersection": for set in set_list[1:]: result.intersect_with(set) elif word == "difference": for set in set_list[1:]: result.subtract(set) elif word in character_set_list: reg_expr = special_character_set_db[word] result = traditional_character_set.do_string(reg_expr) elif word != "": error.verify_word_in_list(word, character_set_list + operation_list, "Unknown keyword '%s'." % word, stream) else: stream.seek(position) result = snap_set_expression(stream, PatternDict) return __debug_exit(result, stream)
def snap_set_term(stream, PatternDict): global special_character_set_db __debug_entry("set_term", stream) operation_list = [ "union", "intersection", "difference", "inverse"] character_set_list = special_character_set_db.keys() skip_whitespace(stream) position = stream.tell() # if there is no following '(', then enter the 'snap_expression' block below word = read_identifier(stream) if word in operation_list: set_list = snap_set_list(stream, word, PatternDict) # if an error occurs during set_list parsing, an exception is thrown about syntax error L = len(set_list) result = set_list[0] if word == "inverse": # The inverse of multiple sets, is to be the inverse of the union of these sets. if L > 1: for character_set in set_list[1:]: result.unite_with(character_set) return __debug_exit(result.get_complement(Setup.buffer_codec.source_set), stream) if L < 2: raise RegularExpressionException("Regular Expression: A %s operation needs at least\n" % word + \ "two sets to operate on them.") if word == "union": for set in set_list[1:]: result.unite_with(set) elif word == "intersection": for set in set_list[1:]: result.intersect_with(set) elif word == "difference": for set in set_list[1:]: result.subtract(set) elif word in character_set_list: reg_expr = special_character_set_db[word] result = traditional_character_set.do_string(reg_expr) elif word != "": verify_word_in_list(word, character_set_list + operation_list, "Unknown keyword '%s'." % word, stream) else: stream.seek(position) result = snap_set_expression(stream, PatternDict) return __debug_exit(result, stream)
def parse_pattern_name_definitions(fh): """Parses pattern definitions of the form: WHITESPACE [ \t\n] IDENTIFIER [a-zA-Z0-9]+ OP_PLUS "+" That means: 'name' whitespace 'regular expression' whitespace newline. Comments can only be '//' nothing else and they have to appear at the beginning of the line. One regular expression can have more than one name, but one name can only have one regular expression. """ skip_whitespace(fh) if not check(fh, "{"): error.log("define region must start with opening '{'.", fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, "}"): return # -- get the name of the pattern skip_whitespace(fh) pattern_name = read_identifier( fh, OnMissingStr="Missing identifier for pattern definition.") if blackboard.shorthand_db.has_key(pattern_name): error.log("Second definition of pattern '%s'.\n" % pattern_name + \ "Pattern names must be unique.", fh) skip_whitespace(fh) if check(fh, "}"): error.log("Missing regular expression for pattern definition '%s'." % \ pattern_name, fh) # A regular expression state machine # (No possible transformation into a particular codec whatever. # the state machines are transformed once, after they are expanded # as patterns in a mode.) pattern = regular_expression.parse(fh, AllowNothingIsFineF=True) if pattern.has_pre_or_post_context(): error.log("Pattern definition with pre- and/or post-context.\n" + \ "Pre- and Post-Contexts can only be defined inside mode definitions.", fh) state_machine = pattern.sm blackboard.shorthand_db[pattern_name] = \ PatternShorthand(pattern_name, state_machine, SourceRef.from_FileHandle(fh), pattern.pattern_string())
def _read_value_specifier(fh, Keyword, Default=None): skip_whitespace(fh) value = read_integer(fh) if value is not None: return value # not a number received, is it an identifier? variable = read_identifier(fh) if variable != "": return variable elif Default is not None: return Default error.log("Missing integer or variable name after keyword '%s'." % Keyword, fh)
def parse_pattern_name_definitions(fh): """Parses pattern definitions of the form: WHITESPACE [ \t\n] IDENTIFIER [a-zA-Z0-9]+ OP_PLUS "+" That means: 'name' whitespace 'regular expression' whitespace newline. Comments can only be '//' nothing else and they have to appear at the beginning of the line. One regular expression can have more than one name, but one name can only have one regular expression. """ skip_whitespace(fh) if not check(fh, "{"): error.log("define region must start with opening '{'.", fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, "}"): return # -- get the name of the pattern skip_whitespace(fh) pattern_name = read_identifier(fh, OnMissingStr="Missing identifier for pattern definition.") if blackboard.shorthand_db.has_key(pattern_name): error.log("Second definition of pattern '%s'.\n" % pattern_name + \ "Pattern names must be unique.", fh) skip_whitespace(fh) if check(fh, "}"): error.log("Missing regular expression for pattern definition '%s'." % \ pattern_name, fh) # A regular expression state machine # (No possible transformation into a particular codec whatever. # the state machines are transformed once, after they are expanded # as patterns in a mode.) pattern = regular_expression.parse(fh, AllowNothingIsFineF = True) if pattern.has_pre_or_post_context(): error.log("Pattern definition with pre- and/or post-context.\n" + \ "Pre- and Post-Contexts can only be defined inside mode definitions.", fh) state_machine = pattern.sm blackboard.shorthand_db[pattern_name] = \ PatternShorthand(pattern_name, state_machine, SourceRef.from_FileHandle(fh), pattern.pattern_string())
def __parse_element(new_mode, fh): """Returns: False, if a closing '}' has been found. True, else. """ position = fh.tell() try: description = "pattern or event handler" skip_whitespace(fh) # NOTE: Do not use 'read_word' since we need to continue directly after # whitespace, if a regular expression is to be parsed. position = fh.tell() identifier = read_identifier(fh) if identifier == "keyword_list": return __parse_keyword_list(new_mode, fh) elif similarity.get(identifier, ["keyword_list", "key words"]) != -1: error.warning( "'%s' is similar to keyword 'keyword_list'.\n" "For clarity, use quotes." % identifier, fh) elif identifier == "brief": return __parse_brief(new_mode, fh) elif similarity.get(identifier, ["brief", "briefing", "briefly"]) != -1: error.warning( "'%s' is similar to keyword 'brief'.\n" "For clarity, use quotes." % identifier, fh) fh.seek(position) word = read_until_whitespace(fh) if word == "}": return False # -- check for 'on_entry', 'on_exit', ... elif __parse_event(new_mode, fh, word): return True fh.seek(position) description = "start of mode element: regular expression" pattern = regular_expression.parse(fh) pattern.set_source_reference( SourceRef.from_FileHandle(fh, new_mode.name)) position = fh.tell() description = "start of mode element: code fragment for '%s'" % pattern.pattern_string( ) __parse_action(new_mode, fh, pattern.pattern_string(), pattern) except EndOfStreamException: fh.seek(position) error.error_eof(description, fh) return True
def parse_pattern_name_definitions(fh): """Parses pattern definitions of the form: WHITESPACE [ \t\n] IDENTIFIER [a-zA-Z0-9]+ OP_PLUS "+" That means: 'name' whitespace 'regular expression' whitespace newline. Comments can only be '//' nothing else and they have to appear at the beginning of the line. One regular expression can have more than one name, but one name can only have one regular expression. """ skip_whitespace(fh) if not check(fh, "{"): error_msg("define region must start with opening '{'.", fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, "}"): return # -- get the name of the pattern skip_whitespace(fh) pattern_name = read_identifier(fh) if pattern_name == "": error_msg("Missing identifier for pattern definition.", fh) skip_whitespace(fh) if check(fh, "}"): error_msg("Missing regular expression for pattern definition '%s'." % \ pattern_name, fh) # A regular expression state machine # (No possible transformation into a particular codec whatever. # the state machines are transformed once, after they are expanded # as patterns in a mode.) regular_expression_str, pattern = \ regular_expression.parse(fh, AllowNothingIsFineF = True, AllowStateMachineTrafoF = False) if pattern.has_pre_or_post_context(): error_msg("Pattern definition with pre- and/or post-context.\n" + \ "Pre- and Post-Contexts can only be defined inside mode definitions.", fh) state_machine = pattern.sm blackboard.shorthand_db[pattern_name] = \ blackboard.PatternShorthand(pattern_name, state_machine, fh.name, get_current_line_info_number(fh), regular_expression_str)
def snap_replacement(stream, PatternDict, StateMachineF=True): """Snaps a predefined pattern from the input string and returns the resulting state machine. """ skip_whitespace(stream) pattern_name = read_identifier(stream) if pattern_name == "": raise RegularExpressionException("Pattern replacement expression misses identifier after '{'.") skip_whitespace(stream) if not check(stream, "}"): raise RegularExpressionException("Pattern replacement expression misses closing '}' after '%s'." \ % pattern_name) verify_word_in_list(pattern_name, PatternDict.keys(), "Specifier '%s' not found in any preceeding 'define { ... }' section." % pattern_name, stream) reference = PatternDict[pattern_name] assert reference.__class__.__name__ == "PatternShorthand" # The replacement may be a state machine or a number set if StateMachineF: # Get a cloned version of state machine state_machine = reference.get_state_machine() assert isinstance(state_machine, StateMachine) # It is essential that state machines defined as patterns do not # have origins. Otherwise, the optimization of patterns that # contain pattern replacements might get confused and can # not find all optimizations. assert state_machine.has_origins() == False # A state machine, that contains pre- or post- conditions cannot be part # of a replacement. The addition of new post-contexts would mess up the pattern. ## if state_machine.has_pre_or_post_context(): ## error_msg("Pre- or post-conditioned pattern was used in replacement.\n" + \ ## "Quex's regular expression grammar does not allow this.", stream) return state_machine else: # Get a cloned version of character set character_set = reference.get_character_set() if character_set is None: error_msg("Replacement in character set expression must be a character set.\n" "Specifier '%s' relates to a pattern state machine." % pattern_name, stream) if character_set.is_empty(): error_msg("Referenced character set '%s' is empty.\nAborted." % pattern_name, stream) return character_set
def read_option_start(fh): skip_whitespace(fh) # (*) base modes if fh.read(1) != "<": ##fh.seek(-1, 1) return None skip_whitespace(fh) identifier = read_identifier(fh, OnMissingStr="Missing identifer after start of mode option '<'").strip() skip_whitespace(fh) if fh.read(1) != ":": error.log("missing ':' after option name '%s'" % identifier, fh) skip_whitespace(fh) return identifier
def _parse_definition_head(fh, IdentifierList): if check(fh, "\\default"): error.log("'\\default' has been replaced by keyword '\\else' since quex 0.64.9!", fh) elif check(fh, "\\else"): pattern = None else: pattern = regular_expression.parse(fh) skip_whitespace(fh) check_or_die(fh, "=>", " after character set definition.") skip_whitespace(fh) identifier = read_identifier(fh, OnMissingStr="Missing identifier following '=>'.") error.verify_word_in_list(identifier, IdentifierList, "Unrecognized specifier '%s'." % identifier, fh) skip_whitespace(fh) return pattern, identifier, SourceRef.from_FileHandle(fh)
def read_option_start(fh): skip_whitespace(fh) # (*) base modes if fh.read(1) != "<": ##fh.seek(-1, 1) return None skip_whitespace(fh) identifier = read_identifier( fh, OnMissingStr="Missing identifer after start of mode option '<'").strip( ) skip_whitespace(fh) if fh.read(1) != ":": error.log("missing ':' after option name '%s'" % identifier, fh) skip_whitespace(fh) return identifier
def parse_section(fh): global default_token_type_definition_triggered_by_mode_definition_f # NOTE: End of File is supposed to be reached when trying to read a new # section. Thus, the end-of-file catcher does not encompass the beginning. position = fh.tell() skip_whitespace(fh) word = read_identifier(fh, OnMissingStr="Missing section title") error.verify_word_in_list(word, blackboard.all_section_title_list, "Unknown quex section '%s'" % word, fh) try: # (*) determine what is defined # # -- 'mode { ... }' => define a mode # -- 'start = ...;' => define the name of the initial mode # -- 'header { ... }' => define code that is to be pasted on top # of the engine (e.g. "#include<...>") # -- 'body { ... }' => define code that is to be pasted in the class' body # of the engine (e.g. "public: int my_member;") # -- 'init { ... }' => define code that is to be pasted in the class' constructors # of the engine (e.g. "my_member = -1;") # -- 'define { ... }' => define patterns shorthands such as IDENTIFIER for [a-z]+ # -- 'repeated_token_id = QUEX_TKN_ ...;' => enables token repetition, defines # the token id to be repeated. # -- 'token { ... }' => define token ids # -- 'token_type { ... }' => define a customized token type # if word in blackboard.fragment_db.keys(): element_name = blackboard.fragment_db[word] fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) blackboard.__dict__[element_name] = fragment return elif word == "start": mode_name = parse_identifier_assignment(fh) if mode_name == "": error.log("Missing mode_name after 'start ='", fh) elif not blackboard.initial_mode.sr.is_void(): error.log("start mode defined more than once!", fh, DontExitF=True) error.log("previously defined here", blackboard.initial_mode.sr) blackboard.initial_mode = CodeUser(mode_name, SourceRef.from_FileHandle(fh)) return elif word == "repeated_token": blackboard.token_repetition_token_id_list = parse_token_id_definitions( fh, NamesOnlyF=True) for token_name in blackboard.token_repetition_token_id_list: error.verify_word_in_list( token_name[len(Setup.token_id_prefix):], blackboard.token_id_db.keys(), "Token ID '%s' not yet defined." % token_name, fh, ExitF=False, SuppressCode=NotificationDB. warning_repeated_token_not_yet_defined) return elif word == "define": parse_pattern_name_definitions(fh) return elif word == "token": if Setup.token_id_foreign_definition: error.log("Token id file '%s' has been specified.\n" \ % Setup.token_id_foreign_definition_file \ + "All token ids must be specified there. Section 'token'\n" \ + "is not allowed.", fh) parse_token_id_definitions(fh) return elif word == "token_type": if Setup.token_class_file != "": error.log("Section 'token_type' is intended to generate a token class.\n" \ + "However, the manually written token class file '%s'" \ % repr(Setup.token_class_file) \ + "has been specified on the command line.", fh) if blackboard.token_type_definition is None: blackboard.token_type_definition = token_type.parse(fh) return # Error case: if default_token_type_definition_triggered_by_mode_definition_f: error.log( "Section 'token_type' must appear before first mode definition.", fh) else: error.log("Section 'token_type' has been defined twice.", fh, DontExitF=True) error.log("Previously defined here.", blackboard.token_type_definition.sr) return elif word == "mode": # When the first mode is parsed then a token_type definition must be # present. If not, the default token type definition is considered. if blackboard.token_type_definition is None: parse_default_token_definition() default_token_type_definition_triggered_by_mode_definition_f = True mode.parse(fh) return else: # This case should have been caught by the 'verify_word_in_list' function assert False except EndOfStreamException: fh.seek(position) error.error_eof(word, fh)
def parse_token_id_definitions(fh, NamesOnlyF=False): """NamesOnlyF == True: Allow only definition of names, no numeric values may be assigned to it. 'NamesOnlyF' indicates that data is not written to the global 'token_id_db'. Then only a list of names is returned. """ # NOTE: Catching of EOF happens in caller: parse_section(...) # prefix = Setup.token_id_prefix prefix_plain = Setup.token_id_prefix_plain # i.e. without name space included if NamesOnlyF: result = set() skip_whitespace(fh) if not check(fh, "{"): error.log("Missing opening '{' for after 'token' section identifier.", fh) while check(fh, "}") == False: skip_whitespace(fh) candidate = read_identifier( fh, TolerantF=True, OnMissingStr="Missing valid token identifier.") # -- check the name, if it starts with the token prefix paste a warning suspicious_prefix = None if len(prefix) != 0 and candidate.find(prefix) == 0: suspicious_prefix = prefix elif len(prefix_plain) != 0 and candidate.find(prefix_plain) == 0: suspicious_prefix = prefix_plain if suspicious_prefix is not None: error.warning("Token identifier '%s' starts with token prefix '%s'.\n" \ % (candidate, suspicious_prefix) \ + "Token prefix is mounted automatically. This token id appears in the source\n" \ + "code as '%s%s'." \ % (prefix, candidate), \ fh, SuppressCode=NotificationDB.warning_token_id_prefix_appears_in_token_id_name) skip_whitespace(fh) if NamesOnlyF: result.add(prefix + candidate) if check(fh, ";") == False: error.log("Missing ';' after token identifier '%s'.\n" \ % candidate, fh) continue # Parse a possible numeric value after '=' numeric_value = None if check(fh, "="): skip_whitespace(fh) numeric_value = read_integer(fh) if numeric_value is None: error.log( "Missing number after '=' for token identifier '%s'." % candidate, fh) if check(fh, ";") == False: error.log("Missing ';' after token identifier '%s'." % candidate, fh) if not NamesOnlyF: ti = TokenInfo(candidate, numeric_value, SourceReference=SourceRef.from_FileHandle(fh)) blackboard.token_id_db[candidate] = ti if NamesOnlyF: return sorted(list(result)) else: return # Changes are applied to 'blackboard.token_id_db'
def do(fh): """Parses pattern definitions of the form: [ \t] => grid 4; [:intersection([:alpha:], [\X064-\X066]):] => space 1; In other words the right hand side *must* be a character set. """ indentation_setup = IndentationSetup(fh) # NOTE: Catching of EOF happens in caller: parse_section(...) # skip_whitespace(fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, ">"): indentation_setup.seal() indentation_setup.consistency_check(fh) return indentation_setup # A regular expression state machine pattern_str, pattern = regular_expression.parse(fh) skip_whitespace(fh) if not check(fh, "=>"): error_msg("Missing '=>' after character set definition.", fh) skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": error_msg("Missing identifier for indentation element definition.", fh) verify_word_in_list(identifier, ["space", "grid", "bad", "newline", "suppressor"], "Unrecognized indentation specifier '%s'." % identifier, fh) trigger_set = None if identifier in ["space", "bad", "grid"]: if len(pattern.sm.states) != 2: error_msg("For indentation '%s' only patterns are addmissible which\n" % identifier + \ "can be matched by a single character, e.g. \" \" or [a-z].", fh) transition_map = pattern.sm.get_init_state().transitions().get_map() assert len(transition_map) == 1 trigger_set = transition_map.values()[0] skip_whitespace(fh) if identifier == "space": value = read_integer(fh) if value is not None: indentation_setup.specify_space(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? variable = read_identifier(fh) if variable != "": indentation_setup.specify_space(pattern_str, trigger_set, variable, fh) else: indentation_setup.specify_space(pattern_str, trigger_set, 1, fh) elif identifier == "grid": value = read_integer(fh) if value is not None: indentation_setup.specify_grid(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? skip_whitespace(fh) variable = read_identifier(fh) if variable != "": indentation_setup.specify_grid(pattern_str, trigger_set, variable, fh) else: error_msg("Missing integer or variable name after keyword 'grid'.", fh) elif identifier == "bad": indentation_setup.specify_bad(pattern_str, trigger_set, fh) elif identifier == "newline": indentation_setup.specify_newline(pattern_str, pattern.sm, fh) elif identifier == "suppressor": indentation_setup.specify_suppressor(pattern_str, pattern.sm, fh) else: assert False, "Unreachable code reached." if not check(fh, ";"): error_msg("Missing ';' after indentation '%s' specification." % identifier, fh)
def parse_section(fh): global default_token_type_definition_triggered_by_mode_definition_f # NOTE: End of File is supposed to be reached when trying to read a new # section. Thus, the end-of-file catcher does not encompass the beginning. position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": error_msg("Missing section title.", fh) verify_word_in_list(word, blackboard.all_section_title_list, "Unknown quex section '%s'" % word, fh) try: # (*) determine what is defined # # -- 'mode { ... }' => define a mode # -- 'start = ...;' => define the name of the initial mode # -- 'header { ... }' => define code that is to be pasted on top # of the engine (e.g. "#include<...>") # -- 'body { ... }' => define code that is to be pasted in the class' body # of the engine (e.g. "public: int my_member;") # -- 'init { ... }' => define code that is to be pasted in the class' constructors # of the engine (e.g. "my_member = -1;") # -- 'define { ... }' => define patterns shorthands such as IDENTIFIER for [a-z]+ # -- 'repeated_token_id = QUEX_TKN_ ...;' => enables token repetition, defines # the token id to be repeated. # -- 'token { ... }' => define token ids # -- 'token_type { ... }' => define a customized token type # if word in blackboard.fragment_db.keys(): element_name = blackboard.fragment_db[word] fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) blackboard.__dict__[element_name] = fragment return elif word == "start": mode_name = parse_identifier_assignment(fh) if mode_name == "": error_msg("Missing mode_name after 'start ='", fh) elif blackboard.initial_mode.get_pure_code() != "": error_msg("start mode defined more than once!", fh, DontExitF=True) error_msg("previously defined here", blackboard.initial_mode.filename, blackboard.initial_mode.line_n) blackboard.initial_mode = UserCodeFragment(mode_name, fh.name, get_current_line_info_number(fh)) return elif word == "repeated_token": blackboard.token_repetition_token_id_list = parse_token_id_definitions(fh, NamesOnlyF=True) for token_name in blackboard.token_repetition_token_id_list: verify_word_in_list(token_name[len(Setup.token_id_prefix):], blackboard.token_id_db.keys(), "Token ID '%s' not yet defined." % token_name, fh, ExitF=False) return elif word == "define": parse_pattern_name_definitions(fh) return elif word == "token": parse_token_id_definitions(fh) return elif word == "token_type": if Setup.token_class_file != "": error_msg("Token type definition inadmissible while specifying on the command line\n" + \ "the file %s to contain a manually written token class." % repr(Setup.token_class_file), fh) if blackboard.token_type_definition is None: blackboard.token_type_definition = token_type.parse(fh) return # Error case: if default_token_type_definition_triggered_by_mode_definition_f: error_msg("Section 'token_type' must appear before first mode definition.", fh) else: error_msg("Section 'token_type' has been defined twice.", fh, DontExitF=True) error_msg("Previously defined here.", blackboard.token_type_definition.file_name_of_token_type_definition, blackboard.token_type_definition.line_n_of_token_type_definition) return elif word == "mode": # When the first mode is parsed then a token_type definition must be # present. If not, the default token type definition is considered. if blackboard.token_type_definition is None: parse_default_token_definition() default_token_type_definition_triggered_by_mode_definition_f = True mode.parse(fh) return else: # This case should have been caught by the 'verify_word_in_list' function assert False except EndOfStreamException: fh.seek(position) error_msg("End of file reached while parsing '%s' section" % word, fh)
def parse_variable_definition(fh, GroupF=False, already_defined_list=[]): """PURPOSE: Parsing of a variable definition consisting of 'type' and 'name. Members can be mentioned together in a group, which means that they can appear simultaneously. Possible expresions are (1) single variables: name0 : type; name1 : type[32]; name2 : type*; (2) combined variables { sub_name0 : type0; sub_name1 : type[64]; sub_name2 : type1*; } ARGUMENTS: 'GroupF' allows to have 'nested variable groups' in curly brackets 'already_defined_list' informs about variable names that have been already chosen. It is only used for groups. RETURNS: None on failure to pass a variable definition. array when a single variable definition was found. array[0] = UserCodeFragment containing the type. array[1] = name of the variable. dictionary if it was a combined variable definition. The dictionary maps: (variable name) ---> (UserCodeFragment with type) """ position = fh.tell() skip_whitespace(fh) name_str = read_identifier(fh) if name_str == "": if not GroupF or not check(fh, "{"): fh.seek(position); return None sub_db = parse_variable_definition_list(fh, "Concurrent union variables", already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg("Missing closing '}' after concurrent variable definition.", fh) return [ sub_db ] else: name_str = name_str.strip() if not check(fh, ":"): error_msg("Missing ':' after identifier '%s'." % name_str, fh) if fh.read(1).isspace() == False: error_msg("Missing whitespace after ':' after identifier '%s'.\n" % name_str \ + "The notation has to be: variable-name ':' type ';'.", fh) type_str, i = read_until_letter(fh, ";", Verbose=True) if i == -1: error_msg("missing ';'", fh) type_str = type_str.strip() return [ CodeUser(type_str, SourceRef.from_FileHandle(fh)), name_str ]
def parse_token_id_definitions(fh, NamesOnlyF=False): # NOTE: Catching of EOF happens in caller: parse_section(...) # token_prefix = Setup.token_id_prefix token_prefix_plain = Setup.token_id_prefix_plain # i.e. without name space included if NamesOnlyF: db = {} else: db = blackboard.token_id_db skip_whitespace(fh) if not check(fh, "{"): error_msg( "missing opening '{' for after 'token' section identifier.\n", fh) while check(fh, "}") == False: skip_whitespace(fh) candidate = read_identifier(fh, TolerantF=True) if candidate == "": error_msg("Missing valid token identifier." % candidate, fh) # -- check the name, if it starts with the token prefix paste a warning if candidate.find(token_prefix) == 0: error_msg("Token identifier '%s' starts with token prefix '%s'.\n" % (candidate, token_prefix) + \ "Token prefix is mounted automatically. This token id appears in the source\n" + \ "code as '%s%s'." % (token_prefix, candidate), \ fh, DontExitF=True) elif candidate.find(token_prefix_plain) == 0: error_msg("Token identifier '%s' starts with token prefix '%s'.\n" % (candidate, token_prefix) + \ "Token prefix is mounted automatically. This token id appears in the source\n" + \ "code as '%s%s'." % (token_prefix, candidate), \ fh, DontExitF=True) skip_whitespace(fh) if NamesOnlyF: db[token_prefix + candidate] = True if check(fh, ";") == False: error_msg("Missing ';' after definition of token identifier '%s'.\n" % candidate + \ "This is mandatory since Quex version 0.50.1.", fh) continue # Parse a possible numeric value after '=' numeric_value = None if check(fh, "="): skip_whitespace(fh) numeric_value = read_integer(fh) if numeric_value is None: error_msg( "Missing number after '=' for token identifier '%s'." % candidate, fh) if check(fh, ";") == False: error_msg("Missing ';' after definition of token identifier '%s'.\n" % candidate + \ "This is mandatory since Quex version 0.50.1.", fh) db[candidate] = TokenInfo(candidate, numeric_value, Filename=fh.name, LineN=get_current_line_info_number(fh)) if NamesOnlyF: result = db.keys() result.sort() return result
def parse_section(fh): global default_token_type_definition_triggered_by_mode_definition_f # NOTE: End of File is supposed to be reached when trying to read a new # section. Thus, the end-of-file catcher does not encompass the beginning. position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": error_msg("Missing section title.", fh) verify_word_in_list(word, blackboard.all_section_title_list, "Unknown quex section '%s'" % word, fh) try: # (*) determine what is defined # # -- 'mode { ... }' => define a mode # -- 'start = ...;' => define the name of the initial mode # -- 'header { ... }' => define code that is to be pasted on top # of the engine (e.g. "#include<...>") # -- 'body { ... }' => define code that is to be pasted in the class' body # of the engine (e.g. "public: int my_member;") # -- 'init { ... }' => define code that is to be pasted in the class' constructors # of the engine (e.g. "my_member = -1;") # -- 'define { ... }' => define patterns shorthands such as IDENTIFIER for [a-z]+ # -- 'repeated_token_id = QUEX_TKN_ ...;' => enables token repetition, defines # the token id to be repeated. # -- 'token { ... }' => define token ids # -- 'token_type { ... }' => define a customized token type # if word in blackboard.fragment_db.keys(): element_name = blackboard.fragment_db[word] fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) blackboard.__dict__[element_name] = fragment return elif word == "start": mode_name = parse_identifier_assignment(fh) if mode_name == "": error_msg("Missing mode_name after 'start ='", fh) elif blackboard.initial_mode.get_pure_code() != "": error_msg("start mode defined more than once!", fh, DontExitF=True) error_msg("previously defined here", blackboard.initial_mode.filename, blackboard.initial_mode.line_n) blackboard.initial_mode = UserCodeFragment( mode_name, fh.name, get_current_line_info_number(fh)) return elif word == "repeated_token": blackboard.token_repetition_token_id_list = parse_token_id_definitions( fh, NamesOnlyF=True) for token_name in blackboard.token_repetition_token_id_list: verify_word_in_list(token_name[len(Setup.token_id_prefix):], blackboard.token_id_db.keys(), "Token ID '%s' not yet defined." % token_name, fh, ExitF=False) return elif word == "define": parse_pattern_name_definitions(fh) return elif word == "token": parse_token_id_definitions(fh) return elif word == "token_type": if Setup.token_class_file != "": error_msg("Token type definition inadmissible while specifying on the command line\n" + \ "the file %s to contain a manually written token class." % repr(Setup.token_class_file), fh) if blackboard.token_type_definition is None: blackboard.token_type_definition = token_type.parse(fh) return # Error case: if default_token_type_definition_triggered_by_mode_definition_f: error_msg( "Section 'token_type' must appear before first mode definition.", fh) else: error_msg("Section 'token_type' has been defined twice.", fh, DontExitF=True) error_msg( "Previously defined here.", blackboard. token_type_definition.file_name_of_token_type_definition, blackboard.token_type_definition. line_n_of_token_type_definition) return elif word == "mode": # When the first mode is parsed then a token_type definition must be # present. If not, the default token type definition is considered. if blackboard.token_type_definition is None: parse_default_token_definition() default_token_type_definition_triggered_by_mode_definition_f = True mode.parse(fh) return else: # This case should have been caught by the 'verify_word_in_list' function assert False except EndOfStreamException: fh.seek(position) error_msg("End of file reached while parsing '%s' section" % word, fh)
def parse_section(fh): global default_token_type_definition_triggered_by_mode_definition_f # NOTE: End of File is supposed to be reached when trying to read a new # section. Thus, the end-of-file catcher does not encompass the beginning. position = fh.tell() skip_whitespace(fh) word = read_identifier(fh, OnMissingStr="Missing section title") error.verify_word_in_list(word, blackboard.all_section_title_list, "Unknown quex section '%s'" % word, fh) try: # (*) determine what is defined # # -- 'mode { ... }' => define a mode # -- 'start = ...;' => define the name of the initial mode # -- 'header { ... }' => define code that is to be pasted on top # of the engine (e.g. "#include<...>") # -- 'body { ... }' => define code that is to be pasted in the class' body # of the engine (e.g. "public: int my_member;") # -- 'init { ... }' => define code that is to be pasted in the class' constructors # of the engine (e.g. "my_member = -1;") # -- 'define { ... }' => define patterns shorthands such as IDENTIFIER for [a-z]+ # -- 'repeated_token_id = QUEX_TKN_ ...;' => enables token repetition, defines # the token id to be repeated. # -- 'token { ... }' => define token ids # -- 'token_type { ... }' => define a customized token type # if word in blackboard.fragment_db.keys(): element_name = blackboard.fragment_db[word] fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) blackboard.__dict__[element_name] = fragment return elif word == "start": mode_name = parse_identifier_assignment(fh) if mode_name == "": error.log("Missing mode_name after 'start ='", fh) elif not blackboard.initial_mode.sr.is_void(): error.log("start mode defined more than once!", fh, DontExitF=True) error.log("previously defined here", blackboard.initial_mode.sr) blackboard.initial_mode = CodeUser(mode_name, SourceRef.from_FileHandle(fh)) return elif word == "repeated_token": blackboard.token_repetition_token_id_list = parse_token_id_definitions(fh, NamesOnlyF=True) for token_name in blackboard.token_repetition_token_id_list: error.verify_word_in_list(token_name[len(Setup.token_id_prefix):], blackboard.token_id_db.keys(), "Token ID '%s' not yet defined." % token_name, fh, ExitF=False, SuppressCode=NotificationDB.warning_repeated_token_not_yet_defined) return elif word == "define": parse_pattern_name_definitions(fh) return elif word == "token": if Setup.token_id_foreign_definition: error.log("Token id file '%s' has been specified.\n" \ % Setup.token_id_foreign_definition_file \ + "All token ids must be specified there. Section 'token'\n" \ + "is not allowed.", fh) parse_token_id_definitions(fh) return elif word == "token_type": if Setup.token_class_file != "": error.log("Section 'token_type' is intended to generate a token class.\n" \ + "However, the manually written token class file '%s'" \ % repr(Setup.token_class_file) \ + "has been specified on the command line.", fh) if blackboard.token_type_definition is None: blackboard.token_type_definition = token_type.parse(fh) return # Error case: if default_token_type_definition_triggered_by_mode_definition_f: error.log("Section 'token_type' must appear before first mode definition.", fh) else: error.log("Section 'token_type' has been defined twice.", fh, DontExitF=True) error.log("Previously defined here.", blackboard.token_type_definition.sr) return elif word == "mode": # When the first mode is parsed then a token_type definition must be # present. If not, the default token type definition is considered. if blackboard.token_type_definition is None: parse_default_token_definition() default_token_type_definition_triggered_by_mode_definition_f = True mode.parse(fh) return else: # This case should have been caught by the 'verify_word_in_list' function assert False except EndOfStreamException: fh.seek(position) error.error_eof(word, fh)
def parse_token_id_definitions(fh, NamesOnlyF=False): """NamesOnlyF == True: Allow only definition of names, no numeric values may be assigned to it. 'NamesOnlyF' indicates that data is not written to the global 'token_id_db'. Then only a list of names is returned. """ # NOTE: Catching of EOF happens in caller: parse_section(...) # prefix = Setup.token_id_prefix prefix_plain = Setup.token_id_prefix_plain # i.e. without name space included if NamesOnlyF: result = set() skip_whitespace(fh) if not check(fh, "{"): error.log("Missing opening '{' for after 'token' section identifier.", fh) while check(fh, "}") == False: skip_whitespace(fh) candidate = read_identifier(fh, TolerantF=True, OnMissingStr="Missing valid token identifier.") # -- check the name, if it starts with the token prefix paste a warning suspicious_prefix = None if len(prefix) != 0 and candidate.find(prefix) == 0: suspicious_prefix = prefix elif len(prefix_plain) != 0 and candidate.find(prefix_plain) == 0: suspicious_prefix = prefix_plain if suspicious_prefix is not None: error.warning("Token identifier '%s' starts with token prefix '%s'.\n" \ % (candidate, suspicious_prefix) \ + "Token prefix is mounted automatically. This token id appears in the source\n" \ + "code as '%s%s'." \ % (prefix, candidate), \ fh, SuppressCode=NotificationDB.warning_token_id_prefix_appears_in_token_id_name) skip_whitespace(fh) if NamesOnlyF: result.add(prefix + candidate) if check(fh, ";") == False: error.log("Missing ';' after token identifier '%s'.\n" \ % candidate, fh) continue # Parse a possible numeric value after '=' numeric_value = None if check(fh, "="): skip_whitespace(fh) numeric_value = read_integer(fh) if numeric_value is None: error.log("Missing number after '=' for token identifier '%s'." % candidate, fh) if check(fh, ";") == False: error.log("Missing ';' after token identifier '%s'." % candidate, fh) if not NamesOnlyF: ti = TokenInfo(candidate, numeric_value, SourceReference=SourceRef.from_FileHandle(fh)) blackboard.token_id_db[candidate] = ti if NamesOnlyF: return sorted(list(result)) else: return # Changes are applied to 'blackboard.token_id_db'
def do(fh): """Parses pattern definitions of the form: [ \t] => grid 4; [:intersection([:alpha:], [\X064-\X066]):] => space 1; In other words the right hand side *must* be a character set. """ indentation_setup = IndentationSetup(fh) # NOTE: Catching of EOF happens in caller: parse_section(...) # skip_whitespace(fh) while 1 + 1 == 2: skip_whitespace(fh) if check(fh, ">"): indentation_setup.seal() indentation_setup.consistency_check(fh) return indentation_setup # A regular expression state machine pattern_str, pattern = regular_expression.parse(fh) skip_whitespace(fh) if not check(fh, "=>"): error_msg("Missing '=>' after character set definition.", fh) skip_whitespace(fh) identifier = read_identifier(fh) if identifier == "": error_msg("Missing identifier for indentation element definition.", fh) verify_word_in_list( identifier, ["space", "grid", "bad", "newline", "suppressor"], "Unrecognized indentation specifier '%s'." % identifier, fh) trigger_set = None if identifier in ["space", "bad", "grid"]: if len(pattern.sm.states) != 2: error_msg("For indentation '%s' only patterns are addmissible which\n" % identifier + \ "can be matched by a single character, e.g. \" \" or [a-z].", fh) transition_map = pattern.sm.get_init_state().transitions().get_map( ) assert len(transition_map) == 1 trigger_set = transition_map.values()[0] skip_whitespace(fh) if identifier == "space": value = read_integer(fh) if value is not None: indentation_setup.specify_space(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? variable = read_identifier(fh) if variable != "": indentation_setup.specify_space(pattern_str, trigger_set, variable, fh) else: indentation_setup.specify_space(pattern_str, trigger_set, 1, fh) elif identifier == "grid": value = read_integer(fh) if value is not None: indentation_setup.specify_grid(pattern_str, trigger_set, value, fh) else: # not a number received, is it an identifier? skip_whitespace(fh) variable = read_identifier(fh) if variable != "": indentation_setup.specify_grid(pattern_str, trigger_set, variable, fh) else: error_msg( "Missing integer or variable name after keyword 'grid'.", fh) elif identifier == "bad": indentation_setup.specify_bad(pattern_str, trigger_set, fh) elif identifier == "newline": indentation_setup.specify_newline(pattern_str, pattern.sm, fh) elif identifier == "suppressor": indentation_setup.specify_suppressor(pattern_str, pattern.sm, fh) else: assert False, "Unreachable code reached." if not check(fh, ";"): error_msg( "Missing ';' after indentation '%s' specification." % identifier, fh)
def __parse_section(fh, descriptor, already_defined_list): global token_type_code_fragment_db assert type(already_defined_list) == list SubsectionList = ["name", "file_name", "standard", "distinct", "union", "inheritable", "noid"] \ + token_type_code_fragment_db.keys() position = fh.tell() skip_whitespace(fh) word = read_identifier(fh) if word == "": fh.seek(position) if check(fh, "}"): fh.seek(position) return False error_msg( "Missing token_type section ('standard', 'distinct', or 'union').", fh) verify_word_in_list( word, SubsectionList, "Subsection '%s' not allowed in token_type section." % word, fh) if word == "name": if not check(fh, "="): error_msg("Missing '=' in token_type 'name' specification.", fh) descriptor.class_name, descriptor.name_space, descriptor.class_name_safe = read_namespaced_name( fh, "token_type") if not check(fh, ";"): error_msg( "Missing terminating ';' in token_type 'name' specification.", fh) elif word == "inheritable": descriptor.open_for_derivation_f = True check_or_die(fh, ";") elif word == "noid": descriptor.token_contains_token_id_f = False check_or_die(fh, ";") elif word == "file_name": if not check(fh, "="): error_msg("Missing '=' in token_type 'file_name' specification.", fh) descriptor.set_file_name(read_until_letter(fh, ";")) if not check(fh, ";"): error_msg( "Missing terminating ';' in token_type 'file_name' specification.", fh) elif word in ["standard", "distinct", "union"]: if word == "standard": parse_standard_members(fh, word, descriptor, already_defined_list) elif word == "distinct": parse_distinct_members(fh, word, descriptor, already_defined_list) elif word == "union": parse_union_members(fh, word, descriptor, already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg( "Missing closing '}' at end of token_type section '%s'." % word, fh) elif word in token_type_code_fragment_db.keys(): fragment = code_fragment.parse(fh, word, AllowBriefTokenSenderF=False) descriptor.__dict__[word] = fragment else: assert False, "This code section section should not be reachable because 'word'\n" + \ "was checked to fit in one of the 'elif' cases." return True
def parse_variable_definition(fh, GroupF=False, already_defined_list=[]): """PURPOSE: Parsing of a variable definition consisting of 'type' and 'name. Members can be mentioned together in a group, which means that they can appear simultaneously. Possible expresions are (1) single variables: name0 : type; name1 : type[32]; name2 : type*; (2) combined variables { sub_name0 : type0; sub_name1 : type[64]; sub_name2 : type1*; } ARGUMENTS: 'GroupF' allows to have 'nested variable groups' in curly brackets 'already_defined_list' informs about variable names that have been already chosen. It is only used for groups. RETURNS: None on failure to pass a variable definition. array when a single variable definition was found. array[0] = UserCodeFragment containing the type. array[1] = name of the variable. dictionary if it was a combined variable definition. The dictionary maps: (variable name) ---> (UserCodeFragment with type) """ position = fh.tell() skip_whitespace(fh) name_str = read_identifier(fh) if name_str == "": if not GroupF or not check(fh, "{"): fh.seek(position) return None sub_db = parse_variable_definition_list(fh, "Concurrent union variables", already_defined_list) if not check(fh, "}"): fh.seek(position) error_msg( "Missing closing '}' after concurrent variable definition.", fh) return [sub_db] else: name_str = name_str.strip() if not check(fh, ":"): error_msg("Missing ':' after identifier '%s'." % name_str, fh) if fh.read(1).isspace() == False: error_msg("Missing whitespace after ':' after identifier '%s'.\n" % name_str \ + "The notation has to be: variable-name ':' type ';'.", fh) type_str, i = read_until_letter(fh, ";", Verbose=True) if i == -1: error_msg("missing ';'", fh) type_str = type_str.strip() return [CodeUser(type_str, SourceRef.from_FileHandle(fh)), name_str]