def get_sm_list(FSM0, FSM1, FSM2): # SPECIALITIES: -- sm0 and sm1 have an intersection between their second # transition. # -- sm1 transits further upon acceptance. # -- sm2 has only one transition. # Generate DFA that does not have any intersection with # the loop transitions. sm0 = DFA() si = sm0.add_transition(sm0.init_state_index, FSM0) si = sm0.add_transition(si, NS_A, AcceptanceF=True) sm0.states[si].mark_acceptance_id(dial.new_incidence_id()) sm1 = DFA() si0 = sm1.add_transition(sm1.init_state_index, FSM1) si = sm1.add_transition(si0, NS_A, AcceptanceF=True) iid1 = dial.new_incidence_id() sm1.states[si].mark_acceptance_id(iid1) si = sm1.add_transition(si, NS_B, si0) sm1.states[si].mark_acceptance_id(iid1) sm2 = DFA() si = sm2.add_transition(sm2.init_state_index, FSM2, AcceptanceF=True) sm2.states[si].mark_acceptance_id(dial.new_incidence_id()) return [sm0, sm1, sm2]
def test(TestString): print "expression = \"" + TestString + "\"" sm = DFA() try: trigger_set = character_set.do(StringIO(TestString + "]")) sm.add_transition(sm.init_state_index, trigger_set, AcceptanceF=True) print "state machine\n", sm except RegularExpressionException, x: print repr(x)
def snap_non_control_character(stream, PatternDict): __debug_entry("non-control characters", stream) # (*) read first character char_code = utf8.__read_one_utf8_code_from_stream(stream) if char_code is None: error.log( "Character could not be interpreted as UTF8 code or End of File reached prematurely.", stream) result = DFA() result.add_transition(result.init_state_index, char_code, AcceptanceF=True) return __debug_exit(result, stream)
def test_on_UCS_range(Trafo, Source, Drain, CharacterBackwardTrafo): sm = DFA() acc_db = {} for x in range(Source.begin, Source.end): ti = sm.add_transition(sm.init_state_index, x, AcceptanceF=True) acc_id = len(acc_db) sm.states[ti].mark_acceptance_id(acc_id) acc_db[x] = acc_id if Setup.bad_lexatom_detection_f: acc_db[None] = E_IncidenceIDs.BAD_LEXATOM else: acc_db[None] = None state_n_before, result = transform(Trafo, sm) # assert state_n_before == len(result.states) init_state = result.get_init_state() count = 0 for y in range(Drain.begin, Drain.end): # Translate character into x = CharacterBackwardTrafo(y) # Transit on the translated charater ti = init_state.target_map.get_resulting_target_state_index(y) # Compare resulting state with the expected state's acceptance assert_only_acceptance_id(sm.states, ti, acc_db, x, y) count += 1 print "<terminated: %i transitions ok>" % count
def do(sh): """Converts a uni-code string into a state machine that parses its letters sequentially. Each state in the sequence correponds to the sucessful triggering of a letter. Only the last state, though, is an acceptance state. Any bailing out before is 'not accepted'. Example: "hey" is translated into the state machine: (0)-- 'h' -->(1)-- 'e' -->(2)-- 'y' --> ACCEPTANCE | | | FAIL FAIL FAIL Note: The state indices are globally unique. But, they are not necessarily 0, 1, 2, ... """ assert sh.__class__.__name__ == "StringIO" \ or sh.__class__.__name__ == "file" # resulting state machine result = DFA() state_idx = result.init_state_index # Only \" is a special character '"', any other backslashed character # remains as the sequence 'backslash' + character for char_code in get_character_code_sequence(sh): state_idx = result.add_transition(state_idx, char_code) # when the last state has trigger it is supposed to end up in 'acceptance' result.states[state_idx].set_acceptance() return result
def create_ALL_BUT_NEWLINE_state_machine(stream): global Setup result = DFA() # NOTE: Buffer control characters are supposed to be filtered out by the code # generator. trigger_set = NumberSet(Interval(ord("\n"))).get_complement( Setup.buffer_encoding.source_set) if trigger_set.is_empty(): error.log( "The set of admissible characters contains only newline.\n" "The '.' for 'all but newline' is an empty set.", SourceRef.from_FileHandle(stream)) result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True) return result
def snap_character_set_expression(stream, PatternDict): # GRAMMAR: # # set_expression: # [: set_term :] # traditional character set # \P '{' propperty string '}' # '{' identifier '}' # # set_term: # "alnum" # "alpha" # "blank" # "cntrl" # "digit" # "graph" # "lower" # "print" # "punct" # "space" # "upper" # "xdigit" # "union" '(' set_term [ ',' set_term ]+ ')' # "intersection" '(' set_term [ ',' set_term ]+ ')' # "difference" '(' set_term [ ',' set_term ]+ ')' # "complement" '(' set_term ')' # set_expression # trigger_set = snap_set_expression(stream, PatternDict) if trigger_set is None: error.log("Regular Expression: snap_character_set_expression called for something\n" + \ "that does not start with '[:', '[' or '\\P'", stream) elif trigger_set.is_empty(): error.warning( "Regular Expression: Character set expression results in empty set.", stream) # Create state machine that triggers with the trigger set to SUCCESS # NOTE: The default for the ELSE transition is FAIL. sm = DFA() sm.add_transition(sm.init_state_index, trigger_set, AcceptanceF=True) return __debug_exit(sm, stream)
def do(SM_List): for sm in SM_List: sm.assert_consistency() if any(sm.is_Empty() for sm in SM_List): # If one state machine is '\Empty', return DFA.Empty() # then the intersection is '\Empty'. init_state_setup = tuple(sm.init_state_index for sm in SM_List) result = DFA(AcceptanceF=intersect_acceptance(init_state_setup, SM_List)) # Result state setup: A result state is setup out of a state from each DFA. # state_setup[i] is the state from DFA 'SM_List[i]'. worklist = [ (result.init_state_index, init_state_setup) ] state_setup_db = {} N = len(SM_List) while worklist: state_index, state_setup = worklist.pop() # Generate Map that shows what lexatoms trigger to what state combination. # # NumberSet Target DFA_State Combination # [0:23] --> [ State1, State24, State56 ] # [0:23] --> [ State5, State21, State55 ] # [24:60] --> [ State1, State23, State51 ] # # 'get_intersection_line_up()' only delivers those transitions where there # is a transition for each state machine's state. line_up = get_intersection_line_up([SM_List[i].states[si].target_map for i, si in enumerate(state_setup)]) for target_state_setup, trigger_set in line_up.iteritems(): assert len(target_state_setup) == N target_index, new_f = state_index_for_combination(state_setup_db, target_state_setup) acceptance_f = intersect_acceptance(target_state_setup, SM_List) result.add_transition(state_index, trigger_set, target_index, AcceptanceF = acceptance_f) if new_f: worklist.append((target_index, target_state_setup)) result.delete_hopeless_states() return result
class X: def __init__(self, Name): sh = StringIO("[:\\P{Script=%s}:]" % Name) self.name = Name self.charset = regex.snap_set_expression(sh, {}) self.sm = DFA() self.sm.add_transition(self.sm.init_state_index, self.charset, AcceptanceF=True) self.id = self.sm.get_id() def check(self, SM, TransformFunc): """This function throws an exception as soon as one single value is not matched according to the expectation. """ print "## [%i] Name = %s" % (self.id, self.name), interval_list = self.charset.get_intervals(PromiseToTreatWellF=True) interval_count = len(interval_list) for interval in interval_list: for i in range(interval.begin, interval.end): lexatom_seq = TransformFunc(i) # Apply sequence to state machine state = SM.apply_sequence(lexatom_seq) if state is None: error(self.sm, SM, lexatom_seq) # All acceptance flags must belong to the original state machine acceptance_id_list = [ cmd.acceptance_id() for cmd in state.single_entry.get_iterable(SeAccept) ] if acceptance_id_list and self.id not in acceptance_id_list: print eval("u'\U%08X'" % i) print "#Seq: ", ["%02X" % x for x in lexatom_seq] print "#acceptance-ids:", acceptance_id_list error(self.sm, SM, lexatom_seq) print " (OK=%i)" % interval_count
def DFA_Newline(): """Creates a state machine matching newline according to what has been specified in the setup (Setup.dos_carriage_return_newline_f). That is, if is DOS newline then the state machine represents '\r\n' and if it is unix only, then it represents '\n'. If both is required they are implemented in parallel. RETURNS: DFA """ UnixF = True DosF = Setup.dos_carriage_return_newline_f NL = ord('\n') # (pure) newline, i.e. line feed CR = ord('\r') # carriage return dfa = DFA() if UnixF: dfa.add_transition(dfa.init_state_index, NL, AcceptanceF=True) if DosF: idx = dfa.add_transition(dfa.init_state_index, CR, AcceptanceF=False) dfa.add_transition(idx, NL, AcceptanceF=True) return beautifier.do(dfa)
ca_list = get_ca_list(0x10, 0x60) sm_list = get_sm_list(NumberSet.from_range(0x10, 0x40), NumberSet.from_range(0x20, 0x50), NumberSet.from_range(0x30, 0x60)) # Test for each 'sm' in 'sm_list' is superfluous. # It is done in 'AppendixNoI'. test(ca_list, sm_list) elif "Split" in sys.argv: # A first transition of a state machine is separated into two, because # it is covered by more than one different count action. NS1 = NumberSet.from_range(0x10, 0x20) NS2 = NumberSet.from_range(0x20, 0x30) NS3 = NumberSet.from_range(0x30, 0x40) NS4 = NumberSet.from_range(0x40, 0x50) ca_list = [ (NS1, CountAction(E_CharacterCountType.COLUMN, 1)), (NS2, CountAction(E_CharacterCountType.COLUMN, 2)), (NS3, CountAction(E_CharacterCountType.COLUMN, 3)), (NS4, CountAction(E_CharacterCountType.COLUMN, 4)), ] sm = DFA() si = sm.init_state_index iid = dial.new_incidence_id() ti0 = sm.add_transition(si, NumberSet.from_range(0x1A, 0x4B)) ac0 = sm.add_transition(ti0, NS_A, AcceptanceF=True) test(ca_list, [sm])
def snap_primary(stream, PatternDict): """primary: " non_double_quote * " = character string [ non_rect_bracket_close ] = set of characters { identifier } = pattern replacement ( expression ) non_control_character+ = lonely characters primary repetition_cmd """ global SPECIAL_TERMINATOR __debug_entry("primary", stream) x = stream.read(1) if x == "": return __debug_exit(None, stream) # -- 'primary' primary if x == "\"": result = snap_character_string.do(stream) elif x == "[": stream.seek(-1, 1) result = snap_character_set_expression(stream, PatternDict) elif x == "{": result = snap_replacement(stream, PatternDict) elif x == ".": result = create_ALL_BUT_NEWLINE_state_machine(stream) elif x == "(": result = snap_bracketed_expression(stream, PatternDict) elif x.isspace(): # a lonestanding space ends the regular expression stream.seek(-1, 1) return __debug_exit(None, stream) elif x in ["*", "+", "?"]: raise RegularExpressionException( "lonely operator '%s' without expression proceeding." % x) elif x == "\\": result = snap_command(stream, PatternDict) if result is None: stream.seek(-1, 1) trigger_set = snap_property_set(stream) if trigger_set is None: # snap the '\' stream.read(1) char_code = snap_backslashed_character.do(stream) if char_code is None: raise RegularExpressionException( "Backslash followed by unrecognized character code.") trigger_set = char_code result = DFA() result.add_transition(result.init_state_index, trigger_set, AcceptanceF=True) elif x not in CONTROL_CHARACTERS and x != SPECIAL_TERMINATOR: # NOTE: The '\' is not inside the control characters---for a reason. # It is used to define for example character codes using '\x' etc. stream.seek(-1, 1) result = snap_non_control_character(stream, PatternDict) else: # NOTE: This includes the '$' sign which means 'end of line' # because the '$' sign is in CONTROL_CHARACTERS, but is not checked # against. Thus, it it good to leave here on '$' because the # '$' sign is handled on the very top level. # this is not a valid primary stream.seek(-1, 1) return __debug_exit(None, stream) # -- optional repetition command? result_repeated = __snap_repetition_range(result, stream) if result_repeated is not None: result = result_repeated return __debug_exit(result, stream)
def test(A_txt, B_txt): """Performs: \CutBegin{P Q} and prints the result! """ print "---------------------------" A, B = parse_REs(A_txt, B_txt) result = core(A, B) print print "result = ", result.get_string(NormalizeF=True) if False: # Selected Test dfa = DFA() dfa.add_transition(dfa.init_state_index, 1, AcceptanceF=True) set_unique_transition_f() # for name in get_sm_shape_names_list(): if True: name = "DEBUG" Q = get_sm_shape_by_name_with_acceptance(name) print "# %s" % name, Q cut_Q_Q = __operation(Q, Q) # \CutBegin{Q Q} = \Empty print "#cut_Q_Q:", cut_Q_Q print "#verdict:", cut_Q_Q.is_Nothing() try: assert cut_Q_Q.is_Nothing() except: