def do(SM): """RETURNS: A state machines that matches anything which is not matched by SM. Idea: The paths along SM do not guide to acceptance states, but to normal states. Any drop-out is translated into a transition into the 'accept all state'. NOTE: This function produces a finite state automaton which is not applicable by itself. It would eat ANYTHING from a certain state on. """ result = deepcopy(SM) # Not clone accept_all_state_index = index.get() state = State(AcceptanceF=True) state.add_transition(NumberSet_All(), accept_all_state_index) result.states[accept_all_state_index] = state def is_accept_all_state(sm, StateIndex): state = sm.states[StateIndex] if not state.is_acceptance(): return False tm = state.target_map.get_map() if len(tm) != 1: return False elif tm.iterkeys().next() != StateIndex: return False elif not tm.itervalues().next().is_all(): return False # Target is an 'Accept-All' state. Delete the transition. return True for state_index, state in SM.states.iteritems(): # deepcopy --> use same state indices in SM and result result_state = result.states[state_index] assert state.target_map.is_DFA_compliant(), \ "State machine must be transformed to DFA first: nfa_to_dfa.do()" # -- Every transition to 'Accept-All' state becomes a drop-out. for target_index in (i for i in state.target_map.get_target_state_index_list() if is_accept_all_state(SM, i)): result_state.target_map.delete_transitions_to_target(target_index) # -- Every drop-out becomes a transition to 'Accept-All' state. trigger_set = state.target_map.get_trigger_set_union() inverse_trigger_set = trigger_set.get_complement(Setup.buffer_codec.source_set) if not inverse_trigger_set.is_empty(): result_state.add_transition(inverse_trigger_set, accept_all_state_index) # Every acceptance state becomes a non-acceptance state. # Every non-acceptance state becomes an acceptance state. for state_index, state in SM.states.iteritems(): if state.is_acceptance(): result.states[state_index].set_acceptance(False) elif state_index != SM.init_state_index: result.states[state_index].set_acceptance(True) result.clean_up() return result.clone()
def add_transition(self, StartStateIdx, TriggerSet, TargetStateIdx=None, AcceptanceF=False): """Adds a transition from Start to Target based on a given Trigger. TriggerSet can be of different types: ... see add_transition() (see comment on 'State::add_transition) RETURNS: The target state index. """ # NOTE: The Transition Constructor is very tolerant, so no tests on TriggerSet() # assert TriggerSet.__class__.__name__ == "NumberSet" assert type(TargetStateIdx) == long or TargetStateIdx is None # If target state is undefined (None) then a new one has to be created if TargetStateIdx is None: TargetStateIdx = state_machine_index.get() if self.states.has_key(StartStateIdx) == False: self.states[StartStateIdx] = State() if self.states.has_key(TargetStateIdx) == False: self.states[TargetStateIdx] = State() if AcceptanceF: self.states[TargetStateIdx].set_acceptance(True) self.states[StartStateIdx].add_transition(TriggerSet, TargetStateIdx) return TargetStateIdx
def get_TemplateState(State, DropOutCatcher, Name=None): if not isinstance(State, TemplateState): State = PseudoTemplateState(State, DropOutCatcher) if Name is not None: print "State %s:" % Name, State.state_index_sequence() print_tm(State.transition_map, State.state_index_sequence()) return State
def get_merged_state(AcceptanceStateIndexList, EpsilonClosure): """Create the new target state in the state machine Accept only if all accept. """ acceptance_f = has_one_from_each(AcceptanceStateIndexList, EpsilonClosure) return State(AcceptanceF=acceptance_f)
def plug_state_sequence_for_trigger_set_sequence(sm, StartStateIdx, EndStateIdx, XList, L, DIdx): """Create a state machine sequence for trigger set list of the same length. L Length of the trigger set list. DIdx Index of first byte that differs, i.e. byte[i] == byte[k] for i, k < DIdx. XList The trigger set list. . . . [A, B, C, 80-BF ] [Start]--(A)-->[1]--(B)-->[2]--(C)-->[3]--(80-BF)-->[End] """ global FullRange assert L <= 6 s_idx = StartStateIdx # For the common bytes it is not essential what list is considered, take list no. 0. for trigger_set in XList[0][:DIdx]: s_idx = sm.add_transition(s_idx, trigger_set) # Store the last state where all bytes are the same sDIdx = s_idx # Indeces of the states that run on 'full range' (frs=full range state) def get_sm_index(frs_db, Key): result = frs_db.get(Key) if result is None: result = state_machine.index.get() frs_db[Key] = result return result frs_db = {} for trigger_set_seq in XList: # How many bytes at the end trigger on 'Min->Max' sbw_idx = EndStateIdx last_idx = EndStateIdx i = L - 1 while i > DIdx and i != 0: if not trigger_set_seq[i].is_equal(FullRange): break last_idx = get_sm_index(frs_db, i - 1) if not sm.states.has_key(last_idx): sm.states[last_idx] = State() sm.add_transition(last_idx, trigger_set_seq[i], sbw_idx) sbw_idx = last_idx i -= 1 sbw_idx = last_idx while i > DIdx: # Maybe, it has already a transition on trigger_set .. (TO DO) last_idx = state_machine.index.get() sm.add_transition(last_idx, trigger_set_seq[i], sbw_idx) sbw_idx = last_idx i -= 1 sm.add_transition(sDIdx, trigger_set_seq[i], last_idx) return
def mount_cloned(self, OtherSM, OperationIndex, OtherStartIndex, OtherEndIndex): """Clone all states in 'OtherSM' which lie on the path from 'OtherStartIndex' to 'OtherEndIndex'. If 'OtherEndIndex' is None, then it ends when there's no further path to go. State indices of the cloned states are generated by pairs of (other_i, OperationIndex). This makes it possible to refer to those states, even before they are generated. """ assert OtherStartIndex is not None work_set = set([OtherStartIndex]) if OtherEndIndex is None: done_set = set() else: done_set = set([OtherEndIndex]) while len(work_set) != 0: other_i = work_set.pop() other_state = OtherSM.states[other_i] state_i = state_machine_index.map_state_combination_to_index( (other_i, OperationIndex)) done_set.add(state_i) state = self.states.get(state_i) if state is None: state = State(AcceptanceF=other_state.is_acceptance()) self.states[state_i] = state for other_ti, other_trigger_set in other_state.target_map.get_map( ).iteritems(): target_i = state_machine_index.map_state_combination_to_index( (other_ti, OperationIndex)) # The state 'target_i' either: # -- exists, because it is in the done_set, or # -- will be created because its correspondance 'other_i' is # added to the work set. state.add_transition(other_trigger_set, target_i) if target_i not in done_set: assert other_i in OtherSM.states work_set.add(other_ti) return
def get_all(): """RETURNS: A state machine that 'eats' absolutely everything, i.e. .--- \Any ---. | | (0)--- \Any --->(( 0 ))<--------' """ result = StateMachine() i = index.get() state = State(AcceptanceF=True) state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i) result.states[i] = state result.get_init_state().add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i) return result
def get_all(): """RETURNS: A state machine that 'eats' absolutely everything, i.e. .--- \Any ---. | | (0)--- \Any --->(( 0 ))<--------' """ result = StateMachine() i = index.get() state = State(AcceptanceF=True) state.add_transition(NumberSet_All(), i) result.states[i] = state result.get_init_state().add_transition(NumberSet_All(), i) return result
def mount_cloned(self, OtherSM, OperationIndex, OtherStartIndex, OtherEndIndex): """Clone all states in 'OtherSM' which lie on the path from 'OtherStartIndex' to 'OtherEndIndex'. If 'OtherEndIndex' is None, then it ends when there's no further path to go. State indices of the cloned states are generated by pairs of (other_i, OperationIndex). This makes it possible to refer to those states, even before they are generated. """ assert OtherStartIndex is not None work_set = set([OtherStartIndex]) if OtherEndIndex is None: done_set = set() else: done_set = set([OtherEndIndex]) while len(work_set) != 0: other_i = work_set.pop() other_state = OtherSM.states[other_i] state_i = state_machine_index.map_state_combination_to_index((other_i, OperationIndex)) done_set.add(state_i) state = self.states.get(state_i) if state is None: state = State(AcceptanceF=other_state.is_acceptance()) self.states[state_i] = state for other_ti, other_trigger_set in other_state.target_map.get_map().iteritems(): target_i = state_machine_index.map_state_combination_to_index((other_ti, OperationIndex)) # The state 'target_i' either: # -- exists, because it is in the done_set, or # -- will be created because its correspondance 'other_i' is # added to the work set. state.add_transition(other_trigger_set, target_i) if target_i not in done_set: assert other_i in OtherSM.states work_set.add(other_ti) return
def add_epsilon_transition(self, StartStateIdx, TargetStateIdx=None, RaiseAcceptanceF=False): assert TargetStateIdx is None or type(TargetStateIdx) == long # create new state if index does not exist if not self.states.has_key(StartStateIdx): self.states[StartStateIdx] = State() if TargetStateIdx is None: TargetStateIdx = self.create_new_state( AcceptanceF=RaiseAcceptanceF) elif not self.states.has_key(TargetStateIdx): self.states[TargetStateIdx] = State() # add the epsilon target state self.states[StartStateIdx].target_map.add_epsilon_target_state( TargetStateIdx) # optionally raise the state of the target to 'acceptance' if RaiseAcceptanceF: self.states[TargetStateIdx].set_acceptance(True) return TargetStateIdx
def test_plug_sequence(ByteSequenceDB): L = len(ByteSequenceDB[0]) for seq in ByteSequenceDB: assert len(seq) == L for x in seq: assert isinstance(x, Interval) first_different_byte_index = -1 for i in range(L): x0 = ByteSequenceDB[0][i] for seq in ByteSequenceDB[1:]: if not seq[i].is_equal(x0): first_different_byte_index = i break if first_different_byte_index != -1: break if first_different_byte_index == -1: first_different_byte_index = 0 print "# Best To be Displayed by:" print "#" print "# > " + sys.argv[0] + " " + sys.argv[1] + " | dot -Tsvg -o tmp.svg" print "#" print "# -------------------------" print "# Byte Sequences: " i = -1 for seq in ByteSequenceDB: i += 1 print "# (%i) " % i, for x in seq: print " " + x.get_string(Option="hex"), print print "# L = %i" % L print "# DIdx = %i" % first_different_byte_index sm = StateMachine() end_index = state_machine.index.get() sm.states[end_index] = State() trafo = EncodingTrafoUTF8() Setup.buffer_codec_set(trafo, 1) trafo._plug_interval_sequences(sm, sm.init_state_index, end_index, ByteSequenceDB, beautifier) if len(sm.get_orphaned_state_index_list()) != 0: print "Error: Orphaned States Detected!" show_graphviz(sm)
def __init__(self, InitStateIndex=None, AcceptanceF=False, InitState=None, DoNothingF=False): # Get a unique state machine id self.set_id(state_machine_index.get_state_machine_id()) if DoNothingF: return if InitStateIndex is None: InitStateIndex = state_machine_index.get() self.init_state_index = InitStateIndex # State Index => State (information about what triggers transition to what target state). if InitState is None: InitState = State(AcceptanceF=AcceptanceF) self.states = {self.init_state_index: InitState}
StateMachine(InitStateIndex=5L, AcceptanceF=True) StateMachine(InitStateIndex=6L, AcceptanceF=True) StateMachine(InitStateIndex=100L, AcceptanceF=False) StateMachine(InitStateIndex=101L, AcceptanceF=False) StateMachine(InitStateIndex=102L, AcceptanceF=False) StateMachine(InitStateIndex=103L, AcceptanceF=False) StateMachine(InitStateIndex=104L, AcceptanceF=False) StateMachine(InitStateIndex=105L, AcceptanceF=False) StateMachine(InitStateIndex=106L, AcceptanceF=False) # (*) add priviledges # add_priority(4L, 0L) # add_priority(6L, 3L) # (1) only acceptance and non-acceptance states si = State() add_origin(1, True) add_origin(4, True) add_origin(0, True) add_origin(5, True) add_origin(6, True) add_origin(3, True) add_origin(2, True) add_origin(7, False) add_origin(8, False) add_origin(9, False) add_origin(10, False) add_origin(11, False) add_origin(12, False) add_origin(13, False)
def setup_state_operation(sm, CmdList, StateIndex): state = State() for cmd in CmdList: state.single_entry.add(cmd) sm.states[StateIndex] = state
def do(StateMachineList, CommonTerminalStateF=True, CloneF=True): """Connect state machines paralell. CommonTerminalStateF tells whether the state machines shall trigger to a common terminal. This may help nfa-to-dfa or hopcroft minimization for ISOLATED patterns. A state machine that consists of the COMBINATION of patterns MUST set this flag to 'False'. CloneF Controls if state machine list is cloned or not. If the single state machines are no longer required after construction, the CloneF can be set to False. If Cloning is disabled the state machines themselves will be altered--which brings some advantage in speed. """ assert len(StateMachineList) != 0 # filter out empty state machines from the consideration state_machine_list = [ sm for sm in StateMachineList if not (sm.is_empty() or special.is_none(sm)) ] empty_state_machine_list = [ sm for sm in StateMachineList if (sm.is_empty() or special.is_none(sm)) ] if len(state_machine_list) < 2: if len(state_machine_list) < 1: result = StateMachine() elif CloneF: result = state_machine_list[0].clone() else: result = state_machine_list[0] return __consider_empty_state_machines(result, empty_state_machine_list) # (*) need to clone the state machines, i.e. provide their internal # states with new ids, but the 'behavior' remains. This allows # state machines to appear twice, or being used in 'larger' # conglomerates. if CloneF: clone_list = map(lambda sm: sm.clone(), state_machine_list) else: clone_list = state_machine_list # (*) collect all transitions from both state machines into a single one # (clone to ensure unique identifiers of states) new_init_state = State() result = StateMachine(InitState=new_init_state) for clone in clone_list: result.states.update(clone.states) # (*) add additional **init** and **end** state # NOTE: when the result state machine was created, it already contains a # new initial state index. thus at this point only the new terminal # state has to be created. # NOTE: it is essential that the acceptance flag stays False, at this # point in time, so that the mounting operations only happen on # the old acceptance states. Later the acceptance state is raised # to 'accepted' (see below) new_terminal_state_index = -1L if CommonTerminalStateF: new_terminal_state_index = index.get() result.states[new_terminal_state_index] = State() # (*) Connect from the new initial state to the initial states of the # clones via epsilon transition. # Connect from each success state of the clones to the new terminal # state via epsilon transition. for clone in clone_list: result.mount_to_initial_state(clone.init_state_index) if CommonTerminalStateF: result.mount_to_acceptance_states(new_terminal_state_index, CancelStartAcceptanceStateF=False) return __consider_empty_state_machines(result, empty_state_machine_list)
def do(SM): """RETURNS: A state machines that matches anything which is not matched by SM. Idea: The paths along SM do not guide to acceptance states, but to normal states. Any drop-out is translated into a transition into the 'accept all state'. NOTE: This function produces a finite state automaton which is not applicable by itself. It would eat ANYTHING from a certain state on. """ result = deepcopy(SM) # Not clone accept_all_state_index = index.get() state = State(AcceptanceF=True) state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), accept_all_state_index) result.states[accept_all_state_index] = state def is_accept_all_state(sm, StateIndex): state = sm.states[StateIndex] if not state.is_acceptance(): return False tm = state.target_map.get_map() if len(tm) != 1: return False elif tm.iterkeys().next() != StateIndex: return False elif not tm.itervalues().next().is_all(): return False # Target is an 'Accept-All' state. Delete the transition. return True for state_index, state in SM.states.iteritems(): # deepcopy --> use same state indices in SM and result result_state = result.states[state_index] assert state.target_map.is_DFA_compliant(), \ "State machine must be transformed to DFA first: nfa_to_dfa.do()" # -- Every transition to 'Accept-All' state becomes a drop-out. for target_index in (i for i in state.target_map.get_target_state_index_list() if is_accept_all_state(SM, i)): result_state.target_map.delete_transitions_to_target(target_index) # -- Every drop-out becomes a transition to 'Accept-All' state. trigger_set = state.target_map.get_trigger_set_union() inverse_trigger_set = trigger_set.get_complement(Setup.buffer_codec.source_set) if not inverse_trigger_set.is_empty(): result_state.add_transition(inverse_trigger_set, accept_all_state_index) # Every acceptance state becomes a non-acceptance state. # Every non-acceptance state becomes an acceptance state. for state_index, state in SM.states.iteritems(): if state.is_acceptance(): result.states[state_index].set_acceptance(False) elif state_index != SM.init_state_index: result.states[state_index].set_acceptance(True) result.clean_up() return result.clone()
def get_state_core(self, AStateIndex): acceptance_f = self.original.states[AStateIndex].is_acceptance() return State(AcceptanceF=acceptance_f)
def create_new_state(self, AcceptanceF=False, StateIdx=None): if StateIdx is None: new_state_index = state_machine_index.get() else: new_state_index = StateIdx self.states[new_state_index] = State(AcceptanceF) return new_state_index