def do(SM):
"""RETURNS: A state machines that matches anything which is
not matched by SM.
Idea: The paths along SM do not guide to acceptance states,
but to normal states.
Any drop-out is translated into a transition into
the 'accept all state'.
NOTE: This function produces a finite state automaton which
is not applicable by itself. It would eat ANYTHING
from a certain state on.
"""
result = deepcopy(SM) # Not clone
accept_all_state_index = index.get()
state = State(AcceptanceF=True)
state.add_transition(NumberSet_All(), accept_all_state_index)
result.states[accept_all_state_index] = state
def is_accept_all_state(sm, StateIndex):
state = sm.states[StateIndex]
if not state.is_acceptance(): return False
tm = state.target_map.get_map()
if len(tm) != 1: return False
elif tm.iterkeys().next() != StateIndex: return False
elif not tm.itervalues().next().is_all(): return False
# Target is an 'Accept-All' state. Delete the transition.
return True
for state_index, state in SM.states.iteritems():
# deepcopy --> use same state indices in SM and result
result_state = result.states[state_index]
assert state.target_map.is_DFA_compliant(), \
"State machine must be transformed to DFA first: nfa_to_dfa.do()"
# -- Every transition to 'Accept-All' state becomes a drop-out.
for target_index in (i for i in state.target_map.get_target_state_index_list()
if is_accept_all_state(SM, i)):
result_state.target_map.delete_transitions_to_target(target_index)
# -- Every drop-out becomes a transition to 'Accept-All' state.
trigger_set = state.target_map.get_trigger_set_union()
inverse_trigger_set = trigger_set.get_complement(Setup.buffer_codec.source_set)
if not inverse_trigger_set.is_empty():
result_state.add_transition(inverse_trigger_set, accept_all_state_index)
# Every acceptance state becomes a non-acceptance state.
# Every non-acceptance state becomes an acceptance state.
for state_index, state in SM.states.iteritems():
if state.is_acceptance():
result.states[state_index].set_acceptance(False)
elif state_index != SM.init_state_index:
result.states[state_index].set_acceptance(True)
result.clean_up()
return result.clone()
def add_transition(self,
StartStateIdx,
TriggerSet,
TargetStateIdx=None,
AcceptanceF=False):
"""Adds a transition from Start to Target based on a given Trigger.
TriggerSet can be of different types: ... see add_transition()
(see comment on 'State::add_transition)
RETURNS: The target state index.
"""
# NOTE: The Transition Constructor is very tolerant, so no tests on TriggerSet()
# assert TriggerSet.__class__.__name__ == "NumberSet"
assert type(TargetStateIdx) == long or TargetStateIdx is None
# If target state is undefined (None) then a new one has to be created
if TargetStateIdx is None: TargetStateIdx = state_machine_index.get()
if self.states.has_key(StartStateIdx) == False:
self.states[StartStateIdx] = State()
if self.states.has_key(TargetStateIdx) == False:
self.states[TargetStateIdx] = State()
if AcceptanceF: self.states[TargetStateIdx].set_acceptance(True)
self.states[StartStateIdx].add_transition(TriggerSet, TargetStateIdx)
return TargetStateIdx
def get_TemplateState(State, DropOutCatcher, Name=None):
if not isinstance(State, TemplateState):
State = PseudoTemplateState(State, DropOutCatcher)
if Name is not None:
print "State %s:" % Name, State.state_index_sequence()
print_tm(State.transition_map, State.state_index_sequence())
return State
def get_TemplateState(State, DropOutCatcher, Name=None):
if not isinstance(State, TemplateState):
State = PseudoTemplateState(State, DropOutCatcher)
if Name is not None:
print "State %s:" % Name, State.state_index_sequence()
print_tm(State.transition_map, State.state_index_sequence())
return State
def get_merged_state(AcceptanceStateIndexList, EpsilonClosure):
"""Create the new target state in the state machine
Accept only if all accept.
"""
acceptance_f = has_one_from_each(AcceptanceStateIndexList,
EpsilonClosure)
return State(AcceptanceF=acceptance_f)
def plug_state_sequence_for_trigger_set_sequence(sm, StartStateIdx,
EndStateIdx, XList, L, DIdx):
"""Create a state machine sequence for trigger set list of the same length.
L Length of the trigger set list.
DIdx Index of first byte that differs, i.e. byte[i] == byte[k] for i, k < DIdx.
XList The trigger set list.
. . .
[A, B, C, 80-BF ]
[Start]--(A)-->[1]--(B)-->[2]--(C)-->[3]--(80-BF)-->[End]
"""
global FullRange
assert L <= 6
s_idx = StartStateIdx
# For the common bytes it is not essential what list is considered, take list no. 0.
for trigger_set in XList[0][:DIdx]:
s_idx = sm.add_transition(s_idx, trigger_set)
# Store the last state where all bytes are the same
sDIdx = s_idx
# Indeces of the states that run on 'full range' (frs=full range state)
def get_sm_index(frs_db, Key):
result = frs_db.get(Key)
if result is None:
result = state_machine.index.get()
frs_db[Key] = result
return result
frs_db = {}
for trigger_set_seq in XList:
# How many bytes at the end trigger on 'Min->Max'
sbw_idx = EndStateIdx
last_idx = EndStateIdx
i = L - 1
while i > DIdx and i != 0:
if not trigger_set_seq[i].is_equal(FullRange): break
last_idx = get_sm_index(frs_db, i - 1)
if not sm.states.has_key(last_idx): sm.states[last_idx] = State()
sm.add_transition(last_idx, trigger_set_seq[i], sbw_idx)
sbw_idx = last_idx
i -= 1
sbw_idx = last_idx
while i > DIdx:
# Maybe, it has already a transition on trigger_set .. (TO DO)
last_idx = state_machine.index.get()
sm.add_transition(last_idx, trigger_set_seq[i], sbw_idx)
sbw_idx = last_idx
i -= 1
sm.add_transition(sDIdx, trigger_set_seq[i], last_idx)
return
def mount_cloned(self, OtherSM, OperationIndex, OtherStartIndex,
OtherEndIndex):
"""Clone all states in 'OtherSM' which lie on the path from 'OtherStartIndex'
to 'OtherEndIndex'. If 'OtherEndIndex' is None, then it ends when there's no further
path to go.
State indices of the cloned states are generated by pairs of (other_i, OperationIndex).
This makes it possible to refer to those states, even before they are generated.
"""
assert OtherStartIndex is not None
work_set = set([OtherStartIndex])
if OtherEndIndex is None: done_set = set()
else: done_set = set([OtherEndIndex])
while len(work_set) != 0:
other_i = work_set.pop()
other_state = OtherSM.states[other_i]
state_i = state_machine_index.map_state_combination_to_index(
(other_i, OperationIndex))
done_set.add(state_i)
state = self.states.get(state_i)
if state is None:
state = State(AcceptanceF=other_state.is_acceptance())
self.states[state_i] = state
for other_ti, other_trigger_set in other_state.target_map.get_map(
).iteritems():
target_i = state_machine_index.map_state_combination_to_index(
(other_ti, OperationIndex))
# The state 'target_i' either:
# -- exists, because it is in the done_set, or
# -- will be created because its correspondance 'other_i' is
# added to the work set.
state.add_transition(other_trigger_set, target_i)
if target_i not in done_set:
assert other_i in OtherSM.states
work_set.add(other_ti)
return
def get_all():
"""RETURNS:
A state machine that 'eats' absolutely everything, i.e.
.--- \Any ---.
| |
(0)--- \Any --->(( 0 ))<--------'
"""
result = StateMachine()
i = index.get()
state = State(AcceptanceF=True)
state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i)
result.states[i] = state
result.get_init_state().add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)), i)
return result
def get_all():
"""RETURNS:
A state machine that 'eats' absolutely everything, i.e.
.--- \Any ---.
| |
(0)--- \Any --->(( 0 ))<--------'
"""
result = StateMachine()
i = index.get()
state = State(AcceptanceF=True)
state.add_transition(NumberSet_All(), i)
result.states[i] = state
result.get_init_state().add_transition(NumberSet_All(), i)
return result
def mount_cloned(self, OtherSM, OperationIndex, OtherStartIndex, OtherEndIndex):
"""Clone all states in 'OtherSM' which lie on the path from 'OtherStartIndex'
to 'OtherEndIndex'. If 'OtherEndIndex' is None, then it ends when there's no further
path to go.
State indices of the cloned states are generated by pairs of (other_i, OperationIndex).
This makes it possible to refer to those states, even before they are generated.
"""
assert OtherStartIndex is not None
work_set = set([OtherStartIndex])
if OtherEndIndex is None: done_set = set()
else: done_set = set([OtherEndIndex])
while len(work_set) != 0:
other_i = work_set.pop()
other_state = OtherSM.states[other_i]
state_i = state_machine_index.map_state_combination_to_index((other_i, OperationIndex))
done_set.add(state_i)
state = self.states.get(state_i)
if state is None:
state = State(AcceptanceF=other_state.is_acceptance())
self.states[state_i] = state
for other_ti, other_trigger_set in other_state.target_map.get_map().iteritems():
target_i = state_machine_index.map_state_combination_to_index((other_ti, OperationIndex))
# The state 'target_i' either:
# -- exists, because it is in the done_set, or
# -- will be created because its correspondance 'other_i' is
# added to the work set.
state.add_transition(other_trigger_set, target_i)
if target_i not in done_set:
assert other_i in OtherSM.states
work_set.add(other_ti)
return
def add_epsilon_transition(self,
StartStateIdx,
TargetStateIdx=None,
RaiseAcceptanceF=False):
assert TargetStateIdx is None or type(TargetStateIdx) == long
# create new state if index does not exist
if not self.states.has_key(StartStateIdx):
self.states[StartStateIdx] = State()
if TargetStateIdx is None:
TargetStateIdx = self.create_new_state(
AcceptanceF=RaiseAcceptanceF)
elif not self.states.has_key(TargetStateIdx):
self.states[TargetStateIdx] = State()
# add the epsilon target state
self.states[StartStateIdx].target_map.add_epsilon_target_state(
TargetStateIdx)
# optionally raise the state of the target to 'acceptance'
if RaiseAcceptanceF: self.states[TargetStateIdx].set_acceptance(True)
return TargetStateIdx
def test_plug_sequence(ByteSequenceDB):
L = len(ByteSequenceDB[0])
for seq in ByteSequenceDB:
assert len(seq) == L
for x in seq:
assert isinstance(x, Interval)
first_different_byte_index = -1
for i in range(L):
x0 = ByteSequenceDB[0][i]
for seq in ByteSequenceDB[1:]:
if not seq[i].is_equal(x0):
first_different_byte_index = i
break
if first_different_byte_index != -1:
break
if first_different_byte_index == -1:
first_different_byte_index = 0
print "# Best To be Displayed by:"
print "#"
print "# > " + sys.argv[0] + " " + sys.argv[1] + " | dot -Tsvg -o tmp.svg"
print "#"
print "# -------------------------"
print "# Byte Sequences: "
i = -1
for seq in ByteSequenceDB:
i += 1
print "# (%i) " % i,
for x in seq:
print " " + x.get_string(Option="hex"),
print
print "# L = %i" % L
print "# DIdx = %i" % first_different_byte_index
sm = StateMachine()
end_index = state_machine.index.get()
sm.states[end_index] = State()
trafo = EncodingTrafoUTF8()
Setup.buffer_codec_set(trafo, 1)
trafo._plug_interval_sequences(sm, sm.init_state_index, end_index,
ByteSequenceDB, beautifier)
if len(sm.get_orphaned_state_index_list()) != 0:
print "Error: Orphaned States Detected!"
show_graphviz(sm)
def __init__(self,
InitStateIndex=None,
AcceptanceF=False,
InitState=None,
DoNothingF=False):
# Get a unique state machine id
self.set_id(state_machine_index.get_state_machine_id())
if DoNothingF: return
if InitStateIndex is None: InitStateIndex = state_machine_index.get()
self.init_state_index = InitStateIndex
# State Index => State (information about what triggers transition to what target state).
if InitState is None: InitState = State(AcceptanceF=AcceptanceF)
self.states = {self.init_state_index: InitState}
StateMachine(InitStateIndex=5L, AcceptanceF=True) StateMachine(InitStateIndex=6L, AcceptanceF=True) StateMachine(InitStateIndex=100L, AcceptanceF=False) StateMachine(InitStateIndex=101L, AcceptanceF=False) StateMachine(InitStateIndex=102L, AcceptanceF=False) StateMachine(InitStateIndex=103L, AcceptanceF=False) StateMachine(InitStateIndex=104L, AcceptanceF=False) StateMachine(InitStateIndex=105L, AcceptanceF=False) StateMachine(InitStateIndex=106L, AcceptanceF=False) # (*) add priviledges # add_priority(4L, 0L) # add_priority(6L, 3L) # (1) only acceptance and non-acceptance states si = State() add_origin(1, True) add_origin(4, True) add_origin(0, True) add_origin(5, True) add_origin(6, True) add_origin(3, True) add_origin(2, True) add_origin(7, False) add_origin(8, False) add_origin(9, False) add_origin(10, False) add_origin(11, False) add_origin(12, False) add_origin(13, False)
def setup_state_operation(sm, CmdList, StateIndex):
state = State()
for cmd in CmdList:
state.single_entry.add(cmd)
sm.states[StateIndex] = state
def do(StateMachineList, CommonTerminalStateF=True, CloneF=True):
"""Connect state machines paralell.
CommonTerminalStateF tells whether the state machines shall trigger
to a common terminal. This may help nfa-to-dfa
or hopcroft minimization for ISOLATED patterns.
A state machine that consists of the COMBINATION
of patterns MUST set this flag to 'False'.
CloneF Controls if state machine list is cloned or not.
If the single state machines are no longer required after
construction, the CloneF can be set to False.
If Cloning is disabled the state machines themselves
will be altered--which brings some advantage in speed.
"""
assert len(StateMachineList) != 0
# filter out empty state machines from the consideration
state_machine_list = [
sm for sm in StateMachineList
if not (sm.is_empty() or special.is_none(sm))
]
empty_state_machine_list = [
sm for sm in StateMachineList if (sm.is_empty() or special.is_none(sm))
]
if len(state_machine_list) < 2:
if len(state_machine_list) < 1: result = StateMachine()
elif CloneF: result = state_machine_list[0].clone()
else: result = state_machine_list[0]
return __consider_empty_state_machines(result,
empty_state_machine_list)
# (*) need to clone the state machines, i.e. provide their internal
# states with new ids, but the 'behavior' remains. This allows
# state machines to appear twice, or being used in 'larger'
# conglomerates.
if CloneF: clone_list = map(lambda sm: sm.clone(), state_machine_list)
else: clone_list = state_machine_list
# (*) collect all transitions from both state machines into a single one
# (clone to ensure unique identifiers of states)
new_init_state = State()
result = StateMachine(InitState=new_init_state)
for clone in clone_list:
result.states.update(clone.states)
# (*) add additional **init** and **end** state
# NOTE: when the result state machine was created, it already contains a
# new initial state index. thus at this point only the new terminal
# state has to be created.
# NOTE: it is essential that the acceptance flag stays False, at this
# point in time, so that the mounting operations only happen on
# the old acceptance states. Later the acceptance state is raised
# to 'accepted' (see below)
new_terminal_state_index = -1L
if CommonTerminalStateF:
new_terminal_state_index = index.get()
result.states[new_terminal_state_index] = State()
# (*) Connect from the new initial state to the initial states of the
# clones via epsilon transition.
# Connect from each success state of the clones to the new terminal
# state via epsilon transition.
for clone in clone_list:
result.mount_to_initial_state(clone.init_state_index)
if CommonTerminalStateF:
result.mount_to_acceptance_states(new_terminal_state_index,
CancelStartAcceptanceStateF=False)
return __consider_empty_state_machines(result, empty_state_machine_list)
def do(SM):
"""RETURNS: A state machines that matches anything which is
not matched by SM.
Idea: The paths along SM do not guide to acceptance states,
but to normal states.
Any drop-out is translated into a transition into
the 'accept all state'.
NOTE: This function produces a finite state automaton which
is not applicable by itself. It would eat ANYTHING
from a certain state on.
"""
result = deepcopy(SM) # Not clone
accept_all_state_index = index.get()
state = State(AcceptanceF=True)
state.add_transition(NumberSet(Interval(-sys.maxint, sys.maxint)),
accept_all_state_index)
result.states[accept_all_state_index] = state
def is_accept_all_state(sm, StateIndex):
state = sm.states[StateIndex]
if not state.is_acceptance(): return False
tm = state.target_map.get_map()
if len(tm) != 1: return False
elif tm.iterkeys().next() != StateIndex: return False
elif not tm.itervalues().next().is_all(): return False
# Target is an 'Accept-All' state. Delete the transition.
return True
for state_index, state in SM.states.iteritems():
# deepcopy --> use same state indices in SM and result
result_state = result.states[state_index]
assert state.target_map.is_DFA_compliant(), \
"State machine must be transformed to DFA first: nfa_to_dfa.do()"
# -- Every transition to 'Accept-All' state becomes a drop-out.
for target_index in (i for i in state.target_map.get_target_state_index_list()
if is_accept_all_state(SM, i)):
result_state.target_map.delete_transitions_to_target(target_index)
# -- Every drop-out becomes a transition to 'Accept-All' state.
trigger_set = state.target_map.get_trigger_set_union()
inverse_trigger_set = trigger_set.get_complement(Setup.buffer_codec.source_set)
if not inverse_trigger_set.is_empty():
result_state.add_transition(inverse_trigger_set, accept_all_state_index)
# Every acceptance state becomes a non-acceptance state.
# Every non-acceptance state becomes an acceptance state.
for state_index, state in SM.states.iteritems():
if state.is_acceptance():
result.states[state_index].set_acceptance(False)
elif state_index != SM.init_state_index:
result.states[state_index].set_acceptance(True)
result.clean_up()
return result.clone()
def get_state_core(self, AStateIndex):
acceptance_f = self.original.states[AStateIndex].is_acceptance()
return State(AcceptanceF=acceptance_f)
def create_new_state(self, AcceptanceF=False, StateIdx=None):
if StateIdx is None: new_state_index = state_machine_index.get()
else: new_state_index = StateIdx
self.states[new_state_index] = State(AcceptanceF)
return new_state_index