def snap_expression(stream, PatternDict):
"""expression: term
term | expression
"""
__debug_entry("expression", stream)
# -- term
result = snap_term(stream, PatternDict)
if result is None:
return __debug_exit(None, stream)
# -- optional '|'
if not check(stream, '|'):
return __debug_exit(result, stream)
position_1 = stream.tell()
__debug_print("'|' (in expression)")
# -- expression
result_2 = snap_expression(stream, PatternDict)
__debug_print("expression(in expression):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
result = parallelize.do([result, result_2], CloneF=True) # CloneF = False (shold be!)
return __debug_exit(beautifier.do(result), stream)
def snap_expression(stream, PatternDict):
"""expression: term
term | expression
"""
__debug_entry("expression", stream)
# -- term
result = snap_term(stream, PatternDict)
if result is None:
return __debug_exit(None, stream)
# -- optional '|'
if not check(stream, '|'):
return __debug_exit(result, stream)
position_1 = stream.tell()
__debug_print("'|' (in expression)")
# -- expression
result_2 = snap_expression(stream, PatternDict)
__debug_print("expression(in expression):", result_2)
if result_2 is None:
stream.seek(position_1)
return __debug_exit(result, stream)
result = parallelize.do([result, result_2])
return __debug_exit(nfa_to_dfa.do(result), stream)
def do(SM_List):
"""The 'parallelize' module does a union of multiple state machines,
even if they have different origins and need to be combined carefully.
There is no reason, why another 'union' operation should be implemented
in this case.
"""
result = parallelize.do(SM_List)
return beautifier.do(result)
def do(SM_List):
"""The 'parallelize' module does a union of multiple state machines,
even if they have different origins and need to be combined carefully.
There is no reason, why another 'union' operation should be implemented
in this case.
"""
result = parallelize.do(SM_List)
return hopcroft_minimization.do(result, CreateNewStateMachineF=False)
def do(SM_List):
"""The 'parallelize' module does a union of multiple state machines,
even if they have different origins and need to be combined carefully.
There is no reason, why another 'union' operation should be implemented
in this case.
"""
result = parallelize.do(SM_List)
return beautifier.do(result)
def test(SmList):
print "-------------------------------------------------------------------------------"
for tsm in SmList:
print "##sm:", tsm
sm = parallelize.do(SmList)
print "RESULT:", sm
complement_sm = complement.do(sm)
assert all(superset.do(sm, tsm) == True for tsm in SmList)
assert all(
DFA.is_Empty(intersection.do([complement_sm, tsm])) for tsm in SmList)
def get_combined_state_machine(StateMachine_List,
FilterDominatedOriginsF=True,
MarkNotSet=set(),
AlllowInitStateAcceptF=False):
"""Creates a DFA state machine that incorporates the paralell
process of all pattern passed as state machines in
the StateMachine_List. Each origins of each state machine
are kept in the final state, if it is not dominated.
Performs: -- parallelization
-- translation from NFA to DFA
-- Frank Schaefers Adapted Hopcroft optimization.
Again: The state machine ids of the original state machines
are traced through the whole process.
FilterDominatedOriginsF, if set to False, can disable the filtering
of dominated origins. This is important for pre-contexts, because,
all successful patterns need to be reported!
"""
if len(StateMachine_List) == 0:
return None
def __check(Place, sm, AlllowInitStateAcceptF):
__check_on_orphan_states(Place, sm)
if not AlllowInitStateAcceptF:
__check_on_init_state_not_acceptance(Place, sm)
def __check_on_orphan_states(Place, sm):
orphan_state_list = sm.get_orphaned_state_index_list()
if len(orphan_state_list) == 0: return
error.log("After '%s'" % Place + "\n" + \
"Orphaned state(s) detected in regular expression (optimization lack).\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n" + \
"Orphan state(s) = " + repr(orphan_state_list))
def __check_on_init_state_not_acceptance(Place, sm):
if sm.get_init_state().is_acceptance():
error.log("After '%s'" % Place + "\n" + \
"Initial state 'accepts'. This should never happen.\n" + \
"Please, log a defect at the projects web site quex.sourceforge.net.\n")
# (1) mark at each state machine the machine and states as 'original'.
#
# This is necessary to trace in the combined state machine the pattern that
# actually matched. Note, that a state machine in the StateMachine_List
# represents one possible pattern that can match the current input.
#
for sm in StateMachine_List:
if sm.get_id() in MarkNotSet: continue
sm.mark_state_origins()
assert sm.is_DFA_compliant(), sm.get_string(Option="hex")
# (2) setup all patterns in paralell
sm = parallelize.do(StateMachine_List,
CommonTerminalStateF=False) #, CloneF=False)
__check("Parallelization", sm, AlllowInitStateAcceptF)
# (4) determine for each state in the DFA what is the dominating original
# state
if FilterDominatedOriginsF: sm.filter_dominated_origins()
__check("Filter Dominated Origins", sm, AlllowInitStateAcceptF)
# (3) convert the state machine to an DFA (paralellization created an NFA)
sm = beautifier.do(sm)
__check("NFA to DFA, Hopcroft Minimization", sm, AlllowInitStateAcceptF)
return sm
out_n = 0
def output(Sm):
global out_n
if "help" not in sys.argv:
print "sm%i: %s" % (out_n, Sm)
else:
open("tmp%i.dot" % out_n, "wb").write(Sm.get_graphviz_string(NormalizeF=True))
print "written 'tmp%i.dot'" % out_n
out_n += 1
sm0.mark_state_origins()
sm1.mark_state_origins()
sm2.mark_state_origins()
output(sm0)
output(sm1)
output(sm2)
sm_list = [sm0, sm1, sm2]
sm = parallelize.do(sm_list)
print "#-------------------------------------------------------------------------------"
output(sm)
complement_sm = complement.do(sm)
assert all(superset.do(sm, tsm) == True
for tsm in sm_list)
assert all(DFA.is_Empty(intersection.do([complement_sm, tsm]))
for tsm in sm_list)
print "<terminated>"
print "Ranking of state machines: Filter dominated origins"
sys.exit(0)
#----------------------------------------------------------------------------------------
# (*) setup the state machines
sm0 = repeat.do(sm0)
sm0.mark_state_origins()
sm1.mark_state_origins()
sm2.mark_state_origins()
sm3 = repeat.do(sm3, 1)
sm3.mark_state_origins()
# -- paralellize the patterns
sm = parallelize.do([sm0, sm1, sm2, sm3])
# -- create the optimized DFA for the patterns running in paralell
sm = beautifier.do(sm)
#----------------------------------------------------------------------------------------
# (*) create some priorities in between the patterns
def add_priority(A, B):
print "priority: %s > %s" % (repr(A.get_id()), repr(B.get_id()))
state_machine_ranking_db_register(A.get_id(), B.get_id())
add_priority(sm3, sm0)
add_priority(sm3, sm1)
add_priority(sm3, sm2)
add_priority(sm0, sm1)
print "Ranking of state machines: Filter dominated origins"
sys.exit(0)
#----------------------------------------------------------------------------------------
# (*) setup the state machines
sm0 = repeat.do(sm0)
sm0.mark_state_origins()
sm1.mark_state_origins()
sm2.mark_state_origins()
sm3 = repeat.do(sm3, 1)
sm3.mark_state_origins()
# -- paralellize the patterns
sm = parallelize.do([sm0, sm1, sm2, sm3])
# -- create the optimized DFA for the patterns running in paralell
sm = beautifier.do(sm)
#----------------------------------------------------------------------------------------
# (*) create some priorities in between the patterns
def add_priority(A, B):
print "priority: %s > %s" % (repr(A.get_id()), repr(B.get_id()))
state_machine_ranking_db_register(A.get_id(), B.get_id())
add_priority(sm3, sm0)
add_priority(sm3, sm1)
add_priority(sm3, sm2)
add_priority(sm0, sm1)
# register some more state relations for some of the states before
import quex.engine.state_machine.algorithm.beautifier as beautifier
import quex.engine.state_machine.algebra.reverse as reverse
if "--hwut-info" in sys.argv:
print "Tracing origin: Inverse"
sys.exit(0)
def invert_this(sm):
print "-------------------------------------------------------------------------------"
print "sm = ", sm
tmp = reverse.do(sm)
print "inverse = ", tmp
return tmp
sm0.mark_state_origins()
# sm1.mark_state_origins()
sm2.mark_state_origins()
sm0 = invert_this(sm0)
# invert_this(sm1)
sm2 = invert_this(sm2)
sm = parallelize.do([sm0, sm2])
print "-------------------------------------------------------------------------------"
print "## paralellized = ", sm
sm = beautifier.do(sm)
print "## result = ", sm
import quex.engine.state_machine.construction.parallelize as parallelize
from quex.engine.state_machine.TEST.test_state_machines import *
if "--hwut-info" in sys.argv:
print "StateMachine Operations: Paralell"
sys.exit(0)
print "-------------------------------------------------------------------------------"
tsm0 = trivial_sm('a')
tsm1 = trivial_sm('b')
tsm2 = trivial_sm('c')
print "##tsm0:", tsm0
print "##tsm1:", tsm1
print "##tsm2:", tsm2
sm = parallelize.do([tsm0, tsm1, tsm2])
print "EXAMPLE 0:", sm
print "-------------------------------------------------------------------------------"
print "##sm0", sm0
print "##sm1", sm1
print "##sm2", sm2
print "-------------------------------------------------------------------------------"
sm = parallelize.do([sm0, sm1, sm2])
print "EXAMPLE A:", sm
print "-------------------------------------------------------------------------------"
empty_state_machine = StateMachine(7777)
empty_state_machine.get_init_state().set_acceptance(True)
sm = parallelize.do([
import os
sys.path.insert(0, os.environ["QUEX_PATH"])
from quex.engine.state_machine.core import *
import quex.input.regular_expression.engine as regex
import quex.engine.state_machine.algorithm.nfa_to_dfa as nfa_to_dfa
import quex.engine.state_machine.construction.parallelize as parallelize
if "--hwut-info" in sys.argv:
print "Tracing origin: NFA to DFA (subset construction) II"
sys.exit(0)
sm1 = regex.do('[A-Z]+', {}).sm
sm2 = regex.do('"PR"', {}).sm
# (*) create the DFA from the specified NFA
sm1.mark_state_origins()
sm2.mark_state_origins()
print "## sm1 = ", sm1.get_string(OriginalStatesF=False, NormalizeF=True)
print "## sm2 = ", sm2.get_string(OriginalStatesF=False, NormalizeF=True)
sm = parallelize.do([sm1, sm2])
print "## parallelized:", sm.get_string(NormalizeF=True, OriginalStatesF=False)
# print "## parallelized:", sm.get_string(NormalizeF=False)
dfa = nfa_to_dfa.do(sm)
print dfa.get_string(OriginalStatesF=False, NormalizeF=True)
import quex.engine.state_machine.construction.parallelize as parallelize
from quex.engine.state_machine.TEST.test_state_machines import *
import quex.engine.state_machine.algorithm.beautifier as beautifier
import quex.engine.state_machine.algebra.reverse as reverse
if "--hwut-info" in sys.argv:
print "Tracing origin: Inverse"
sys.exit(0)
def invert_this(sm):
print "-------------------------------------------------------------------------------"
print "sm = ", sm
tmp = reverse.do(sm)
print "inverse = ", tmp
return tmp
sm0.mark_state_origins()
# sm1.mark_state_origins()
sm2.mark_state_origins()
sm0 = invert_this(sm0)
# invert_this(sm1)
sm2 = invert_this(sm2)
sm = parallelize.do([sm0, sm2])
print "-------------------------------------------------------------------------------"
print "## paralellized = ", sm
sm = beautifier.do(sm)
print "## result = ", sm
import sys
import os
sys.path.insert(0, os.environ["QUEX_PATH"])
from quex.engine.state_machine.core import *
import quex.input.regular_expression.engine as regex
import quex.engine.state_machine.algorithm.nfa_to_dfa as nfa_to_dfa
import quex.engine.state_machine.construction.parallelize as parallelize
if "--hwut-info" in sys.argv:
print "Tracing origin: NFA to DFA (subset construction) II"
sys.exit(0)
sm1 = regex.do('[A-Z]+', {}).sm
sm2 = regex.do('"PR"', {}).sm
# (*) create the DFA from the specified NFA
sm1.mark_state_origins()
sm2.mark_state_origins()
print "## sm1 = ", sm1.get_string(OriginalStatesF=False, NormalizeF=True)
print "## sm2 = ", sm2.get_string(OriginalStatesF=False, NormalizeF=True)
sm = parallelize.do([sm1, sm2])
print "## parallelized:", sm.get_string(NormalizeF=True, OriginalStatesF=False)
# print "## parallelized:", sm.get_string(NormalizeF=False)
dfa = nfa_to_dfa.do(sm)
print dfa.get_string(OriginalStatesF=False, NormalizeF=True)
def get_combined_state_machine(StateMachine_List, FilterDominatedOriginsF=True,
MarkNotSet=set()):
"""Creates a DFA state machine that incorporates the paralell
process of all pattern passed as state machines in
the StateMachine_List. Each origins of each state machine
are kept in the final state, if it is not dominated.
Performs: -- parallelization
-- translation from NFA to DFA
-- Frank Schaefers Adapted Hopcroft optimization.
Again: The state machine ids of the original state machines
are traced through the whole process.
FilterDominatedOriginsF, if set to False, can disable the filtering
of dominated origins. This is important for pre-contexts, because,
all successful patterns need to be reported!
"""
if len(StateMachine_List) == 0:
return None
def __check(Place, sm):
__check_on_orphan_states(Place, sm)
__check_on_init_state_not_acceptance(Place, sm)
def __check_on_orphan_states(Place, sm):
orphan_state_list = sm.get_orphaned_state_index_list()
if len(orphan_state_list) == 0: return
error.log("After '%s'" % Place + "\n" + \
"Orphaned state(s) detected in regular expression (optimization lack).\n" + \
"Please, log a defect at the projects website quex.sourceforge.net.\n" + \
"Orphan state(s) = " + repr(orphan_state_list))
def __check_on_init_state_not_acceptance(Place, sm):
if sm.get_init_state().is_acceptance():
error.log("After '%s'" % Place + "\n" + \
"Initial state 'accepts'. This should never happen.\n" + \
"Please, log a defect at the projects web site quex.sourceforge.net.\n")
# (1) mark at each state machine the machine and states as 'original'.
#
# This is necessary to trace in the combined state machine the
# pattern that actually matched. Note, that a state machine in
# the StateMachine_List represents one possible pattern that can
# match the current input.
#
for sm in StateMachine_List:
if sm.get_id() in MarkNotSet: continue
sm.mark_state_origins()
assert sm.is_DFA_compliant(), sm.get_string(Option="hex")
# (2) setup all patterns in paralell
sm = parallelize.do(StateMachine_List, CommonTerminalStateF=False) #, CloneF=False)
__check("Parallelization", sm)
# (4) determine for each state in the DFA what is the dominating original state
if FilterDominatedOriginsF: sm.filter_dominated_origins()
__check("Filter Dominated Origins", sm)
# (3) convert the state machine to an DFA (paralellization created an NFA)
sm = beautifier.do(sm)
__check("NFA to DFA, Hopcroft Minimization", sm)
return sm
from quex.engine.state_machine.TEST.test_state_machines import *
if "--hwut-info" in sys.argv:
print "StateMachine Operations: Paralell"
sys.exit(0)
print "-------------------------------------------------------------------------------"
tsm0 = trivial_sm('a')
tsm1 = trivial_sm('b')
tsm2 = trivial_sm('c')
print "##tsm0:", tsm0
print "##tsm1:", tsm1
print "##tsm2:", tsm2
sm = parallelize.do([tsm0, tsm1, tsm2])
print "EXAMPLE 0:", sm
print "-------------------------------------------------------------------------------"
print "##sm0", sm0
print "##sm1", sm1
print "##sm2", sm2
print "-------------------------------------------------------------------------------"
sm = parallelize.do([sm0, sm1, sm2])
print "EXAMPLE A:", sm
print "-------------------------------------------------------------------------------"
empty_state_machine = StateMachine(7777)
empty_state_machine.get_init_state().set_acceptance(True)
sm = parallelize.do([empty_state_machine, sm0,