def snap_expression(stream, PatternDict): """expression: term term | expression """ __debug_entry("expression", stream) # -- term result = snap_term(stream, PatternDict) if result is None: return __debug_exit(None, stream) # -- optional '|' if not check(stream, '|'): return __debug_exit(result, stream) position_1 = stream.tell() __debug_print("'|' (in expression)") # -- expression result_2 = snap_expression(stream, PatternDict) __debug_print("expression(in expression):", result_2) if result_2 is None: stream.seek(position_1) return __debug_exit(result, stream) result = parallelize.do([result, result_2], CloneF=True) # CloneF = False (shold be!) return __debug_exit(beautifier.do(result), stream)
def snap_expression(stream, PatternDict): """expression: term term | expression """ __debug_entry("expression", stream) # -- term result = snap_term(stream, PatternDict) if result is None: return __debug_exit(None, stream) # -- optional '|' if not check(stream, '|'): return __debug_exit(result, stream) position_1 = stream.tell() __debug_print("'|' (in expression)") # -- expression result_2 = snap_expression(stream, PatternDict) __debug_print("expression(in expression):", result_2) if result_2 is None: stream.seek(position_1) return __debug_exit(result, stream) result = parallelize.do([result, result_2]) return __debug_exit(nfa_to_dfa.do(result), stream)
def do(SM_List): """The 'parallelize' module does a union of multiple state machines, even if they have different origins and need to be combined carefully. There is no reason, why another 'union' operation should be implemented in this case. """ result = parallelize.do(SM_List) return beautifier.do(result)
def do(SM_List): """The 'parallelize' module does a union of multiple state machines, even if they have different origins and need to be combined carefully. There is no reason, why another 'union' operation should be implemented in this case. """ result = parallelize.do(SM_List) return hopcroft_minimization.do(result, CreateNewStateMachineF=False)
def do(SM_List): """The 'parallelize' module does a union of multiple state machines, even if they have different origins and need to be combined carefully. There is no reason, why another 'union' operation should be implemented in this case. """ result = parallelize.do(SM_List) return beautifier.do(result)
def test(SmList): print "-------------------------------------------------------------------------------" for tsm in SmList: print "##sm:", tsm sm = parallelize.do(SmList) print "RESULT:", sm complement_sm = complement.do(sm) assert all(superset.do(sm, tsm) == True for tsm in SmList) assert all( DFA.is_Empty(intersection.do([complement_sm, tsm])) for tsm in SmList)
def get_combined_state_machine(StateMachine_List, FilterDominatedOriginsF=True, MarkNotSet=set(), AlllowInitStateAcceptF=False): """Creates a DFA state machine that incorporates the paralell process of all pattern passed as state machines in the StateMachine_List. Each origins of each state machine are kept in the final state, if it is not dominated. Performs: -- parallelization -- translation from NFA to DFA -- Frank Schaefers Adapted Hopcroft optimization. Again: The state machine ids of the original state machines are traced through the whole process. FilterDominatedOriginsF, if set to False, can disable the filtering of dominated origins. This is important for pre-contexts, because, all successful patterns need to be reported! """ if len(StateMachine_List) == 0: return None def __check(Place, sm, AlllowInitStateAcceptF): __check_on_orphan_states(Place, sm) if not AlllowInitStateAcceptF: __check_on_init_state_not_acceptance(Place, sm) def __check_on_orphan_states(Place, sm): orphan_state_list = sm.get_orphaned_state_index_list() if len(orphan_state_list) == 0: return error.log("After '%s'" % Place + "\n" + \ "Orphaned state(s) detected in regular expression (optimization lack).\n" + \ "Please, log a defect at the projects website quex.sourceforge.net.\n" + \ "Orphan state(s) = " + repr(orphan_state_list)) def __check_on_init_state_not_acceptance(Place, sm): if sm.get_init_state().is_acceptance(): error.log("After '%s'" % Place + "\n" + \ "Initial state 'accepts'. This should never happen.\n" + \ "Please, log a defect at the projects web site quex.sourceforge.net.\n") # (1) mark at each state machine the machine and states as 'original'. # # This is necessary to trace in the combined state machine the pattern that # actually matched. Note, that a state machine in the StateMachine_List # represents one possible pattern that can match the current input. # for sm in StateMachine_List: if sm.get_id() in MarkNotSet: continue sm.mark_state_origins() assert sm.is_DFA_compliant(), sm.get_string(Option="hex") # (2) setup all patterns in paralell sm = parallelize.do(StateMachine_List, CommonTerminalStateF=False) #, CloneF=False) __check("Parallelization", sm, AlllowInitStateAcceptF) # (4) determine for each state in the DFA what is the dominating original # state if FilterDominatedOriginsF: sm.filter_dominated_origins() __check("Filter Dominated Origins", sm, AlllowInitStateAcceptF) # (3) convert the state machine to an DFA (paralellization created an NFA) sm = beautifier.do(sm) __check("NFA to DFA, Hopcroft Minimization", sm, AlllowInitStateAcceptF) return sm
out_n = 0 def output(Sm): global out_n if "help" not in sys.argv: print "sm%i: %s" % (out_n, Sm) else: open("tmp%i.dot" % out_n, "wb").write(Sm.get_graphviz_string(NormalizeF=True)) print "written 'tmp%i.dot'" % out_n out_n += 1 sm0.mark_state_origins() sm1.mark_state_origins() sm2.mark_state_origins() output(sm0) output(sm1) output(sm2) sm_list = [sm0, sm1, sm2] sm = parallelize.do(sm_list) print "#-------------------------------------------------------------------------------" output(sm) complement_sm = complement.do(sm) assert all(superset.do(sm, tsm) == True for tsm in sm_list) assert all(DFA.is_Empty(intersection.do([complement_sm, tsm])) for tsm in sm_list) print "<terminated>"
print "Ranking of state machines: Filter dominated origins" sys.exit(0) #---------------------------------------------------------------------------------------- # (*) setup the state machines sm0 = repeat.do(sm0) sm0.mark_state_origins() sm1.mark_state_origins() sm2.mark_state_origins() sm3 = repeat.do(sm3, 1) sm3.mark_state_origins() # -- paralellize the patterns sm = parallelize.do([sm0, sm1, sm2, sm3]) # -- create the optimized DFA for the patterns running in paralell sm = beautifier.do(sm) #---------------------------------------------------------------------------------------- # (*) create some priorities in between the patterns def add_priority(A, B): print "priority: %s > %s" % (repr(A.get_id()), repr(B.get_id())) state_machine_ranking_db_register(A.get_id(), B.get_id()) add_priority(sm3, sm0) add_priority(sm3, sm1) add_priority(sm3, sm2) add_priority(sm0, sm1)
print "Ranking of state machines: Filter dominated origins" sys.exit(0) #---------------------------------------------------------------------------------------- # (*) setup the state machines sm0 = repeat.do(sm0) sm0.mark_state_origins() sm1.mark_state_origins() sm2.mark_state_origins() sm3 = repeat.do(sm3, 1) sm3.mark_state_origins() # -- paralellize the patterns sm = parallelize.do([sm0, sm1, sm2, sm3]) # -- create the optimized DFA for the patterns running in paralell sm = beautifier.do(sm) #---------------------------------------------------------------------------------------- # (*) create some priorities in between the patterns def add_priority(A, B): print "priority: %s > %s" % (repr(A.get_id()), repr(B.get_id())) state_machine_ranking_db_register(A.get_id(), B.get_id()) add_priority(sm3, sm0) add_priority(sm3, sm1) add_priority(sm3, sm2) add_priority(sm0, sm1) # register some more state relations for some of the states before
import quex.engine.state_machine.algorithm.beautifier as beautifier import quex.engine.state_machine.algebra.reverse as reverse if "--hwut-info" in sys.argv: print "Tracing origin: Inverse" sys.exit(0) def invert_this(sm): print "-------------------------------------------------------------------------------" print "sm = ", sm tmp = reverse.do(sm) print "inverse = ", tmp return tmp sm0.mark_state_origins() # sm1.mark_state_origins() sm2.mark_state_origins() sm0 = invert_this(sm0) # invert_this(sm1) sm2 = invert_this(sm2) sm = parallelize.do([sm0, sm2]) print "-------------------------------------------------------------------------------" print "## paralellized = ", sm sm = beautifier.do(sm) print "## result = ", sm
import quex.engine.state_machine.construction.parallelize as parallelize from quex.engine.state_machine.TEST.test_state_machines import * if "--hwut-info" in sys.argv: print "StateMachine Operations: Paralell" sys.exit(0) print "-------------------------------------------------------------------------------" tsm0 = trivial_sm('a') tsm1 = trivial_sm('b') tsm2 = trivial_sm('c') print "##tsm0:", tsm0 print "##tsm1:", tsm1 print "##tsm2:", tsm2 sm = parallelize.do([tsm0, tsm1, tsm2]) print "EXAMPLE 0:", sm print "-------------------------------------------------------------------------------" print "##sm0", sm0 print "##sm1", sm1 print "##sm2", sm2 print "-------------------------------------------------------------------------------" sm = parallelize.do([sm0, sm1, sm2]) print "EXAMPLE A:", sm print "-------------------------------------------------------------------------------" empty_state_machine = StateMachine(7777) empty_state_machine.get_init_state().set_acceptance(True) sm = parallelize.do([
import os sys.path.insert(0, os.environ["QUEX_PATH"]) from quex.engine.state_machine.core import * import quex.input.regular_expression.engine as regex import quex.engine.state_machine.algorithm.nfa_to_dfa as nfa_to_dfa import quex.engine.state_machine.construction.parallelize as parallelize if "--hwut-info" in sys.argv: print "Tracing origin: NFA to DFA (subset construction) II" sys.exit(0) sm1 = regex.do('[A-Z]+', {}).sm sm2 = regex.do('"PR"', {}).sm # (*) create the DFA from the specified NFA sm1.mark_state_origins() sm2.mark_state_origins() print "## sm1 = ", sm1.get_string(OriginalStatesF=False, NormalizeF=True) print "## sm2 = ", sm2.get_string(OriginalStatesF=False, NormalizeF=True) sm = parallelize.do([sm1, sm2]) print "## parallelized:", sm.get_string(NormalizeF=True, OriginalStatesF=False) # print "## parallelized:", sm.get_string(NormalizeF=False) dfa = nfa_to_dfa.do(sm) print dfa.get_string(OriginalStatesF=False, NormalizeF=True)
import quex.engine.state_machine.construction.parallelize as parallelize from quex.engine.state_machine.TEST.test_state_machines import * import quex.engine.state_machine.algorithm.beautifier as beautifier import quex.engine.state_machine.algebra.reverse as reverse if "--hwut-info" in sys.argv: print "Tracing origin: Inverse" sys.exit(0) def invert_this(sm): print "-------------------------------------------------------------------------------" print "sm = ", sm tmp = reverse.do(sm) print "inverse = ", tmp return tmp sm0.mark_state_origins() # sm1.mark_state_origins() sm2.mark_state_origins() sm0 = invert_this(sm0) # invert_this(sm1) sm2 = invert_this(sm2) sm = parallelize.do([sm0, sm2]) print "-------------------------------------------------------------------------------" print "## paralellized = ", sm sm = beautifier.do(sm) print "## result = ", sm
import sys import os sys.path.insert(0, os.environ["QUEX_PATH"]) from quex.engine.state_machine.core import * import quex.input.regular_expression.engine as regex import quex.engine.state_machine.algorithm.nfa_to_dfa as nfa_to_dfa import quex.engine.state_machine.construction.parallelize as parallelize if "--hwut-info" in sys.argv: print "Tracing origin: NFA to DFA (subset construction) II" sys.exit(0) sm1 = regex.do('[A-Z]+', {}).sm sm2 = regex.do('"PR"', {}).sm # (*) create the DFA from the specified NFA sm1.mark_state_origins() sm2.mark_state_origins() print "## sm1 = ", sm1.get_string(OriginalStatesF=False, NormalizeF=True) print "## sm2 = ", sm2.get_string(OriginalStatesF=False, NormalizeF=True) sm = parallelize.do([sm1, sm2]) print "## parallelized:", sm.get_string(NormalizeF=True, OriginalStatesF=False) # print "## parallelized:", sm.get_string(NormalizeF=False) dfa = nfa_to_dfa.do(sm) print dfa.get_string(OriginalStatesF=False, NormalizeF=True)
def get_combined_state_machine(StateMachine_List, FilterDominatedOriginsF=True, MarkNotSet=set()): """Creates a DFA state machine that incorporates the paralell process of all pattern passed as state machines in the StateMachine_List. Each origins of each state machine are kept in the final state, if it is not dominated. Performs: -- parallelization -- translation from NFA to DFA -- Frank Schaefers Adapted Hopcroft optimization. Again: The state machine ids of the original state machines are traced through the whole process. FilterDominatedOriginsF, if set to False, can disable the filtering of dominated origins. This is important for pre-contexts, because, all successful patterns need to be reported! """ if len(StateMachine_List) == 0: return None def __check(Place, sm): __check_on_orphan_states(Place, sm) __check_on_init_state_not_acceptance(Place, sm) def __check_on_orphan_states(Place, sm): orphan_state_list = sm.get_orphaned_state_index_list() if len(orphan_state_list) == 0: return error.log("After '%s'" % Place + "\n" + \ "Orphaned state(s) detected in regular expression (optimization lack).\n" + \ "Please, log a defect at the projects website quex.sourceforge.net.\n" + \ "Orphan state(s) = " + repr(orphan_state_list)) def __check_on_init_state_not_acceptance(Place, sm): if sm.get_init_state().is_acceptance(): error.log("After '%s'" % Place + "\n" + \ "Initial state 'accepts'. This should never happen.\n" + \ "Please, log a defect at the projects web site quex.sourceforge.net.\n") # (1) mark at each state machine the machine and states as 'original'. # # This is necessary to trace in the combined state machine the # pattern that actually matched. Note, that a state machine in # the StateMachine_List represents one possible pattern that can # match the current input. # for sm in StateMachine_List: if sm.get_id() in MarkNotSet: continue sm.mark_state_origins() assert sm.is_DFA_compliant(), sm.get_string(Option="hex") # (2) setup all patterns in paralell sm = parallelize.do(StateMachine_List, CommonTerminalStateF=False) #, CloneF=False) __check("Parallelization", sm) # (4) determine for each state in the DFA what is the dominating original state if FilterDominatedOriginsF: sm.filter_dominated_origins() __check("Filter Dominated Origins", sm) # (3) convert the state machine to an DFA (paralellization created an NFA) sm = beautifier.do(sm) __check("NFA to DFA, Hopcroft Minimization", sm) return sm
from quex.engine.state_machine.TEST.test_state_machines import * if "--hwut-info" in sys.argv: print "StateMachine Operations: Paralell" sys.exit(0) print "-------------------------------------------------------------------------------" tsm0 = trivial_sm('a') tsm1 = trivial_sm('b') tsm2 = trivial_sm('c') print "##tsm0:", tsm0 print "##tsm1:", tsm1 print "##tsm2:", tsm2 sm = parallelize.do([tsm0, tsm1, tsm2]) print "EXAMPLE 0:", sm print "-------------------------------------------------------------------------------" print "##sm0", sm0 print "##sm1", sm1 print "##sm2", sm2 print "-------------------------------------------------------------------------------" sm = parallelize.do([sm0, sm1, sm2]) print "EXAMPLE A:", sm print "-------------------------------------------------------------------------------" empty_state_machine = StateMachine(7777) empty_state_machine.get_init_state().set_acceptance(True) sm = parallelize.do([empty_state_machine, sm0,