def _test_search3(self):
re = "/a+b*c+/\n/b+c*d+/\n/c+d*e/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
aut.compute()
self.assertEqual(delay_dfa.search("abcd"), aut.search("abcd"))
self.assertEqual(delay_dfa.search("abcd abc"), aut.search("abcd abc"))
self.assertEqual(delay_dfa.search("gabc"), aut.search("gabc"))
self.assertEqual(delay_dfa.search("ac bd ce"), aut.search("ac bd ce"))
self.assertEqual(delay_dfa.search("ab123a bcd cde"),
aut.search("ab123a bcd cde"))
self.assertEqual(delay_dfa.search("bce"), aut.search("bce"))
self.assertEqual(delay_dfa.search("abe"), aut.search("abe"))
def _test_compute1(self):
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text("/^abcd/")
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.compute()
a = delay_dfa.get_automaton()
b = dfa.get_automaton()
l = len(b.alphabet.keys())
b.add_symbols(DEF_SYMBOLS("default", l))
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags["Delay DFA"])
def _test_search2(self):
re = "/a+b*c+d/\n/abc/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
aut.compute()
self.assertEqual(delay_dfa.search("abcd"), aut.search("abcd"))
self.assertEqual(delay_dfa.search("abcd abc"), aut.search("abcd abc"))
self.assertEqual(delay_dfa.search("abc"), aut.search("abc"))
self.assertEqual(delay_dfa.search("acd"), aut.search("acd"))
self.assertEqual(delay_dfa.search("abd"), aut.search("abd"))
def _test_search2(self):
re = "/a+b*c+d/\n/abc/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
aut.compute()
self.assertEqual(delay_dfa.search("abcd"), aut.search("abcd"))
self.assertEqual(delay_dfa.search("abcd abc"), aut.search("abcd abc"))
self.assertEqual(delay_dfa.search("abc"), aut.search("abc"))
self.assertEqual(delay_dfa.search("acd"), aut.search("acd"))
self.assertEqual(delay_dfa.search("abd"), aut.search("abd"))
def _test_compute_1(self):
"""compute()"""
"""
Test without blow up patterns
"""
hyfa = JHybridFA()
parser = pcre_parser()
hyfa.set_parser(parser)
hyfa.load_file(aux_func.getPatternMatchDir() + "/algorithms/j_hybrid_fa/tests_data/test_compute_1.re")
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser = pcre_parser()
parser.load_file(aux_func.getPatternMatchDir() + "/algorithms/j_hybrid_fa/tests_data/test_compute_1.re")
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.compute()
a = hyfa.dfa.get_automaton(False)
b = dfa.get_automaton(False)
# Test without blow up patterns
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags['Hybrid FA - DFA part'])
self.assertTrue(a.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def _test_search3(self):
re = "/a+b*c+/\n/b+c*d+/\n/c+d*e/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
aut.compute()
self.assertEqual(delay_dfa.search("abcd"), aut.search("abcd"))
self.assertEqual(delay_dfa.search("abcd abc"), aut.search("abcd abc"))
self.assertEqual(delay_dfa.search("gabc"), aut.search("gabc"))
self.assertEqual(delay_dfa.search("ac bd ce"), aut.search("ac bd ce"))
self.assertEqual(delay_dfa.search("ab123a bcd cde"), aut.search("ab123a bcd cde"))
self.assertEqual(delay_dfa.search("bce"), aut.search("bce"))
self.assertEqual(delay_dfa.search("abe"), aut.search("abe"))
def __init__(self):
"""
Constructor - inits object atributes:
dfa - Head DFA part of Hybrid FA
nfas - Tails NFA parts of Hybrid FA
tran_aut - List of border states
Index in self.tran_aut reffer to
self.nfas list for particular NFA tails.
_compute - Flag call to compute()
_depthOfStates - Depth for each state from initial state
_stateTrans -
_maper - Used to map state to (symbol, target) list
_sort - Sorted transitions
_head_size - Size of head DFA part
"""
b_Automaton.__init__(self)
self._compute = False
self._depthOfStates = dict()
self._stateTrans = dict()
self._mapper = None
self.dfa = b_dfa()
self.nfas = dict()
self.tran_aut = dict()
self._sort = {}
self._head_size = 0
self._nfaEpsFree = b_nfa()
self._SPECIAL_MIN_DEPTH = 5 # minimum NFA depth of a special state
self._MAX_TX = 250 # maximum number of outgoing transitions allowed in a expanded NFA state
self._MAX_HEAD_SIZE = 1000 # maximum number of states in the head automaton before starting to create tails
def _test_compute1(self):
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text("/^abcd/")
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.compute()
a = delay_dfa.get_automaton()
b = dfa.get_automaton()
l = len(b.alphabet.keys())
b.add_symbols(DEF_SYMBOLS("default", l))
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags["Delay DFA"])
def _test_compute_2(self):
"""compute()"""
"""
Test with one regular expression, where computed automaton
has one NFA tail
"""
hyfa = hybrid_fa()
parser = pcre_parser()
parser.set_text("/abcd/")
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(2)
hyfa.set_max_head_size(-1)
hyfa.set_max_tx(-1)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser_dfa = pcre_parser()
parser_dfa.set_text("/ab/")
dfa = b_dfa()
dfa.create_by_parser(parser_dfa)
dfa.determinise()
hd = hyfa.dfa.get_automaton()
hn0 = hyfa.nfas[0].get_automaton()
d = dfa.get_automaton()
n = nfa_data().load_from_file(
aux_func.getPatternMatchDir() +
"/algorithms/hybrid_fa/tests_data/test_compute_2_nfa0.nfa_data")
# test on automaton where is one NFA tail
# test of DFA part
self.assertEqual(hd.states.keys(), d.states.keys())
self.assertEqual(hd.alphabet, d.alphabet)
self.assertEqual(hd.start, d.start)
self.assertEqual(len(hd.final), 0)
self.assertEqual(hd.transitions, d.transitions)
self.assertTrue(hd.Flags['Hybrid FA - DFA part'])
self.assertTrue(hd.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 1)
self.assertEqual(hyfa.tran_aut, {0: 2})
# test of NFA part #0
self.assertEqual(hn0.states.keys(), n.states.keys())
self.assertEqual(hn0.alphabet, n.alphabet)
self.assertEqual(hn0.start, n.start)
self.assertEqual(hn0.final, n.final)
self.assertEqual(hn0.transitions, n.transitions)
self.assertTrue(hn0.Flags['Hybrid FA - one NFA part'])
def _test_compute_2(self):
"""compute()"""
"""
Test with one regular expression, where computed automaton
has one NFA tail
"""
hyfa = hybrid_fa()
parser = pcre_parser()
parser.set_text("/abcd/")
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(2)
hyfa.set_max_head_size(-1)
hyfa.set_max_tx(-1)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser_dfa = pcre_parser()
parser_dfa.set_text("/ab/")
dfa = b_dfa()
dfa.create_by_parser(parser_dfa)
dfa.determinise()
hd = hyfa.dfa.get_automaton()
hn0 = hyfa.nfas[0].get_automaton()
d = dfa.get_automaton()
n = nfa_data().load_from_file(aux_func.getPatternMatchDir() + "/algorithms/hybrid_fa/tests_data/test_compute_2_nfa0.nfa_data")
# test on automaton where is one NFA tail
# test of DFA part
self.assertEqual(hd.states.keys(), d.states.keys())
self.assertEqual(hd.alphabet, d.alphabet)
self.assertEqual(hd.start, d.start)
self.assertEqual(len(hd.final), 0)
self.assertEqual(hd.transitions, d.transitions)
self.assertTrue(hd.Flags['Hybrid FA - DFA part'])
self.assertTrue(hd.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 1)
self.assertEqual(hyfa.tran_aut, {0: 2})
# test of NFA part #0
self.assertEqual(hn0.states.keys(), n.states.keys())
self.assertEqual(hn0.alphabet, n.alphabet)
self.assertEqual(hn0.start, n.start)
self.assertEqual(hn0.final, n.final)
self.assertEqual(hn0.transitions, n.transitions)
self.assertTrue(hn0.Flags['Hybrid FA - one NFA part'])
def _test_compute_3(self):
"""compute()"""
"""
Test with more regular expressions, where computed automaton
has only DFA part without any NFA tails
"""
parser = pcre_parser()
parser.load_file(
aux_func.getPatternMatchDir() +
"/algorithms/hybrid_fa/tests_data/test_compute_3_pattern.re")
hyfa = hybrid_fa()
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(10)
hyfa.set_max_head_size(-1)
hyfa.set_max_tx(-1)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser = pcre_parser()
parser.load_file(
aux_func.getPatternMatchDir() +
"/algorithms/hybrid_fa/tests_data/test_compute_3_pattern.re")
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.determinise()
hd = hyfa.dfa.get_automaton()
d = dfa.get_automaton()
# test of DFA part
self.assertEqual(hd.states.keys(), d.states.keys())
self.assertEqual(hd.alphabet, d.alphabet)
self.assertEqual(hd.start, d.start)
self.assertEqual(hd.final, d.final)
self.assertEqual(hd.transitions, d.transitions)
self.assertTrue(hd.Flags['Hybrid FA - DFA part'])
self.assertTrue(hd.Flags['Deterministic'])
# without NFA tails
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def _test_compute_3(self):
"""compute()"""
"""
Test with more regular expressions, where computed automaton
has only DFA part without any NFA tails
"""
parser = pcre_parser()
parser.load_file(aux_func.getPatternMatchDir() + "/algorithms/hybrid_fa/tests_data/test_compute_3_pattern.re")
hyfa = hybrid_fa()
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(10)
hyfa.set_max_head_size(-1)
hyfa.set_max_tx(-1)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser = pcre_parser()
parser.load_file(aux_func.getPatternMatchDir() + "/algorithms/hybrid_fa/tests_data/test_compute_3_pattern.re")
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.determinise()
hd = hyfa.dfa.get_automaton()
d = dfa.get_automaton()
# test of DFA part
self.assertEqual(hd.states.keys(), d.states.keys())
self.assertEqual(hd.alphabet, d.alphabet)
self.assertEqual(hd.start, d.start)
self.assertEqual(hd.final, d.final)
self.assertEqual(hd.transitions, d.transitions)
self.assertTrue(hd.Flags['Hybrid FA - DFA part'])
self.assertTrue(hd.Flags['Deterministic'])
# without NFA tails
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def __init__(self):
"""
Constructor - inits object atributes:
dfa - Head DFA part of Hybrid FA
nfas - Tails NFA parts of Hybrid FA
tran_aut - List of border states
Index in self.tran_aut reffer to
self.nfas list for particular NFA tails.
"""
b_Automaton.__init__(self)
self.dfa = b_dfa()
self.nfas = dict()
self.tran_aut = dict()
self._patterns = [] # compiled pattern to finding blow up
self._nfaEpsFree = b_nfa()
self._init_blowup_expresion()
def _test_search1(self):
re = "/a+b*c.*a|bc+/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
self.assertEqual(delay_dfa.search("ac123ac"), aut.search("ac123ac"))
self.assertEqual(delay_dfa.search("aacac"), aut.search("aacac"))
self.assertEqual(delay_dfa.search("abbb"), aut.search("abbb"))
def _test_search1(self):
re = "/a+b*c.*a|bc+/"
delay_dfa = DELAY_DFA()
parser = pcre_parser()
parser.set_text(re)
delay_dfa.create_by_parser(parser)
delay_dfa.compute()
self.assertTrue(delay_dfa.get_compute())
aut = b_dfa()
parser = pcre_parser()
parser.set_text(re)
aut.create_by_parser(parser)
self.assertEqual(delay_dfa.search("ac123ac"), aut.search("ac123ac"))
self.assertEqual(delay_dfa.search("aacac"), aut.search("aacac"))
self.assertEqual(delay_dfa.search("abbb"), aut.search("abbb"))
def _test_compute_1(self):
"""compute()"""
"""
Test without blow up patterns
"""
hyfa = JHybridFA()
parser = pcre_parser()
hyfa.set_parser(parser)
hyfa.load_file(aux_func.getPatternMatchDir() +
"/algorithms/j_hybrid_fa/tests_data/test_compute_1.re")
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
parser = pcre_parser()
parser.load_file(
aux_func.getPatternMatchDir() +
"/algorithms/j_hybrid_fa/tests_data/test_compute_1.re")
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.compute()
a = hyfa.dfa.get_automaton(False)
b = dfa.get_automaton(False)
# Test without blow up patterns
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags['Hybrid FA - DFA part'])
self.assertTrue(a.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def _test_compute_1(self):
"""compute()"""
"""
Test with one regular expression, where computed automaton
has only DFA part without any NFA tails.
"""
hyfa = hybrid_fa()
parser = pcre_parser()
parser.set_text("/abcd/")
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(10)
hyfa.set_max_head_size(10)
hyfa.set_max_tx(10)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.determinise()
a = hyfa.dfa.get_automaton()
b = dfa.get_automaton()
# test on automaton where is only DFA part without NFA tails
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags['Hybrid FA - DFA part'])
self.assertTrue(a.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def _test_compute_1(self):
"""compute()"""
"""
Test with one regular expression, where computed automaton
has only DFA part without any NFA tails.
"""
hyfa = hybrid_fa()
parser = pcre_parser()
parser.set_text("/abcd/")
hyfa.create_by_parser(parser)
hyfa.set_special_min_depth(10)
hyfa.set_max_head_size(10)
hyfa.set_max_tx(10)
hyfa.compute()
# self.get_compute() has to be True
self.assertTrue(hyfa.get_compute())
dfa = b_dfa()
dfa.create_by_parser(parser)
dfa.determinise()
a = hyfa.dfa.get_automaton()
b = dfa.get_automaton()
# test on automaton where is only DFA part without NFA tails
self.assertEqual(a.states.keys(), b.states.keys())
self.assertEqual(a.alphabet, b.alphabet)
self.assertEqual(a.start, b.start)
self.assertEqual(a.final, b.final)
self.assertEqual(a.transitions, b.transitions)
self.assertTrue(a.Flags['Hybrid FA - DFA part'])
self.assertTrue(a.Flags['Deterministic'])
self.assertEqual(len(hyfa.nfas), 0)
self.assertEqual(hyfa.tran_aut, {})
def compute(self):
"""
Computes Hybrid automaton.
"""
b_Automaton.compute(self)
self._compute = False
# save input NFA automaton
self._nfaEpsFree = b_nfa()
parserInstance = self._parser
parserInstance.load_file(self._file_name)
self._nfaEpsFree.create_by_parser(parserInstance)
self._nfaEpsFree.remove_epsilons()
fr = open(self._file_name, "r")
# split input REs into DFA and NFA parts
# RE can have blow up on start, then is not DFA part
# RE can have not blow up, then is not NFA part
dfa_patterns = "" # dfa patterns are set into parser
nfa_patterns = [] # each nfa pattern is set to parser
# create mapping beetween number of input RE and number of
# RE in DFA patterns and NFA patterns
re_to_nfa = dict()
re_to_dfa = dict()
nfa_to_re = dict()
dfa_to_re = dict()
nfa_i = 0
dfa_i = 0
i = 0
for line in fr.readlines():
(a, b) = self._split_expresion(line.rstrip("\n"))
if a is not "":
re_to_dfa[i] = dfa_i
dfa_to_re[dfa_i] = i
dfa_i+=1
dfa_patterns += a+"\n"
if b is not "":
nfa_patterns.append(b)
re_to_nfa[i] = nfa_i
nfa_to_re[nfa_i] = i
nfa_i+=1
i+=1
dfa_patterns = dfa_patterns[:-1] # delete last new line
# compute DFA head part
self.dfa = b_dfa()
self.dfa.set_multilanguage(self.get_multilanguage())
parserInstance = self._parser
parserInstance.set_text(dfa_patterns)
self.dfa.create_by_parser(parserInstance)
self.dfa.compute()
self.dfa.get_automaton(False).Flags["Hybrid FA - DFA part"] = True
# compute NFA tails
for key in range(0, len(nfa_patterns)):
nfa = b_nfa()
parserInstance = self._parser
parserInstance.set_text(nfa_patterns[key])
nfa.create_by_parser(parserInstance)
nfa.remove_epsilons()
nfa.compute()
nfa.get_automaton(False).Flags["Hybrid FA - one NFA part"] = True
# update RE number in finals states
for state in nfa.get_automaton(False).states:
if nfa.get_automaton(False).states[state].is_final():
new_regexp_number = set()
new_regexp_number.add(nfa_to_re[key])
nfa.get_automaton(False).states[state].set_regexp_number(new_regexp_number)
self.nfas[len(self.nfas)] = nfa
# update RE number in finals states or map border state to NFA
for state in self.dfa.get_automaton().states:
if self.dfa.get_automaton().states[state].is_final() == True:
new_regexp_numbers = set()
for re_number in self.dfa.get_automaton(False).states[state].get_regexp_number():
if re_to_nfa.has_key(dfa_to_re[re_number]):
self.tran_aut[re_to_nfa[dfa_to_re[re_number]]] = state
else:
new_regexp_numbers.add(dfa_to_re[re_number])
self.dfa.get_automaton(False).states[state].set_regexp_number(new_regexp_numbers)
# add RE having blow up pattern on start
new_tran_aut = dict()
for nfa in nfa_to_re:
if self.tran_aut.has_key(nfa) is False:
new_tran_aut[nfa] = self.dfa.get_automaton(False).start
else:
new_tran_aut[nfa] = self.tran_aut[nfa]
self.tran_aut = new_tran_aut
self._compute = True