Exemplo n.º 1
0
    def testEmptyClosure(self):
        nfa = NFA('01')

        s0 = nfa.new_state(initial=True)

        closure = nfa.closure(s0)
        self.assertIn(s0, closure, "Closure start state should be in closure")
  def test_cat(self):
    a = NFA.build_a('a')
    b = NFA.build_a('b')
    ab = NFA.build_cat(a, b)

    self.assert_(ab.initial)
    self.assert_(ab.finals)
class TestNFA(unittest.TestCase):
  def setUp(self):
    self.dg = {
          0: {'': [1, 7],},
          1: {'': [2, 4],},
          2: {'a': [3,]},
          3: {'': [6,], },
          4: {'b': [5,], },
          5: {'': [6,], },
          6: {'': [1, 7,], },
          7: {'a': [8,], },
          8: {'b': [9,], },
          9: {'b': [10,], },
          10: {}
    }
        
    self.nfa = NFA(self.dg, 0, frozenset([10]))

  def test_eclosure_0(self):
    given = frozenset((0,))
    expected = frozenset((0, 1, 2, 4, 7))
    self.assertEqual(self.nfa.eclosure(given), expected)

  def test_eclosure_2(self):
    given = frozenset((2,))
    expected = frozenset((2,))
    self.assertEqual(self.nfa.eclosure(given), expected)

  def test_move_2_3(self):
    given = frozenset((2,))
    expected = frozenset((3,))
    self.assertEqual(self.nfa.move(given, 'a'), expected)

  def test_move_6_8(self):
    given = frozenset((6,))
    expected = frozenset((3, 8,))
    self.assertEqual(self.nfa.move(given, 'a'), expected)
    
  def test_makeDFA(self):
    '''
      from p143
    '''
    A = frozenset([0, 1, 2, 4, 7])
    B = frozenset([1, 2, 3, 4, 6, 7, 8])
    C = frozenset([1, 2, 4, 5, 6, 7])
    D = frozenset([1, 2, 4, 5, 6, 7, 9])
    E = frozenset([1, 2, 4, 5, 6, 7, 10])

    expected = {
      A: {'a': B, 'b': C},
      B: {'a': B, 'b': D},
      C: {'a': B, 'b': C},
      D: {'a': B, 'b': E},
      E: {'a': B, 'b': C},
    }
    self.assertEqual(self.nfa.makeDFA(), expected)
Exemplo n.º 4
0
def test(pattern, path, count):
  print 'fnmatch(%s, %s) %d times...' % (path, pattern, count)
  start_compile = datetime.now()
  compiled = Compiled(DFA(NFA.fnmatch(pattern)))
  compiled.optimize() # slow!
  end_compile = datetime.now()

  start_execution = datetime.now()
  for x in range(count):
    compiled(path)
  end_execution = datetime.now()

  native_test = compile_native_test(compiled, pattern, path, count)

  start_native = datetime.now()
  native_test()
  end_native = datetime.now()

  from fnmatch import fnmatch

  start_builtin = datetime.now()
  for x in range(count):
    fnmatch(path, pattern)
  end_builtin = datetime.now()

  return {
    'pattern': pattern,
    'path': path,
    'count': count,
    'compile_time': (end_compile-start_compile),
    'execution_time': (end_execution-start_execution),
    'native_time': (end_native-start_native),
    'builtin_time': (end_builtin-start_builtin),
  }
 def test_a(self):
   a = NFA.build_a('a')
   self.assert_(a.initial)
   self.assert_(a.finals)
   self.assertEqual(len(a.states) , 2)
   self.assertEqual(
     a.states[a.initial],
     dict({'a': a.finals})
   )
 def test_Empty(self):
   emp = NFA.build_empty()
   self.assert_(emp.initial)
   self.assert_(emp.finals)
   self.assertEqual(len(emp.states) , 2)
   self.assertEqual(
     emp.states[emp.initial],
     dict({'': emp.finals})
   )
 def result(self):
   if len(self._result) == 1:
     return self._result[0]
   elif len(self._result) == 0:
     return NFA.build_empty()
   else:
     print 'got multiple nfas?!'
     for n in self._result:
       print n, n.states
     assert False
Exemplo n.º 8
0
class RegexMatcher(object):
    """ A basic regex parser for ascii characters -- initialize with a
        pattern passed to either the class constructor or set_pattern
        
        Use match* functions to find matches in input strings
    """
    def __init__(self, pattern):
        self.setPattern(pattern)
        # pass

# generates the state machine for the regex specified by pattern
    def setPattern(self, pattern):
        self.pattern = pattern
        self.stateMachine = NFA(pattern)

    def findLongestMatch(self, inStr, i):
        self.stateMachine.reset()
        j = i
        prevFinishedIndex = -1
        while j < len(inStr) and self.stateMachine.advanceStates(inStr[j]):
            j += 1
            if self.stateMachine.finished():
                prevFinishedIndex = j

        if prevFinishedIndex != -1:
            return inStr[i:prevFinishedIndex]
        else:
            return ""

# returns a tuple of (int, str) that contains the integer of the index
# of the first char matching the pattern in input string and the str that
# matches the pattern
# if no match if found, returns a () empty tuple
    def matchFirst(self, inStr, searchStart=0):
        self.stateMachine.reset()
        for i in range(searchStart, len(inStr)):
            match = self.findLongestMatch(inStr, i)
            if match:
                return (i, match)
            
        return ()

# same as matchFirst, except returns a list of all (int, str)
# pairs that match the pattern sent to set_pattern
# if no matches returns []
    def matchAll(self, inStr):
        result = []
        prevMatch = self.matchFirst(inStr, 0)
        while prevMatch:
            result.append(prevMatch)
            startIndex, match = prevMatch
            iOfLastMatch = startIndex + len(match)
            prevMatch = self.matchFirst(inStr, iOfLastMatch)

        return result

    def __str__(self):
        return self.pattern + "\n" + str(self.stateMachine)
 def setUp(self):
   self.dg = {
         0: {'': [1, 7],},
         1: {'': [2, 4],},
         2: {'a': [3,]},
         3: {'': [6,], },
         4: {'b': [5,], },
         5: {'': [6,], },
         6: {'': [1, 7,], },
         7: {'a': [8,], },
         8: {'b': [9,], },
         9: {'b': [10,], },
         10: {}
   }
       
   self.nfa = NFA(self.dg, 0, frozenset([10]))
 def alpha(self, t):
   nfa = NFA.build_a(t.raw) #ugh! t.value?
   self.push(nfa)
Exemplo n.º 11
0
    def testClosure(self):
        nfa = NFA('01')

        s0 = nfa.new_state(initial=True)
        s1 = nfa.new_state()
        s2 = nfa.new_state()
        s3 = nfa.new_state(final=True)
        s4 = nfa.new_state()

        nfa.new_edge(s0, '0', s1)
        nfa.new_edge(s0, Epsilon, s2)
        nfa.new_edge(s1, '0', s3)
        nfa.new_edge(s1, Epsilon, s4)
        nfa.new_edge(s2, '1', s2)
        nfa.new_edge(s2, Epsilon, s3)

        closure = nfa.closure(s0)
        self.assertIn(s0, closure, "Closure start state s0 should be in closure")
        self.assertNotIn(s1, closure, "State s1 should not be in closure")
        self.assertIn(s2, closure, "State s2 not in closure")
        self.assertIn(s3, closure, "State s3 not in closure")
        self.assertNotIn(s4, closure, "State s4 should not be in closure")
Exemplo n.º 12
0
    def setUp(self):
        """
        Define one NFA a that accept all strings in the regular language '00*1'
        Define a second NFA accepting all string in the regular language '11*0'
        """
        fa = NFA('01')
        sa0 = fa.new_state(initial=True, name='sa0')
        sa1 = fa.new_state(name='sa1')
        sa2 = fa.new_state(final=True, name='sa2')

        fa.new_edge(sa0, '0', sa1)
        fa.new_edge(sa1, '0', sa1)
        fa.new_edge(sa1, '1', sa2)

        self.assertTrue(fa.test_input('01'), 'String "01" not accepted')
        self.assertFalse(fa.test_input('00'), 'String "00" not rejected')

        fb = NFA('01')
        sb0 = fb.new_state(initial=True, name='sb0')
        sb1 = fb.new_state(name='sb1')
        sb2 = fb.new_state(final=True, name='sb2')

        fb.new_edge(sb0, '1', sb1)
        fb.new_edge(sb1, '1', sb1)
        fb.new_edge(sb1, '0', sb2)

        self.assertTrue(fb.test_input('10'), 'String "10" not accepted')
        self.assertFalse(fb.test_input('11'), 'String "11" not rejected')

        self.nfa1 = fa
        self.nfa2 = fb
Exemplo n.º 13
0
 def setUp(self):
     self.fa = NFA('012')
Exemplo n.º 14
0
class TestSimpleDFA(unittest.TestCase):
    def setUp(self):
        self.fa = NFA('012')

    def test_instantiation(self):
        self.assertFalse(self.fa.no_of_states, 'States not empty at start')

    def test_add_state(self):
        """Newly created state should not have any transitions"""
        sid = self.fa.new_state()
        for s, p in self.fa.get_edges_from_state(sid).items():
            self.assertFalse(p, 'Found edge ({}, {}) in newly added state {}'.format(s, p, sid))

    def test_edge(self):
        fa = self.fa
        sid = fa.new_state()
        fa.new_edge(sid, '0', sid)
        edges = fa.get_edges_from_state(sid)
        q = edges['0']
        self.assertTrue(sid in q, 'Expected edge not found in {} for state {}'.format(q, sid))
        self.assertTrue(q - {sid} == set(), 'Unexpected edges found in {} for state {}'.format(q, sid))

    def test_has_edge(self):
        fa = self.fa
        s0 = fa.new_state()
        s1 = fa.new_state()
        s2 = fa.new_state()
        self.assertFalse(fa.has_edge(s0, s0), 'Should not have an edge s0->s0')
        self.assertFalse(fa.has_edge(s0, s1), 'Should not have an edge s0->s1')
        self.assertFalse(fa.has_edge(s0, s2), 'Should not have an edge s0->s2')
        self.assertFalse(fa.has_edge(s1, s1), 'Should not have an edge s1->s1')
        self.assertFalse(fa.has_edge(s1, s2), 'Should not have an edge s1->s2')
        self.assertFalse(fa.has_edge(s1, s0), 'Should not have an edge s1->s0')
        self.assertFalse(fa.has_edge(s2, s2), 'Should not have an edge s2->s2')
        self.assertFalse(fa.has_edge(s2, s1), 'Should not have an edge s2->s1')
        self.assertFalse(fa.has_edge(s2, s0), 'Should not have an edge s2->s0')

        fa.new_edge(s0, '0', s1)
        self.assertFalse(fa.has_edge(s0, s0), 'Should not have an edge s0->s0')
        self.assertTrue(fa.has_edge(s0, s1), 'Should have an edge s0->s1')
        self.assertFalse(fa.has_edge(s0, s2), 'Should not have an edge s0->s2')
        self.assertFalse(fa.has_edge(s1, s1), 'Should not have an edge s1->s1')
        self.assertFalse(fa.has_edge(s1, s2), 'Should not have an edge s1->s2')
        self.assertFalse(fa.has_edge(s1, s0), 'Should not have an edge s1->s0')
        self.assertFalse(fa.has_edge(s2, s2), 'Should not have an edge s2->s2')
        self.assertFalse(fa.has_edge(s2, s1), 'Should not have an edge s2->s1')
        self.assertFalse(fa.has_edge(s2, s0), 'Should not have an edge s2->s0')
        fa.new_edge(s0, '1', s2)
        self.assertFalse(fa.has_edge(s0, s0), 'Should not have an edge s0->s0')
        self.assertTrue(fa.has_edge(s0, s1), 'Should have an edge s0->s1')
        self.assertTrue(fa.has_edge(s0, s2), 'Should have an edge s0->s2')
        self.assertFalse(fa.has_edge(s1, s1), 'Should not have an edge s1->s1')
        self.assertFalse(fa.has_edge(s1, s2), 'Should not have an edge s1->s2')
        self.assertFalse(fa.has_edge(s1, s0), 'Should not have an edge s1->s0')
        self.assertFalse(fa.has_edge(s2, s2), 'Should not have an edge s2->s2')
        self.assertFalse(fa.has_edge(s2, s1), 'Should not have an edge s2->s1')
        self.assertFalse(fa.has_edge(s2, s0), 'Should not have an edge s2->s0')
        fa.new_edge(s0, '1', s0)
        self.assertTrue(fa.has_edge(s0, s0), 'Should have an edge s0->s0')
        self.assertTrue(fa.has_edge(s0, s1), 'Should have an edge s0->s1')
        self.assertTrue(fa.has_edge(s0, s2), 'Should have an edge s0->s2')
        self.assertFalse(fa.has_edge(s1, s1), 'Should not have an edge s1->s1')
        self.assertFalse(fa.has_edge(s1, s2), 'Should not have an edge s1->s2')
        self.assertFalse(fa.has_edge(s1, s0), 'Should not have an edge s1->s0')
        self.assertFalse(fa.has_edge(s2, s2), 'Should not have an edge s2->s2')
        self.assertFalse(fa.has_edge(s2, s1), 'Should not have an edge s2->s1')
        self.assertFalse(fa.has_edge(s2, s0), 'Should not have an edge s2->s0')
        fa.new_edge(s1, '0', s1)
        fa.new_edge(s1, '1', s0)
        fa.new_edge(s1, '1', s2)
        fa.new_edge(s2, '0', s0)
        self.assertTrue(fa.has_edge(s0, s0), 'Should have an edge s0->s0')
        self.assertTrue(fa.has_edge(s0, s1), 'Should have an edge s0->s1')
        self.assertTrue(fa.has_edge(s0, s2), 'Should have an edge s0->s2')
        self.assertTrue(fa.has_edge(s1, s1), 'Should have an edge s1->s1')
        self.assertTrue(fa.has_edge(s1, s2), 'Should have an edge s1->s2')
        self.assertTrue(fa.has_edge(s1, s0), 'Should have an edge s1->s0')
        self.assertFalse(fa.has_edge(s2, s2), 'Should not have an edge s2->s2')
        self.assertFalse(fa.has_edge(s2, s1), 'Should not have an edge s2->s1')
        self.assertTrue(fa.has_edge(s2, s0), 'Should have an edge s2->s0')

    def test_del_state(self):
        """Deleting a state should remove all edges to that state"""
        fa = self.fa
        s0 = fa.new_state()
        s1 = fa.new_state()
        s2 = fa.new_state()
        fa.new_edge(s0, '0', s1)
        fa.new_edge(s0, '1', s0)
        fa.new_edge(s0, '1', s2)
        fa.new_edge(s1, '0', s1)
        fa.new_edge(s1, '1', s0)
        fa.new_edge(s1, '1', s2)
        fa.new_edge(s2, '0', s0)

        fa.del_state(s0)
        self.assertEqual(fa.no_of_states, 2, 'Wrong number of states after delete ({})'.format(fa.no_of_states))
        edges = fa.get_edges_from_state(s1)
        self.assertFalse(s0 in [q for q in edges.values()], 'State {} remains a destination after delete')
        self.assertTrue(fa.has_edge_on_symbol(s1, '1', s2), 'Transition s1->s2 over 1 was deleted')
        self.assertFalse(fa.has_edge_on_symbol(s2, '0', s0), 'Transition s2->s0 over 1 not removed by delete')

    def test_process(self):
        fa = self.fa
        s0 = fa.new_state(initial=True, final=True)
        s1 = fa.new_state()

        fa.new_edge(s0, '0', s0)
        fa.new_edge(s0, '1', s1)
        fa.new_edge(s1, '0', s1)
        fa.new_edge(s1, '1', s0)

        vectors = {
            'accepting': [
                '0',
                '00',
                '000',
                '0110',
                '01010',
                '001100',
                '0011',
                '11000000',
                '11',
                '1111',
                '111111'],
            'rejecting': [
                '1',
                '01',
                '001',
                '10',
                '100',
                '111',
                '11111',
                '1111111',
            ]
        }

        for v in vectors['accepting']:
            self.assertTrue(fa.test_input(v), 'String "{}" was not accepted as it should'.format(v))

        for v in vectors['rejecting']:
            self.assertFalse(fa.test_input(v), 'String "{}" was not rejected as it should'.format(v))
Exemplo n.º 15
0
 def setPattern(self, pattern):
     self.pattern = pattern
     self.stateMachine = NFA(pattern)
Exemplo n.º 16
0
#!/usr/bin/env python

# our code
from dfa import DFA
from nfa import NFA
from compiler import Compiled

# the python implementation
import fnmatch

# run unit self tests
from characterset import test_CharacterSet
test_CharacterSet()
from dfa import test_distinctArcs
test_distinctArcs()

PATTERNS = ('*.txt', '*', '*.*', 'README.*')
PATHS = ('test.c', 'README.txt', 'README')

for pattern in PATTERNS:
  nfa = NFA.fnmatch(pattern)
  dfa = DFA(nfa)
  compiled = Compiled(dfa, debug=False)
  compiled.optimize()
  for path in PATHS:
    expected = fnmatch.fnmatch(path, pattern)
    assert nfa(path) == expected
    assert dfa(path) == expected
    assert compiled(path) == expected

 def Or(self, t):
   b = self.pop()
   a = self.pop()
   ab = NFA.build_or(a, b)
   self.push(ab)
 def test_zom(self):
   a = NFA.build_a('a')
   azom = NFA.build_zom(a)
 
   self.assert_(azom.initial)
   self.assert_(azom.finals)
 def ZOM(self, Ns):
   a = self.pop()
   azom = NFA.build_zom(a)
   self.push(azom)
 def Cat(self, t):
   b = self.pop()
   a = self.pop()
   ab = NFA.build_cat(a, b)
   self.push(ab)
Exemplo n.º 21
0
 def __init__(self):
     NFA.__init__(self)
Exemplo n.º 22
0
if __name__ == '__main__':
  from optparse import OptionParser
  op = OptionParser(usage='usage: %prog [options] pattern')
  op.add_option('--dont-optimize', dest='optimize', 
      help='optimize the generated code', 
      action='store_false', default=True)
  op.add_option('--native', dest='native',
      help='generate native assembly rather than LLVM assembly',
      action='store_true', default=False)
      
  (options, args) = op.parse_args()
  if len(args) != 1:
    op.print_help()
  else:
    from nfa import NFA
    from dfa import DFA
    
    dfa = DFA(NFA.fnmatch(args[0]))
    compiled = Compiled(dfa)
    if options.optimize:
      compiled.optimize()
    if options.native:
      from subprocess import Popen, PIPE
      asm = Popen('llvm-as | llc', shell=True, stdin=PIPE, stdout=PIPE)
      asm.stdin.write(str(compiled))
      asm.stdin.close()
      print asm.stdout.read()
    else:
      print compiled