def test_iterators():
i = cpppo.chaining()
j = cpppo.chainable( i )
k = cpppo.chainable( 'abc' )
assert i is j
try:
next( i )
assert False, "stream with no iterable should raise StopIteration"
except StopIteration:
pass
assert k is not j
assert isinstance( k, cpppo.chaining )
assert cpppo.peekable( i ) is i
p = cpppo.peekable( '123' )
assert cpppo.peekable( p ) is p
assert cpppo.chainable( p ) is not p
assert list( p ) == ['1', '2', '3']
assert p.sent == 3
p.push('x')
assert p.sent == 2
assert list( p ) == ['x']
assert list( p ) == []
i.chain('abc')
i.chain('')
i.chain( '123' )
assert list( i ) == ['a','b','c','1','2','3']
assert i.sent == 6
i.chain( 'y' )
i.push( 'x' )
assert list( i ) == ['x','y']
i.chain( None )
try:
next( i )
assert False, "Expected TypeError to be raised"
except TypeError:
pass
except Exception as e:
assert False, "Expected TypeError, not %r" % ( e )
r = cpppo.rememberable( '123' )
assert next( r ) == '1'
assert r.memory == [ '1' ]
try:
r.push( 'x' )
assert False, "Should have rejected push of inconsistent symbol"
except AssertionError:
pass
assert r.sent == 1
r.push( '1' )
assert r.sent == 0
assert r.memory == []
assert list( r ) == r.memory == [ '1', '2','3' ]
def test_iterators():
i = cpppo.chaining()
j = cpppo.chainable(i)
k = cpppo.chainable("abc")
assert i is j
try:
next(i)
assert False, "stream with no iterable should raise StopIteration"
except StopIteration:
pass
assert k is not j
assert isinstance(k, cpppo.chaining)
assert cpppo.peekable(i) is i
p = cpppo.peekable("123")
assert cpppo.peekable(p) is p
assert cpppo.chainable(p) is not p
assert list(p) == ["1", "2", "3"]
assert p.sent == 3
p.push("x")
assert p.sent == 2
assert list(p) == ["x"]
assert list(p) == []
i.chain("abc")
i.chain("")
i.chain("123")
assert list(i) == ["a", "b", "c", "1", "2", "3"]
assert i.sent == 6
i.chain("y")
i.push("x")
assert list(i) == ["x", "y"]
i.chain(None)
try:
next(i)
assert False, "Expected TypeError to be raised"
except TypeError:
pass
except Exception as e:
assert False, "Expected TypeError, not %r" % (e)
r = cpppo.rememberable("123")
assert next(r) == "1"
assert r.memory == ["1"]
try:
r.push("x")
assert False, "Should have rejected push of inconsistent symbol"
except AssertionError:
pass
assert r.sent == 1
r.push("1")
assert r.sent == 0
assert r.memory == []
assert list(r) == r.memory == ["1", "2", "3"]
def echo_server( conn, addr ):
"""Serve one echo client 'til EOF; then close the socket"""
source = cpppo.chainable()
with echo_machine( "echo_%s" % addr[1] ) as echo_line:
eof = False
while not eof:
data = cpppo.dotdict()
# See if a line has been recognized, stopping at terminal state. If this machine
# is ended early due to an EOF, it should still terminate in a terminal state
for mch, sta in echo_line.run( source=source, data=data ):
if sta is not None:
continue
# Non-transition; check for input, blocking if non-terminal and none left. On
# EOF, terminate early; this will raise a GeneratorExit.
timeout = 0 if echo_line.terminal or source.peek() is not None else None
msg = network.recv( conn, timeout=timeout )
if msg is not None:
eof = not len( msg )
log.info( "%s recv: %5d: %s", echo_line.name_centered(), len( msg ),
"EOF" if eof else cpppo.reprlib.repr( msg ))
source.chain( msg )
if eof:
break
# Terminal state (or EOF).
log.detail( "%s: byte %5d: data: %r", echo_line.name_centered(), source.sent, data )
if echo_line.terminal:
conn.send( data.echo )
log.info( "%s done", echo_line.name_centered() )
def test_tnet_machinery():
# parsing integers
path = "machinery"
data = cpppo.dotdict()
source = cpppo.chainable(b'123:')
with cpppo.integer_bytes(name="SIZE", context="size",
terminal=True) as SIZE:
with contextlib.closing(SIZE.run(source=source, data=data,
path=path)) as engine:
for m, s in engine:
if s is None:
break
log.info("After SIZE: %r", data)
assert SIZE.terminal
assert data.machinery.size == 123
# repeat, limited by parent context's 'value' in data
source.chain(b"abc" * 123)
with tnet.data_parser(name="DATA", context="data",
repeat="..size") as DATA:
with contextlib.closing(DATA.run(source=source, data=data,
path=path)) as engine:
for m, s in engine:
if s is None:
break
log.info("After DATA: %r", data)
def tnet_server( conn, addr ):
"""Serve one tnet client 'til EOF; then close the socket"""
source = cpppo.chainable()
with tnet_machine( "tnet_%s" % addr[1] ) as tnet_mesg:
eof = False
while not eof:
data = cpppo.dotdict()
# Loop blocking for input, while we've consumed input from source since the last time.
# If we hit this again without having used any input, we know we've hit a symbol
# unacceptable to the state machine; stop
for mch, sta in tnet_mesg.run( source=source, data=data ):
if sta is not None:
continue
# Non-transition; check for input, blocking if non-terminal and none left. On
# EOF, terminate early; this will raise a GeneratorExit.
timeout = 0 if tnet_mesg.terminal or source.peek() is not None else None
msg = network.recv( conn, timeout=timeout ) # blocking
if msg is not None:
eof = not len( msg )
log.info( "%s: recv: %5d: %s", tnet_mesg.name_centered(), len( msg ),
"EOF" if eof else cpppo.reprlib.repr( msg ))
source.chain( msg )
if eof:
break
# Terminal state (or EOF).
log.detail( "%s: byte %5d: data: %r", tnet_mesg.name_centered(), source.sent, data )
if tnet_mesg.terminal:
res = json.dumps( data.tnet.type.input, indent=4, sort_keys=True )
conn.send(( res + "\n\n" ).encode( "utf-8" ))
log.info( "%s done", tnet_mesg.name_centered() )
def tnet_server( conn, addr ):
"""Serve one tnet client 'til EOF; then close the socket"""
source = cpppo.chainable()
with tnet_machine( "tnet_%s" % addr[1] ) as tnet_mesg:
eof = False
while not eof:
data = cpppo.dotdict()
# Loop blocking for input, while we've consumed input from source since the last time.
# If we hit this again without having used any input, we know we've hit a symbol
# unacceptable to the state machine; stop
for mch, sta in tnet_mesg.run( source=source, data=data ):
if sta is not None:
continue
# Non-transition; check for input, blocking if non-terminal and none left. On
# EOF, terminate early; this will raise a GeneratorExit.
timeout = 0 if tnet_mesg.terminal or source.peek() is not None else None
msg = network.recv( conn, timeout=timeout ) # blocking
if msg is not None:
eof = not len( msg )
log.info( "%s: recv: %5d: %s", tnet_mesg.name_centered(), len( msg ),
"EOF" if eof else reprlib.repr( msg ))
source.chain( msg )
if eof:
break
# Terminal state (or EOF).
log.detail( "%s: byte %5d: data: %r", tnet_mesg.name_centered(), source.sent, data )
if tnet_mesg.terminal:
res = json.dumps( data.tnet.type.input, indent=4, sort_keys=True )
conn.send(( res + "\n\n" ).encode( "utf-8" ))
log.info( "%s done", tnet_mesg.name_centered() )
def echo_server(conn, addr):
"""Serve one echo client 'til EOF; then close the socket"""
source = cpppo.chainable()
with echo_machine("echo_%s" % addr[1]) as echo_line:
eof = False
while not eof:
data = cpppo.dotdict()
# See if a line has been recognized, stopping at terminal state. If this machine
# is ended early due to an EOF, it should still terminate in a terminal state
for mch, sta in echo_line.run(source=source, data=data):
if sta is not None:
continue
# Non-transition; check for input, blocking if non-terminal and none left. On
# EOF, terminate early; this will raise a GeneratorExit.
timeout = 0 if echo_line.terminal or source.peek(
) is not None else None
msg = network.recv(conn, timeout=timeout)
if msg is not None:
eof = not len(msg)
log.info("%s recv: %5d: %s", echo_line.name_centered(),
len(msg),
"EOF" if eof else cpppo.reprlib.repr(msg))
source.chain(msg)
if eof:
break
# Terminal state (or EOF).
log.detail("%s: byte %5d: data: %r", echo_line.name_centered(),
source.sent, data)
if echo_line.terminal:
conn.send(data.echo)
log.info("%s done", echo_line.name_centered())
def test_struct():
dtp = cpppo.type_bytes_array_symbol
abt = cpppo.type_bytes_iter
ctx = 'val'
a = cpppo.state_input( "First", alphabet=abt, typecode=dtp, context=ctx )
a[True] = b = cpppo.state_input( "Second", alphabet=abt, typecode=dtp, context=ctx )
b[True] = c = cpppo.state_input( "Third", alphabet=abt, typecode=dtp, context=ctx )
c[True] = d = cpppo.state_input( "Fourth", alphabet=abt, typecode=dtp, context=ctx )
d[None] = cpppo.state_struct( "int32", context=ctx,
format=str("<i"),
terminal=True )
machine = cpppo.dfa( initial=a )
with machine:
material = b'\x01\x02\x03\x80\x99'
segment = 3
source = cpppo.chainable()
log.info( "States; %r input, by %d", material, segment )
inp = None
data = cpppo.dotdict()
path = "struct"
sequence = machine.run( source=source, path=path, data=data )
for num in range( 10 ):
try:
mch,sta = next( sequence )
inp = source.peek()
except StopIteration:
inp = source.peek()
log.info( "%s <- %-10.10r test done", cpppo.centeraxis( mch, 25, clip=True ), inp )
break
log.info( "%s <- %-10.10r test rcvd", cpppo.centeraxis( mch, 25, clip=True ), inp )
if sta is None:
log.info( "%s <- %-10.10r test no next state", cpppo.centeraxis( mch, 25, clip=True ), inp )
if inp is None:
if not material:
log.info( "%s <- %-10.10r test source finished", cpppo.centeraxis( mch, 25, clip=True ), inp )
# Will load consecutive empty iterables; chainable must handle
source.chain( material[:segment] )
material = material[segment:]
inp = source.peek()
log.info( "%s <- %-10.10r test chain", cpppo.centeraxis( mch, 25, clip=True ), inp )
if num == 0: assert inp == b'\x01'[0]; assert sta.name == "First"
if num == 1: assert inp == b'\x02'[0]; assert sta.name == "Second"
if num == 2: assert inp == b'\x03'[0]; assert sta.name == "Third"
if num == 3: assert inp == b'\x80'[0]; assert sta is None
if num == 4: assert inp == b'\x80'[0]; assert sta.name == "Fourth"
if num == 5: assert inp == b'\x99'[0]; assert sta.name == "int32"
if num == 6: assert inp == b'\x99'[0]; assert sta.name == "int32"
assert inp == b'\x99'[0]
assert num == 6
assert sta.name == "int32"
assert data.struct.val == -2147286527
def __init__(self, host, port=None, timeout=None):
"""Connect to the EtherNet/IP client, waiting """
self.addr = (host or address[0], port or address[1])
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(self.addr)
self.session = None
self.source = cpppo.chainable()
self.data = None
# Parsers
self.engine = None # EtherNet/IP frame parsing in progress
self.frame = enip.enip_machine(terminal=True)
self.cip = enip.CIP(terminal=True) # Parses a CIP request in an EtherNet/IP frame
self.lgx = logix.Logix().parser # Parses a Logix request in an EtherNet/IP CIP request
def __init__(self, host, port=None, timeout=None):
"""Connect to the EtherNet/IP client, waiting """
self.addr = (host or address[0], port or address[1])
self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn.connect(self.addr)
self.session = None
self.source = cpppo.chainable()
self.data = None
# Parsers
self.engine = None # EtherNet/IP frame parsing in progress
self.frame = enip.enip_machine(terminal=True)
self.cip = enip.CIP(
terminal=True) # Parses a CIP request in an EtherNet/IP frame
self.lgx = logix.Logix(
).parser # Parses a Logix request in an EtherNet/IP CIP request
def test_regex_demo():
regex = str( '(ab+)((,[ ]*)(ab+))*' )
machine = cpppo.regex( name=str( 'demo' ), initial=regex )
data = cpppo.dotdict()
with machine:
source = cpppo.chainable( str( 'abbb, abb, ab' ))
for i,(m,s) in enumerate( machine.run( source=source, data=data )):
log.info( "%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r", m.name_centered(),
i, s, source.sent, source.peek(), data )
assert i == 14
assert source.sent == 13
regexstr, lego, machine, initial = cpppo.state_input.from_regex(
regex, alphabet=cpppo.type_str_iter, encoder=None,
typecode=cpppo.type_str_array_symbol, context=None )
assert str( lego ) == "ab+(, *ab+)*"
assert str( machine ) == """\
def test_tnet_machinery():
# parsing integers
path = "machinery"
SIZE = cpppo.integer_bytes(name="SIZE", context="size")
data = cpppo.dotdict()
source = cpppo.chainable(b'123:')
with SIZE:
for m, s in SIZE.run(source=source, data=data, path=path):
if s is None:
break
log.info("After SIZE: %r", data)
assert s and s.terminal
assert data.machinery.size == 123
# repeat, limited by parent context's 'value' in data
DATA = tnet.data_parser(name="DATA", context="data", repeat="..size")
source.chain(b"abc" * 123)
with DATA:
for m, s in DATA.run(source=source, data=data, path=path):
if s is None:
break
log.info("After DATA: %r", data)
def test_tnet_machinery():
# parsing integers
path = "machinery"
SIZE = cpppo.integer_bytes( name="SIZE", context="size", terminal=True )
data = cpppo.dotdict()
source = cpppo.chainable( b'123:' )
with SIZE:
for m,s in SIZE.run( source=source, data=data, path=path ):
if s is None:
break
log.info( "After SIZE: %r", data )
assert SIZE.terminal
assert data.machinery.size == 123
# repeat, limited by parent context's 'value' in data
DATA = tnet.data_parser(
name="DATA", context="data", repeat="..size" )
source.chain( b"abc" * 123 )
with DATA:
for m,s in DATA.run( source=source, data=data, path=path ):
if s is None:
break
log.info( "After DATA: %r", data )
def test_regex():
# This forces plain strings in 2.x, unicode in 3.x (counteracts import unicode_literals above)
regex = str('a*b.*x')
machine = cpppo.regex(name=str('test1'), initial=regex)
with machine:
source = cpppo.chainable(str('aaab1230xoxx'))
sequence = machine.run(source=source)
for num in range(20):
try:
mch, sta = next(sequence)
inp = source.peek()
except StopIteration:
inp = source.peek()
log.info("%s <- %-10.10r test done",
cpppo.centeraxis(mch, 25, clip=True), inp)
break
log.info("%s <- %-10.10r test rcvd",
cpppo.centeraxis(mch, 25, clip=True), inp)
if sta is None:
log.info("%s <- %-10.10r test no next state",
cpppo.centeraxis(mch, 25, clip=True), inp)
if inp is None:
log.info("%s <- %-10.10r test source finished",
cpppo.centeraxis(mch, 25, clip=True), inp)
# Initial state does *not* consume a source symbol
if num == 0:
assert inp == 'a'
assert sta.name == "0'"
assert source.sent == 0
if num == 1:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 0
if num == 2:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 1
if num == 3:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 2
if num == 4:
assert inp == 'b'
assert sta.name == "2"
if num == 5:
assert inp == '1'
assert sta.name == "2"
if num == 6:
assert inp == '2'
assert sta.name == "2"
if num == 7:
assert inp == '3'
assert sta.name == "2"
if num == 8:
assert inp == '0'
assert sta.name == "2"
if num == 9:
assert inp == 'x'
assert sta.name == "3"
if num == 10:
assert inp == 'o'
assert sta.name == "2" # Trans. from term. to non-term. state!))
if num == 11:
assert inp == 'x'
assert sta.name == "3"
if num == 12:
assert inp == 'x'
assert sta.name == "3"
if num == 13:
assert inp == None
assert sta is None
assert num < 14
assert inp is None
assert num == 14
assert sta is None and machine.current.name == '3'
regex = str('.*')
machine = cpppo.regex(name=str('dot'), initial=regex, terminal=True)
data = cpppo.dotdict()
with machine:
source = cpppo.chainable(str('aaab1230xoxx\0'))
try:
for i, (m, s) in enumerate(machine.run(source=source, data=data)):
log.info("%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
m.name_centered(), i, s, source.sent, source.peek(),
data)
except cpppo.NonTerminal:
pass
assert machine.terminal
assert i == 14
assert source.sent == 13
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230xoxx\x00'
else:
assert data.input.input.tounicode() == 'aaab1230xoxx\x00'
regex = str('[^xyz]*')
machine = cpppo.regex(name=str('not_xyz'), initial=regex)
data = cpppo.dotdict()
with machine:
source = cpppo.chainable(str('aaab1230xoxx\0'))
try:
for i, (m, s) in enumerate(machine.run(source=source, data=data)):
log.info("%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
m.name_centered(), i, s, source.sent, source.peek(),
data)
except cpppo.NonTerminal:
pass
assert not machine.terminal
assert i == 9
assert source.sent == 8
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230'
else:
assert data.input.input.tounicode() == 'aaab1230'
regex = str('[^\x00]*')
machine = cpppo.regex(name=str('not_NUL'), initial=regex)
data = cpppo.dotdict()
with machine:
source = cpppo.chainable(str('aaab1230xoxx\0'))
for i, (m, s) in enumerate(machine.run(source=source, data=data)):
log.info("%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
m.name_centered(), i, s, source.sent, source.peek(), data)
assert i == 13
assert source.sent == 12
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230xoxx'
else:
assert data.input.input.tounicode() == 'aaab1230xoxx'
def test_dfa():
# Simple DFA with states consuming no input. A NULL (None) state transition
# doesn't require input for state change. The Default (True) transition
# requires input to make the transition, but none of these states consume
# it, so it'll be left over at the end.
a = cpppo.state("Initial")
a[None] = b = cpppo.state("Middle")
b[True] = cpppo.state("Terminal", terminal=True)
source = cpppo.chainable()
i = a.run(source=source)
m, s = next(i)
assert m is None
assert s is not None and s.name == "Middle"
try:
next(i)
assert False, "Expected no more non-transition events"
except StopIteration:
pass
machine = cpppo.dfa(initial=a)
with machine:
log.info("DFA:")
for initial in machine.initial.nodes():
for inp, target in initial.edges():
log.info(
"%s <- %-10.10r -> %s" %
(cpppo.centeraxis(initial, 25, clip=True), inp, target))
# Running with no input will yield the initial state, with None input; since it is a NULL
# state (no input processed), it will simply attempt to transition. This will require the
# next input from source, which is empty, so it will return input,state=(None, None)
# indicating a non-terminal state and no input left. This gives the caller an opportunity
# to reload input and try again. If a loop is detected (same state and input conditions
# seen repeatedly), the DFA will terminate; if not in a terminal state, an exception will be
# raised.
log.info("States; No input")
source = cpppo.chainable()
sequence = machine.run(source=source)
for num in range(10):
try:
mch, sta = next(sequence)
except StopIteration:
sequence = None
break
except cpppo.NonTerminal as e:
assert "non-terminal state" in str(e)
break
inp = source.peek()
log.info("%s <- %r" % (cpppo.centeraxis(mch, 25, clip=True), inp))
if num == 0:
assert inp is None
assert sta.name == "Initial"
if num == 1:
assert inp is None
assert sta.name == "Middle"
if num == 2:
assert inp is None
assert sta is None # And no more no-input transitions
assert num < 3 # If we get here, we didn't detect loop
assert num == 3
# since the iterator did not stop cleanly (after processing a state's input,
# and then trying to determine the next state), it'll continue indefinitely
assert sta is None
assert sequence is not None
# Try with some input loaded into source stream, using an identical generator sequence.
# Only the first element is gotten, and is reused for every NULL state transition, and is
# left over at the end.
log.info("States; 'abc' input")
assert source.peek() is None
source.chain(b'abc')
assert source.peek() == b'a'[0] # python2: str, python3: int
sequence = machine.run(source=source)
for num in range(10):
try:
mch, sta = next(sequence)
except StopIteration:
break
inp = source.peek()
log.info("%s <- %r", cpppo.centeraxis(mch, 25, clip=True), inp)
if num == 0:
assert inp == b'a'[0]
assert sta.name == "Initial"
if num == 1:
assert inp == b'a'[0]
assert sta.name == "Middle"
if num == 2:
assert inp == b'a'[0]
assert sta.name == "Terminal"
assert num < 3
assert num == 3
assert inp == b'a'[0]
assert sta.name == "Terminal"
def test_codecs():
# In Python3, the greenery.fsm is able to handle the Unicode str type; under
# Python2, it can sanely only handle the non-Unicode str type.
if sys.version_info[0] < 3:
return
# Test parsing of greenery.fsm/lego regexes specified in Unicode. Then,
# generate corresponding cpppo state machines that accept Unicode input
# symbols, and byte input symbols. These tests will accept as much of the
# input as matches the regular expression.
texts = [
'pi: π',
'abcdé\u4500123',
'This contains π,π and more πs',
'a 480Ω resistor',
]
tests = [
('[^π]*(π[^π]*)+', True), # Optional non-π's, followed by at least one string of π and non-π's
('[^π]*[^π]', False) # Any number of non-π, ending in a non-π
]
for text in texts:
for re,tr in tests:
# First, convert the unicode regex to a state machine in unicode symbols. Only if both
# the dfa and its sub-state are "terminal", will it be terminal.
with cpppo.regex(
name='pies', context="pies", initial=re, terminal=True ) as pies:
original = text
source = cpppo.chainable( original )
data = cpppo.dotdict()
try:
for mch, sta in pies.run( source=source, data=data ):
pass
except cpppo.NonTerminal:
pass
accepted = pies.terminal and data.pies.input.tounicode() == original
log.info( "%s ends w/ re %s: %s: %r", pies.name_centered(), re,
"string accepted" if accepted else "string rejected", data )
# Each of these are greedy, and so run 'til the end of input (next state is None); they
# collect the full input string, unless they run into a non-matching input.
expected = tr == ('π' in text )
assert accepted == expected
for text in texts:
# Then convert the unicode regex to a state machine in bytes symbols.
# Our encoder generates 1 or more bytes for each unicode symbol.
for re,tr in tests:
original = text.encode( 'utf-8' ) # u'...' --> b'...'
source = cpppo.chainable( original )
data = cpppo.dotdict()
with cpppo.regex(
name='pies', context="pies", initial=re, terminal=True,
regex_alphabet=int,
regex_typecode='B',
regex_encoder=lambda s: ( b for b in s.encode( 'utf-8' ))) as pies:
try:
for mch, sta in pies.run( source=source, data=data ):
pass
except cpppo.NonTerminal:
pass
accepted = pies.terminal and data.pies.input.tobytes() == original
log.detail( "%s ends w/ re: %s: %s: %r", pies.name_centered(), re,
"string accepted" if accepted else "string rejected", data )
expected = tr == ('π' in text )
assert accepted == expected
assert original.startswith( data.pies.input.tobytes() )
def test_regex():
# This forces plain strings in 2.x, unicode in 3.x (counteracts import unicode_literals above)
regex = str('a*b.*x')
machine = cpppo.regex( name=str('test1'), initial=regex )
with machine:
source = cpppo.chainable( str('aaab1230xoxx') )
sequence = machine.run( source=source )
for num in range( 20 ):
try:
mch,sta = next( sequence )
inp = source.peek()
except StopIteration:
inp = source.peek()
log.info( "%s <- %-10.10r test done", cpppo.centeraxis( mch, 25, clip=True ), inp )
break
log.info( "%s <- %-10.10r test rcvd", cpppo.centeraxis( mch, 25, clip=True ), inp )
if sta is None:
log.info( "%s <- %-10.10r test no next state", cpppo.centeraxis( mch, 25, clip=True ), inp )
if inp is None:
log.info( "%s <- %-10.10r test source finished", cpppo.centeraxis( mch, 25, clip=True ), inp )
# Initial state does *not* consume a source symbol
if num == 0: assert inp == 'a'; assert sta.name == "0'"; assert source.sent == 0
if num == 1: assert inp == 'a'; assert sta.name == "0"; assert source.sent == 0
if num == 2: assert inp == 'a'; assert sta.name == "0"; assert source.sent == 1
if num == 3: assert inp == 'a'; assert sta.name == "0"; assert source.sent == 2
if num == 4: assert inp == 'b'; assert sta.name == "2"
if num == 5: assert inp == '1'; assert sta.name == "2"
if num == 6: assert inp == '2'; assert sta.name == "2"
if num == 7: assert inp == '3'; assert sta.name == "2"
if num == 8: assert inp == '0'; assert sta.name == "2"
if num == 9: assert inp == 'x'; assert sta.name == "3"
if num ==10: assert inp == 'o'; assert sta.name == "2" # Trans. from term. to non-term. state!))
if num ==11: assert inp == 'x'; assert sta.name == "3"
if num ==12: assert inp == 'x'; assert sta.name == "3"
if num ==13: assert inp ==None; assert sta is None
assert num < 14
assert inp is None
assert num == 14
assert sta is None and machine.current.name == '3'
regex = str('.*')
machine = cpppo.regex( name=str('dot'), initial=regex, terminal=True )
data = cpppo.dotdict()
with machine:
source = cpppo.chainable( str('aaab1230xoxx\0') )
try:
for i,(m,s) in enumerate( machine.run( source=source, data=data )):
log.info( "%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r", m.name_centered(),
i, s, source.sent, source.peek(), data )
except cpppo.NonTerminal:
pass
assert machine.terminal
assert i == 14
assert source.sent == 13
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230xoxx\x00'
else:
assert data.input.input.tounicode() == 'aaab1230xoxx\x00'
regex = str('[^xyz]*')
machine = cpppo.regex( name=str('not_xyz'), initial=regex )
data = cpppo.dotdict()
with machine:
source = cpppo.chainable( str('aaab1230xoxx\0') )
try:
for i,(m,s) in enumerate( machine.run( source=source, data=data )):
log.info( "%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r", m.name_centered(),
i, s, source.sent, source.peek(), data )
except cpppo.NonTerminal:
pass
assert not machine.terminal
assert i == 9
assert source.sent == 8
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230'
else:
assert data.input.input.tounicode() == 'aaab1230'
regex = str('[^\x00]*')
machine = cpppo.regex( name=str('not_NUL'), initial=regex )
data = cpppo.dotdict()
with machine:
source = cpppo.chainable( str('aaab1230xoxx\0') )
for i,(m,s) in enumerate( machine.run( source=source, data=data )):
log.info( "%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r", m.name_centered(),
i, s, source.sent, source.peek(), data )
assert i == 13
assert source.sent == 12
if sys.version_info[0] < 3:
assert data.input.input.tostring() == 'aaab1230xoxx'
else:
assert data.input.input.tounicode() == 'aaab1230xoxx'
def test_codecs():
# In Python3, the greenery.fsm is able to handle the Unicode str type; under
# Python2, it can sanely only handle the non-Unicode str type.
if sys.version_info.major < 3:
return
# Test parsing of greenery.fsm/lego regexes specified in Unicode. Then,
# generate corresponding cpppo state machines that accept Unicode input
# symbols, and byte input symbols.
texts = ["pi: π", "abcdé\u4500123", "This contains π,π and more πs", "a 480Ω resistor"]
for text in texts:
# First, convert the unicode regex to a state machine in unicode symbols.
with cpppo.regex(name="pies", context="pies", initial=".*π.*", terminal=True) as pies:
source = cpppo.chainable(text)
data = cpppo.dotdict()
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
log.info(
"%s ends: %s: %r", pies.name_centered(), "string accepted" if pies.terminal else "string rejected", data
)
# Each of these are greedy, and so run 'til the end of input (next state
# is None); they collect the full input string.
assert pies.terminal == ("π" in text)
assert data.pies.input.tounicode() == text
for text in texts:
# Then convert the unicode regex to a state machine in bytes symbols.
# Our encoder generates 1 or more bytes for each unicode symbol.
pies = cpppo.regex(
name="pies",
context="pies",
initial=".*π.*",
terminal=True,
regex_alphabet=int,
regex_typecode="B",
regex_encoder=lambda s: (b for b in s.encode("utf-8")),
)
source = cpppo.chainable(text.encode("utf-8"))
data = cpppo.dotdict()
with pies:
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
log.info(
"%s ends: %s: %r", pies.name_centered(), "string accepted" if pies.terminal else "string rejected", data
)
assert pies.terminal == ("π" in text)
assert data.pies.input.tobytes().decode("utf-8") == text
def test_codecs():
# In Python3, the greenery.fsm is able to handle the Unicode str type; under
# Python2, it can sanely only handle the non-Unicode str type.
if sys.version_info[0] < 3:
return
# Test parsing of greenery.fsm/lego regexes specified in Unicode. Then,
# generate corresponding cpppo state machines that accept Unicode input
# symbols, and byte input symbols. These tests will accept as much of the
# input as matches the regular expression.
texts = [
'pi: π',
'abcdé\u4500123',
'This contains π,π and more πs',
'a 480Ω resistor',
]
tests = [
(
'[^π]*(π[^π]*)+', True
), # Optional non-π's, followed by at least one string of π and non-π's
('[^π]*[^π]', False) # Any number of non-π, ending in a non-π
]
for text in texts:
for re, tr in tests:
# First, convert the unicode regex to a state machine in unicode symbols. Only if both
# the dfa and its sub-state are "terminal", will it be terminal.
with cpppo.regex(name='pies',
context="pies",
initial=re,
terminal=True) as pies:
original = text
source = cpppo.chainable(original)
data = cpppo.dotdict()
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
accepted = pies.terminal and data.pies.input.tounicode(
) == original
log.info("%s ends w/ re %s: %s: %r", pies.name_centered(), re,
"string accepted" if accepted else "string rejected",
data)
# Each of these are greedy, and so run 'til the end of input (next state is None); they
# collect the full input string, unless they run into a non-matching input.
expected = tr == ('π' in text)
assert accepted == expected
for text in texts:
# Then convert the unicode regex to a state machine in bytes symbols.
# Our encoder generates 1 or more bytes for each unicode symbol.
for re, tr in tests:
original = text.encode('utf-8') # u'...' --> b'...'
source = cpppo.chainable(original)
data = cpppo.dotdict()
with cpppo.regex(name='pies',
context="pies",
initial=re,
terminal=True,
regex_alphabet=int,
regex_typecode='B',
regex_encoder=lambda s:
(b for b in s.encode('utf-8'))) as pies:
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
accepted = pies.terminal and data.pies.input.tobytes(
) == original
log.detail(
"%s ends w/ re: %s: %s: %r", pies.name_centered(), re,
"string accepted" if accepted else "string rejected", data)
expected = tr == ('π' in text)
assert accepted == expected
assert original.startswith(data.pies.input.tobytes())
def tnet_from( conn, addr,
server = cpppo.dotdict({'done': False}),
timeout = None,
latency = None,
ignore = None,
source = None ): # Provide a cpppo.chainable, if desire, to receive into and parse from
"""Parse and yield TNET messages from a socket w/in timeout, blocking 'til server.done or EOF
between messages. If ignore contains symbols, they are ignored between TNET messages (eg. b'\n').
An absense of a TNET string within 'timeout' will yield None, allowing the user to decide to
fail or continue trying for another 'timeout' period. A 0 timeout will "poll", and a None
timeout will simply wait forever (the default). If desired, a separate 'latency' can be
supplied, in order to pop out regularly and check server.done (eg. to allow a server Thread to
exit cleanly).
"""
if source is None:
source = cpppo.chainable()
with tnet_machine( "tnet_%s" % addr[1] ) as engine:
eof = False
while not ( eof or server.done ):
while ignore and source.peek() and source.peek() in ignore:
next( source )
data = cpppo.dotdict()
started = cpppo.timer() # When did we start the current attempt at a TNET string?
for mch,sta in engine.run( source=source, data=data ):
if sta is not None or source.peek() is not None:
continue
# Non-transition state, and we need more data: check for more data, enforce timeout.
# Waits up to latency, or remainder of timeout -- or forever, if both are None.
duration = cpppo.timer() - started
msg = None
while msg is None and not server.done: # Get input, forever or 'til server.done
remains = latency if timeout is None else min( # If no timeout, wait for latency (or forever, if None)
timeout if latency is None else latency, # Or, we know timeout is numeric; get min of any latency
max( timeout - duration, 0 )) # ... and remaining unused timeout
log.info( "%s: After %7.3fs, awaiting symbols (after %d processed) w/ %s recv timeout",
engine.name_centered(), duration, source.sent,
remains if remains is None else ( "%7.3fs" % remains ))
msg = network.recv( conn, timeout=remains )
duration = cpppo.timer() - started
if msg is None and timeout is not None and duration >= timeout:
# No data w/in given timeout expiry! Inform the consumer, and then try again w/ fresh timeout.
log.info( "%s: After %7.3fs, no TNET message after %7.3fs recv timeout",
engine.name_centered(), duration, remains )
yield None
started = cpppo.timer()
# Only way to get here without EOF/data, is w/ server.done
if server.done:
break
assert msg is not None
# Got EOF or data
eof = len( msg ) == 0
log.info( "%s: After %7.3fs, recv: %5d: %s",
engine.name_centered(), duration, len( msg ),
'EOF' if eof else cpppo.reprlib.repr( msg ))
if eof:
break
source.chain( msg )
# Terminal state, or EOF, or server.done. Only yield another TNET message if terminal.
duration = cpppo.timer() - started
if engine.terminal:
log.debug( "%s: After %7.3fs, found a TNET: %r", engine.name_centered(), duration, data.tnet.type.input )
yield data.tnet.type.input # Could be a 0:~ / null ==> None
log.detail( "%s: done w/ %s", engine.name_centered(),
', '.join( ['EOF'] if eof else [] + ['done'] if server.done else [] ))
def test_codecs():
# In Python3, the greenery.fsm is able to handle the Unicode str type; under
# Python2, it can sanely only handle the non-Unicode str type.
if sys.version_info.major < 3:
return
# Test parsing of greenery.fsm/lego regexes specified in Unicode. Then,
# generate corresponding cpppo state machines that accept Unicode input
# symbols, and byte input symbols.
texts = [
'pi: π',
'abcdé\u4500123',
'This contains π,π and more πs',
'a 480Ω resistor',
]
for text in texts:
# First, convert the unicode regex to a state machine in unicode symbols.
with cpppo.regex(name='pies',
context="pies",
initial='.*π.*',
terminal=True) as pies:
source = cpppo.chainable(text)
data = cpppo.dotdict()
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
log.info("%s ends: %s: %r", pies.name_centered(),
"string accepted" if pies.terminal else "string rejected",
data)
# Each of these are greedy, and so run 'til the end of input (next state
# is None); they collect the full input string.
assert pies.terminal == ('π' in text)
assert data.pies.input.tounicode() == text
for text in texts:
# Then convert the unicode regex to a state machine in bytes symbols.
# Our encoder generates 1 or more bytes for each unicode symbol.
pies = cpppo.regex(name='pies',
context="pies",
initial='.*π.*',
terminal=True,
regex_alphabet=int,
regex_typecode='B',
regex_encoder=lambda s:
(b for b in s.encode('utf-8')))
source = cpppo.chainable(text.encode('utf-8'))
data = cpppo.dotdict()
with pies:
try:
for mch, sta in pies.run(source=source, data=data):
pass
except cpppo.NonTerminal:
pass
log.info("%s ends: %s: %r", pies.name_centered(),
"string accepted" if pies.terminal else "string rejected",
data)
assert pies.terminal == ('π' in text)
assert data.pies.input.tobytes().decode('utf-8') == text
def test_regex():
# This forces plain strings in 2.x, unicode in 3.x (counteracts import unicode_literals above)
regex = str('a*b.*x')
machine = cpppo.regex(name=str('test1'), initial=regex)
with machine:
source = cpppo.chainable(str('aaab1230xoxx'))
sequence = machine.run(source=source)
for num in range(20):
try:
mch, sta = next(sequence)
inp = source.peek()
except StopIteration:
inp = source.peek()
log.info("%s <- %-10.10r test done",
cpppo.centeraxis(mch, 25, clip=True), inp)
break
log.info("%s <- %-10.10r test rcvd",
cpppo.centeraxis(mch, 25, clip=True), inp)
if sta is None:
log.info("%s <- %-10.10r test no next state",
cpppo.centeraxis(mch, 25, clip=True), inp)
if inp is None:
log.info("%s <- %-10.10r test source finished",
cpppo.centeraxis(mch, 25, clip=True), inp)
# Initial state does *not* consume a source symbol
if num == 0:
assert inp == 'a'
assert sta.name == "0'"
assert source.sent == 0
if num == 1:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 0
if num == 2:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 1
if num == 3:
assert inp == 'a'
assert sta.name == "0"
assert source.sent == 2
if num == 4:
assert inp == 'b'
assert sta.name == "2"
if num == 5:
assert inp == '1'
assert sta.name == "2"
if num == 6:
assert inp == '2'
assert sta.name == "2"
if num == 7:
assert inp == '3'
assert sta.name == "2"
if num == 8:
assert inp == '0'
assert sta.name == "2"
if num == 9:
assert inp == 'x'
assert sta.name == "3"
if num == 10:
assert inp == 'o'
assert sta.name == "2" # Trans. from term. to non-term. state!))
if num == 11:
assert inp == 'x'
assert sta.name == "3"
if num == 12:
assert inp == 'x'
assert sta.name == "3"
if num == 13:
assert inp == None
assert sta is None
assert num < 14
assert inp is None
assert num == 14
assert sta is None and machine.current.name == '3'
def test_CIP_HART(repeat=1):
"""HART protocol enip CIP messages
"""
enip.lookup_reset() # Flush out any existing CIP Objects for a fresh start
ENIP = enip.enip_machine(context='enip')
CIP = enip.CIP()
# We'll use a HART Message Router, to handle its expanded porfolio of commands
MR = HART(instance_id=1)
for pkt, tst in client.recycle(CIP_HART_tests, times=repeat):
# Parse just the CIP portion following the EtherNet/IP encapsulation header
data = cpppo.dotdict()
source = cpppo.chainable(pkt)
with ENIP as machine:
for i, (m, s) in enumerate(machine.run(source=source, data=data)):
log.detail("%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
machine.name_centered(), i, s, source.sent,
source.peek(), data)
# In a real protocol implementation, an empty header (EOF with no input at all) is
# acceptable; it indicates a session closed by the client.
if not data:
log.normal("EtherNet/IP Request: Empty (session terminated): %s",
enip.enip_format(data))
continue
if log.isEnabledFor(logging.NORMAL):
log.normal("EtherNet/IP Request: %s", enip.enip_format(data))
# Parse the encapsulated .input
with CIP as machine:
for i, (m, s) in enumerate(
machine.run(path='enip',
source=cpppo.peekable(
data.enip.get('input', b'')),
data=data)):
log.detail("%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
machine.name_centered(), i, s, source.sent,
source.peek(), data)
if log.isEnabledFor(logging.NORMAL):
log.normal("EtherNet/IP CIP Request: %s", enip.enip_format(data))
# Assume the request in the CIP's CPF items are HART requests.
# Now, parse the encapsulated message(s). We'll assume it is destined for a HART Object.
if 'enip.CIP.send_data' in data:
for item in data.enip.CIP.send_data.CPF.item:
if 'unconnected_send.request' in item:
# An Unconnected Send that contained an encapsulated request (ie. not just a Get
# Attribute All)
with MR.parser as machine:
if log.isEnabledFor(logging.NORMAL):
log.normal(
"Parsing %3d bytes using %s.parser, from %s",
len(item.unconnected_send.request.input), MR,
enip.enip_format(item))
# Parse the unconnected_send.request.input octets, putting parsed items into the
# same request context
for i, (m, s) in enumerate(
machine.run(
source=cpppo.peekable(
item.unconnected_send.request.input),
data=item.unconnected_send.request)):
log.detail(
"%s #%3d -> %10.10s; next byte %3d: %-10.10r: %r",
machine.name_centered(), i, s, source.sent,
source.peek(), data)
# Post-processing of some parsed items is only performed after lock released!
if log.isEnabledFor(logging.NORMAL):
log.normal(
"Parsed %3d bytes using %s.parser, into %s",
len(item.unconnected_send.request.input), MR,
enip.enip_format(data))
try:
for k, v in tst.items():
assert data[k] == v, ("data[%r] == %r\n"
"expected: %r" % (k, data[k], v))
except:
log.warning("%r not in data, or != %r: %s", k, v,
enip.enip_format(data))
raise
# Ensure that we can get the original EtherNet/IP CIP back
for k in list(data.keys()):
if k.endswith('input') and 'sender_context' not in k:
log.detail("del data[%r]", k)
del data[k]
try:
# First reconstruct any SendRRData CPF items, containing encapsulated requests/responses
if 'enip.CIP.send_data' in data:
cpf = data.enip.CIP.send_data
for item in cpf.CPF.item:
if 'unconnected_send' in item:
item.unconnected_send.request.input = bytearray(
MR.produce(item.unconnected_send.request))
log.normal("Produce HART message from: %r",
item.unconnected_send.request)
# Next, reconstruct the CIP Register, ListIdentity, ListServices, or SendRRData. The CIP.produce must
# be provided the EtherNet/IP header, because it contains data (such as .command)
# relevant to interpreting the .CIP... contents.
data.enip.input = bytearray(enip.CIP.produce(data.enip))
# And finally the EtherNet/IP encapsulation itself
data.input = bytearray(enip.enip_encode(data.enip))
log.detail("EtherNet/IP CIP Request produced payload: %r",
bytes(data.input))
assert data.input == pkt, "original:\n" + hexdump(
pkt) + "\nproduced:\n" + hexdump(data.input)
except Exception as exc:
log.warning(
"Exception %s; Invalid packet produced from EtherNet/IP CIP data: %s",
exc, enip.enip_format(data))
raise
def test_dfa():
# Simple DFA with states consuming no input. A NULL (None) state transition
# doesn't require input for state change. The Default (True) transition
# requires input to make the transition, but none of these states consume
# it, so it'll be left over at the end.
a = cpppo.state( "Initial" )
a[None] = b = cpppo.state( "Middle" )
b[True] = cpppo.state( "Terminal", terminal=True )
source = cpppo.chainable()
i = a.run( source=source )
m,s = next( i )
assert m is None
assert s is not None and s.name == "Middle"
try:
next( i )
assert False, "Expected no more non-transition events"
except StopIteration:
pass
machine = cpppo.dfa( initial=a )
with machine:
log.info( "DFA:" )
for initial in machine.initial.nodes():
for inp,target in initial.edges():
log.info( "%s <- %-10.10r -> %s" % ( cpppo.centeraxis( initial, 25, clip=True ),
inp, target ))
# Running with no input will yield the initial state, with None input; since it is a NULL
# state (no input processed), it will simply attempt to transition. This will require the
# next input from source, which is empty, so it will return input,state=(None, None)
# indicating a non-terminal state and no input left. This gives the caller an opportunity
# to reload input and try again. If a loop is detected (same state and input conditions
# seen repeatedly), the DFA will terminate; if not in a terminal state, an exception will be
# raised.
log.info( "States; No input" )
source = cpppo.chainable()
sequence = machine.run( source=source )
for num in range( 10 ):
try:
mch,sta = next( sequence )
except StopIteration:
sequence = None
break
except cpppo.NonTerminal as e:
assert "non-terminal state" in str( e )
break
inp = source.peek()
log.info( "%s <- %r" % ( cpppo.centeraxis( mch, 25, clip=True ), inp ))
if num == 0: assert inp is None; assert sta.name == "Initial"
if num == 1: assert inp is None; assert sta.name == "Middle"
if num == 2: assert inp is None; assert sta is None # And no more no-input transitions
assert num < 3 # If we get here, we didn't detect loop
assert num == 3
# since the iterator did not stop cleanly (after processing a state's input,
# and then trying to determine the next state), it'll continue indefinitely
assert sta is None
assert sequence is not None
# Try with some input loaded into source stream, using an identical generator sequence.
# Only the first element is gotten, and is reused for every NULL state transition, and is
# left over at the end.
log.info( "States; 'abc' input" )
assert source.peek() is None
source.chain( b'abc' )
assert source.peek() == b'a'[0] # python2: str, python3: int
sequence = machine.run( source=source )
for num in range( 10 ):
try:
mch,sta = next( sequence )
except StopIteration:
break
inp = source.peek()
log.info( "%s <- %r", cpppo.centeraxis( mch, 25, clip=True ), inp )
if num == 0: assert inp == b'a'[0]; assert sta.name == "Initial"
if num == 1: assert inp == b'a'[0]; assert sta.name == "Middle"
if num == 2: assert inp == b'a'[0]; assert sta.name == "Terminal"
assert num < 3
assert num == 3
assert inp == b'a'[0]
assert sta.name == "Terminal"
def test_regex():
# This forces plain strings in 2.x, unicode in 3.x (counteracts import unicode_literals above)
regex = str("a*b.*x")
machine = cpppo.regex(name=str("test1"), initial=regex)
with machine:
source = cpppo.chainable(str("aaab1230xoxx"))
sequence = machine.run(source=source)
for num in range(20):
try:
mch, sta = next(sequence)
inp = source.peek()
except StopIteration:
inp = source.peek()
log.info("%s <- %-10.10r test done", cpppo.centeraxis(mch, 25, clip=True), inp)
break
log.info("%s <- %-10.10r test rcvd", cpppo.centeraxis(mch, 25, clip=True), inp)
if sta is None:
log.info("%s <- %-10.10r test no next state", cpppo.centeraxis(mch, 25, clip=True), inp)
if inp is None:
log.info("%s <- %-10.10r test source finished", cpppo.centeraxis(mch, 25, clip=True), inp)
# Initial state does *not* consume a source symbol
if num == 0:
assert inp == "a"
assert sta.name == "0'"
assert source.sent == 0
if num == 1:
assert inp == "a"
assert sta.name == "0"
assert source.sent == 0
if num == 2:
assert inp == "a"
assert sta.name == "0"
assert source.sent == 1
if num == 3:
assert inp == "a"
assert sta.name == "0"
assert source.sent == 2
if num == 4:
assert inp == "b"
assert sta.name == "2"
if num == 5:
assert inp == "1"
assert sta.name == "2"
if num == 6:
assert inp == "2"
assert sta.name == "2"
if num == 7:
assert inp == "3"
assert sta.name == "2"
if num == 8:
assert inp == "0"
assert sta.name == "2"
if num == 9:
assert inp == "x"
assert sta.name == "3"
if num == 10:
assert inp == "o"
assert sta.name == "2" # Trans. from term. to non-term. state!))
if num == 11:
assert inp == "x"
assert sta.name == "3"
if num == 12:
assert inp == "x"
assert sta.name == "3"
if num == 13:
assert inp == None
assert sta is None
assert num < 14
assert inp is None
assert num == 14
assert sta is None and machine.current.name == "3"