def test_next_while_no_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) r1 = lf.next_while(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current( ) == expected, "lf.current() not yielding the same result"
def test_ignore_partial(): """ Test using ignore when the buffer isn't fully consumed => the unconsumed parts of the buffer has to survive """ lf = lexer.Lexer(StringIO(PANGRAM)) print("pos_0:", lf.pos) lf.next(8) pos_8 = lf.pos print("pos_8:", pos_8) lf.next(3) assert lf.current() == "the quick b", "ABORT: error with next()/current()" pos_11 = lf.pos print("pos_11:", pos_11) lf.rewind(3) print("pos11 + rewind(3):", lf.pos) assert lf.current() == "the quic", "ABORT: error with rewind/current()" print("LAST IGNORE") lf.ignore() print("post-ignore pos:", lf.pos) assert lf.current() == "", "ignore should discard all consumed characters" assert lf.buf.startswith("k b"), ( "lexer buffer should start with remaining unconsumed characters\n" "\tunconsumed: 'k b'\n" "\tactual: {}".format(lf.buf[:+3] + "...")) assert lf.pos == pos_8, "stream position number incorrectly inferred"
def test_peek_until_idempotent(): lf = lexer.Lexer(StringIO("hello {{world}}")) r1 = lf.peek_until("w") assert r1 == "hello {{", "sanity-check - peek_until is broken" assert lf.current() == "", "lf.current() should yield ''" r2 = lf.peek_until("w") assert r2 == "hello {{", "should match nothing more the second time" assert lf.current() == "", "lf.current() should yield ''"
def test_expect_next_multiple(offset, results): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) lf.next(offset) # We rely on this call NOT raising an exception as sufficient testing # (expect should not raise an exception if the expectation is met) for ndx, expected in enumerate(results): print("{}: expecting '{}'".format(ndx, expected)) lf.expect_next(expected)
def test_expect_peek_idempotent(offset, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) lf.next(offset) # We rely on this call NOT raising an exception as sufficient testing # (expect should not raise an exception if the expectation is met) lf.expect_peek(expected) lf.expect_peek(expected) # called again, should be the same
def test_peek_start(txt, peek_siz, exp_res): """ Test peeking from the start of a string """ lf = lexer.Lexer(StringIO(txt)) res = lf.peek(peek_siz) assert res == exp_res, f"peek(n={peek_siz}) error, exp: '{exp_res}', got: {res}" assert not lf.eof, f"peek(n={peek_siz}) EOF value error, exp: '{exp_res}', got: '{lf.eof}'"
def test_peek_next_mix(description, commands): lf = lexer.Lexer(StringIO(PANGRAM)) for ndx, command in enumerate(commands): op, n, exp_res = command assert op in ['p', 'n'], "test error - unknown command (p/n)" op = lf.peek if op == 'p' else lf.next act_res = op(n) assert act_res == exp_res, "{}: {}(n={}) failed, expected '{}'\n\tCase: {}".format( ndx, "peek" if op == lf.peek else "next", n, exp_res, description)
def test_next_while_idempotent(): lf = lexer.Lexer(StringIO("hello {{world}}")) r1 = lf.next_while(ALPHABET_EN + " ") assert r1 == "hello ", "sanity-check - next_while is broken" assert lf.current() == "hello ", "lf.current() should yield 'hello '" r2 = lf.next_while(ALPHABET_EN + " ") assert r2 == "", "should match nothing more the second time" assert lf.current() == "hello ", "lf.current() should still yield 'hello '"
def test_next_until_partially_consumed_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) # ensure we start with: # (1) a line in buffer # (2) an non-zero offset within that buffer assert lf.next(6) == "hello ", "error with next() call" r1 = lf.next_until(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current( ) == "hello " + expected, "lf.current() not returning the aggregate result read"
def test_rewind_too_far(): lf = lexer.Lexer(StringIO(PANGRAM)) with pytest.raises(RuntimeError): lf.rewind(1) lf.peek(1) with pytest.raises(RuntimeError): lf.rewind(1) lf.next(5) with pytest.raises(RuntimeError): lf.rewind(6)
def test_next_until_idempotent(): lf = lexer.Lexer(StringIO("hello {{world}}")) r1 = lf.next_until("w") assert r1 == "hello {{", "sanity-check - next_until is broken" assert lf.current( ) == "hello {{", "lf.current() should return same result as read" r2 = lf.next_until("w") assert r2 == "", "should match nothing more the second time" assert lf.current( ) == "hello {{", "lf.current() should return 'hello {{' still"
def test_peek_while_partially_consumed_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) # ensure we start with: # (1) a line in buffer # (2) an non-zero offset within that buffer assert lf.next(6) == "hello ", "error with next() call" old_current = lf.current() r1 = lf.peek_while(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current() == old_current, "lf.current() not yielding same result"
def test_peek_until_partially_consumed_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) # ensure we start with: # (1) a line in buffer # (2) an non-zero offset within that buffer assert lf.next(6) == "hello ", "error with next() call" old_current = lf.current() r1 = lf.peek_until(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current( ) == old_current, "lf.current() should not change because of peeking"
def test_peek_while_idempotent(): lf = lexer.Lexer(StringIO("hello {{world}}")) old_current = lf.current() r1 = lf.peek_while(ALPHABET_EN + " ") assert r1 == "hello ", "sanity-check - peek_while is broken" assert lf.current() == old_current, "lf.current() should not have changed" r2 = lf.peek_while(ALPHABET_EN + " ") assert r2 == "hello ", "should match same string the second time" assert lf.current() == old_current, "lf.current() should not have changed"
def test_pos_peek(): lf = lexer.Lexer(StringIO(PANGRAM)) assert lf.pos == 0, "initial position must be 0" lf.peek(3) assert lf.pos == 0, "peek() should not alter position" lf.next(2) assert lf.pos == 2, "next() should alter position" lf.next(len(PANGRAM)) assert lf.pos == len(PANGRAM), "position should advance to EOF"
def test_peek_while(txt, alphabet, exp): lf = lexer.Lexer(StringIO(txt)) pos_start = lf.pos cur_start = lf.buf_cursor res = lf.peek_while(alphabet) assert res == exp, "given alphabet '{}', expected result: '{}'".format( alphabet, exp) assert pos_start == lf.pos, "position in buffer shouldn't change" assert cur_start == lf.buf_cursor, "cursor offset shouldn't change"
def test_ignore_all(): """ Test using ignore when the buffer cursor is at the end => whole buffer is to be ignore()'ed """ lf = lexer.Lexer(StringIO(PANGRAM)) lf.next(4) assert lf.current() == "the ", "error in current()/next(), abort" lf.ignore() assert lf.current() == "", "ignore should skip all 'consumed' characters" res = lf.next(3) assert res == "qui", "error in next()/ignore"
def test_next1(): """ Test reading string in pieces of 1. Successive calls to next() to return the successive character in the string. """ msg = "hello, world" lf = lexer.Lexer(StringIO(msg)) for ndx, val in enumerate(list(msg)): actual = lf.next() assert actual == val, "failed @ {} - exp: {}, got: {}".format( ndx, val, actual)
def next_chunks(txt, readsiz, reads): """ Read multiple chunks of text, also in chunks where less than n chars come back (EOF) """ lf = lexer.Lexer(StringIO(txt)) for ndx, exp in enumerate(reads): act = lf.next(readsiz) assert act == exp, "read[{}] failed, exp: '{}', got: '{}'".format( ndx, exp, act) res = lf.next() assert res == "", "expected EOF, got a value" assert lf.eof, "expected EOF to be true"
def test_emit(): lf = lexer.Lexer(StringIO(PROG)) res = lf.next(3) assert res == "def", "ABORT: next()/test error" tok = lf.emit("KW") assert tok == lexer.Token(type="KW", literal="def", startpos=0) lf.next(1) lf.ignore() lf.next(3) tok = lf.emit("fn") assert tok == lexer.Token(type="fn", literal="foo", startpos=4)
def test_rewind(): lf = lexer.Lexer(StringIO(PANGRAM)) read = lf.next(9) assert read == "the quick", "error in next() abort test" assert lf.current() == "the quick", "error in current(), abort test" lf.rewind(3) assert lf.current() == "the qu", "[1] rewind(3) failed" lf.rewind(2) assert lf.current() == "the ", "[2] rewind(2) failed" lf.next(4) lf.rewind(1) assert lf.current( ) == "the qui", "[3] failed where we re-consumed and rewound characters"
def test_current(): lf = lexer.Lexer(StringIO(PANGRAM)) assert lf.current() == "", "initially, current should be an empty string" lf.peek(3) assert lf.current() == "", "peek should not affect current" lf.next(3) assert lf.current( ) == "the", "next() should move character(s) into current" lf.next(2) assert lf.current( ) == "the q", "next(2) should've added 2 characters to current" lf.peek(3) assert lf.current( ) == "the q", "peek should continue to not affect current" lf.next(len(PANGRAM)) assert lf.current( ) == PANGRAM, "reading past EOF should produce the entire contents"
def test_peek_until_no_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) r1 = lf.peek_until(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current() == "", "lf.current() should be empty"
def test_next_until_seq(seq, expected): lf = lexer.Lexer( StringIO("hello {{world}}\n{{#names}}{{> user}}{{/names}}")) print("-new-test-") res = lf.next_until_seq(seq) assert res == expected, "did not get expected string back"
def test_expect_peek_err(offset, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) lf.next(offset) with pytest.raises(lexer.LexerExpectError): lf.expect_peek(expected)
def test_expect_next_ok(offset, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) lf.next(offset) # We rely on this call NOT raising an exception as sufficient testing # (expect should not raise an exception if the expectation is met) lf.expect_next(expected)
def test_peek_until_eof(alphabet, expected): lf = lexer.Lexer(StringIO("1111\n1212\nx")) assert lf.peek_until( alphabet) == expected, "test failure or peek_until broken" assert not lf.eof, "peek_until should not set EOF"
def test_next_until_no_buffer(alphabet, expected): lf = lexer.Lexer(StringIO("hello {{world}}\nhello {{back}}!")) r1 = lf.next_until(alphabet) assert r1 == expected, "did not get expected string back" assert lf.current( ) == expected, "lf.current() should be the same as what's just read"
def test_next_until_eof(alphabet, expected, exp_eof): lf = lexer.Lexer(StringIO("1111\n1212\nx")) assert lf.next_until( alphabet) == expected, "test failure or next_until broken" assert lf.eof == exp_eof, "next_until does not set EOF as appropriate"