def test_properties(self):
parser = LevelParser()
self.assertEqual(parser.levels, 1)
self.assertEqual(parser.parsers, [Parser()])
with self.assertRaises(ValueError):
parser.levels = 0
with self.assertRaises(ValueError):
parser.levels = -1
self.assertEqual(parser.levels, 1)
parser.levels = 2
self.assertEqual(parser.parsers, [Parser(), Parser()])
level = Parser(state_sizes=[2, 3])
parser.parsers = level
self.assertEqual(parser.parsers, [level, level])
parser.parsers = [Parser(), level, Parser()]
self.assertEqual(parser.parsers, [Parser(), level])
parser.levels = 1
self.assertEqual(parser.parsers, [Parser()])
parser.levels = 2
self.assertIs(parser.parsers[1], level)
def test_level_parser_properties():
parser = LevelParser()
assert parser.levels == 1
assert parser.parsers == [Parser()]
with pytest.raises(ValueError):
parser.levels = 0
with pytest.raises(ValueError):
parser.levels = -1
assert parser.levels == 1
parser.levels = 2
assert parser.parsers == [Parser(), Parser()]
level = Parser(state_sizes=[2, 3])
parser.parsers = level
assert parser.parsers == [level, level]
parser.parsers = [Parser(), level, Parser()]
assert parser.parsers == [Parser(), level]
parser.levels = 1
assert parser.parsers == [Parser()]
parser.levels = 2
assert parser.parsers[1] is level
def test_generate_state_size(self):
m = self.Markov(separator=':',
parser=Parser(state_sizes=[2, 3]),
scanner=Scanner(lambda x: x))
m.data(['x', 'y', 'z'])
self.assertEqual(''.join(m.generate(10, state_size=2)), 'xyz')
self.assertEqual(''.join(m.generate(10, state_size=3)), 'xyz')
def test_properties_parse(self):
scanner = Scanner(lambda x: x)
parser = Parser(state_sizes=[2],
reset_on_sentence_end=False)
self.assertEqual(self.parse(parser, scanner, ['a', scanner.END, 'b'],
True, '::'),
[('::', 'a'), ('::a', 'b')])
self.assertEqual(self.parse(parser, scanner, 'c', separator='::'),
[('a::b', 'c')])
def test_parser_parse(mocker):
state_size_dataset = mocker.patch('markovchain.parser.state_size_dataset',
return_value='0')
parser = Parser(state_sizes=[2], reset_on_sentence_end=False)
res = parse(parser, ['a', Scanner.END, 'b'], True, '::')
assert res == [('0', '::', 'a'), ('0', '::a', None), ('0', '::a', 'b')]
res = parse(parser, 'c', separator='::')
assert res == [('0', 'a::b', 'c')]
state_size_dataset.assert_has_calls(
call(parser.state_sizes[0]) for _ in range(2))
def test_markov_image_generate(test, res):
scanner = Scanner(lambda x: x)
scanner.traversal = [HLines()]
scanner.levels = 1
scanner.level_scale = []
markov = MarkovImage(
scanner=scanner,
parser=Parser()
)
markov.imgtype.convert = lambda x: [x]
markov.data([['\x00', '\x01', '\x02', '\x03']])
assert list(markov(*test).getdata()) == res
def test_state_size(self):
scanner = Scanner(lambda x: x)
parser = Parser(state_sizes=[3])
self.assertEqual(self.parse(parser, scanner, 'abcde'),
[(' ', 'a'), (' a', 'b'), (' a b', 'c'),
('a b c', 'd'), ('b c d', 'e')])
self.assertEqual(self.parse(parser, scanner,
['a', 'b', 'c', Scanner.END, 'd', 'e']),
[(' ', 'a'), (' a', 'b'), (' a b', 'c'),
(' ', 'd'), (' d', 'e')])
self.assertEqual(self.parse(parser, scanner,
['a', 'b', 'c', (Scanner.START, 'd'), 'e']),
[(' ', 'a'), (' a', 'b'), (' a b', 'c'),
(' d', 'e')])
def test_default_parse(self):
scanner = Scanner(lambda x: x)
parser = Parser()
self.assertEqual(self.parse(parser, scanner, ''), [])
self.assertEqual(self.parse(parser, scanner, 'abc', True),
[('', 'a'), ('a', 'b'), ('b', 'c')])
self.assertEqual(self.parse(parser, scanner,
['a', 'b', scanner.END, 'c']),
[('c', 'a'), ('a', 'b'), ('', 'c')])
self.assertEqual(self.parse(parser, scanner,
['a', Scanner.END, Scanner.END, 'c']),
[('', 'a'), ('', 'c')])
self.assertEqual(self.parse(parser, scanner, [Scanner.END] * 4), [])
def test_save_load(self):
db = os.path.join(self.tmpdir.name, 'test.db')
m = self.Markov(db=db,
separator=':',
parser=Parser(state_sizes=[2, 3]),
scanner=Scanner(lambda x: x))
m.data(['x', 'y', 'z'])
m.scanner = CharScanner()
m.save()
loaded = self.Markov.load(db)
self.assertEqual(m, loaded)
self.assertEqual(''.join(loaded.generate(10, state_size=2)), 'xyz')
loaded = self.Markov.load(db, {'separator': ''})
self.assertNotEqual(m, loaded)
self.assertEqual(loaded.separator, '')
def test_generate(self):
scanner = Scanner(lambda x: x)
scanner.traversal = [HLines()]
scanner.levels = 1
scanner.level_scale = []
m = self.Markov(palette=self.palette, scanner=scanner, parser=Parser())
with self.assertRaises(RuntimeError):
m.image(2, 2)
m.data([['00', '01', '02', '03']])
data = list(m.image(2, 2).getdata())
self.assertEqual(data, [0, 1, 2, 3])
data = list(m.image(4, 2).getdata())
self.assertEqual(data, [0, 1, 2, 3, 0, 1, 2, 3])
def test_parser_parse_default(mocker):
state_size_dataset = mocker.patch('markovchain.parser.state_size_dataset',
return_value='0')
parser = Parser()
assert parse(parser, '') == []
assert parse(parser, 'abc', True) == [('0', '', 'a'), ('0', 'a', 'b'),
('0', 'b', 'c')]
assert parse(parser, ['a', 'b', Scanner.END, 'c']) == [('0', 'c', 'a'),
('0', 'a', 'b'),
('0', 'b', None),
('0', '', 'c')]
assert parse(parser,
['a', Scanner.END, Scanner.END, 'c']) == [('0', '', 'a'),
('0', 'a', None),
('0', '', 'c')]
assert parse(parser, [Scanner.END] * 4) == []
state_size_dataset.assert_has_calls(
call(parser.state_sizes[0]) for _ in range(4))
def test_properties(self):
parser = Parser()
with self.assertRaises(ValueError):
parser.state_sizes = [1, 0, 2]
with self.assertRaises(ValueError):
parser.state_sizes = []
self.assertEqual(parser.state_sizes, [1])
self.assertEqual(parser.state_size, 1)
parser.state_sizes = [2, 1]
self.assertEqual(parser.state_sizes, [2, 1])
self.assertEqual(parser.state_size, 2)
self.assertEqual(parser.state.maxlen, 2)
self.assertEqual(list(parser.state), ['', ''])
parser.state.append('test')
parser.state_size = parser.state_size
self.assertEqual(list(parser.state), ['', 'test'])
parser.state_sizes = [3]
self.assertEqual(list(parser.state), ['', '', ''])
def test_save_load(self):
m = self.Markov(separator=':',
parser=Parser(state_sizes=[2, 3]),
scanner=Scanner(lambda x: x))
m.data(['', 'x', 'y', 'z', None])
m.scanner = CharScanner()
fp = StringIO()
m.save(fp)
fp.seek(0)
loaded = self.Markov.load(fp)
self.assertEqual(m, loaded)
fp.seek(0)
fp1 = BytesIO()
fp1.write(fp.read().encode('utf-8'))
fp1.seek(0)
loaded = self.Markov.load(fp1)
self.assertEqual(m, loaded)
fp.seek(0)
loaded = self.Markov.load(fp, {'separator': ''})
self.assertNotEqual(m, loaded)
self.assertEqual(loaded.separator, '')
def test_parser_properties():
parser = Parser()
with pytest.raises(ValueError):
parser.state_sizes = [1, 0, 2]
with pytest.raises(ValueError):
parser.state_sizes = []
assert parser.state_sizes == [1]
assert parser.state_size == 1
parser.state_sizes = [2, 1]
assert parser.state_sizes == [2, 1]
assert parser.state_size == 2
assert parser.state.maxlen == 2
assert list(parser.state) == ['', '']
parser.state.append('test')
parser.state_size = parser.state_size
assert list(parser.state) == ['', 'test']
parser.state_sizes = [3]
assert list(parser.state) == ['', '', '']
def test_parser_eq(test, test2, res):
parser = Parser(*test)
parser2 = Parser(*test2)
assert (parser == parser2) == res
def test_parser_state_size(mocker, test, res):
state_size_dataset = mocker.patch('markovchain.parser.state_size_dataset',
return_value='0')
parser = Parser(state_sizes=[3])
assert parse(parser, test) == res
state_size_dataset.assert_called_once_with(3)
def test_save_load(self):
m = self.Markov(separator=':', parser=Parser())
saved = m.get_save_data()
loaded = self.Markov(**saved)
self.assertEqual(m, loaded)
def construct_model():
return MarkovText(parser=Parser([3]))
def test_level_parser_save_load():
level = Parser(state_sizes=[2, 3])
parser = LevelParser(levels=3, parsers=[level, Parser()])
saved = parser.save()
loaded = Parser.load(saved)
assert parser == loaded
def test_save_load(self):
level = Parser(state_sizes=[2, 3])
parser = LevelParser(levels=3, parsers=[level, Parser()])
saved = parser.save()
loaded = Parser.load(saved)
self.assertEqual(parser, loaded)
def test_parser_save_load(test):
parser = Parser(*test)
saved = parser.save()
loaded = Parser.load(saved)
assert parser == loaded
def test_level_parser_parse(mocker, test, res):
level_dataset = mocker.patch('markovchain.parser.level_dataset',
return_value='0')
parsers = [
Mock(wraps=ParserBase(lambda x: [0])),
Mock(wraps=ParserBase(lambda x: [1]))
]
parser = LevelParser(levels=2, parsers=parsers)
assert list(parser(test, dataset='data')) == res
if test:
assert level_dataset.call_count == 2
level_dataset.assert_has_calls([call(0), call(1)])
for parser, data in zip(parsers, test):
parser.assert_called_once_with(data, False, 'data0')
@pytest.mark.parametrize('test,test2,res',
[((), (), True), ((2, Parser()), (2, Parser()), True),
((1, ), (2, ), False),
((2, [Parser(), Parser(3)]), (2, Parser()), False)])
def test_level_parser_eq(test, test2, res):
assert (LevelParser(*test) == LevelParser(*test2)) == res
def test_level_parser_save_load():
level = Parser(state_sizes=[2, 3])
parser = LevelParser(levels=3, parsers=[level, Parser()])
saved = parser.save()
loaded = Parser.load(saved)
assert parser == loaded
def test_save_load(self):
parser = Parser(state_sizes=[1, 2, 3],
reset_on_sentence_end=False)
saved = parser.save()
loaded = Parser.load(saved)
self.assertEqual(parser, loaded)