def store_word(self, db, title, text, url):
url_index = db.store_url(url, title)
if text != 'None' :
seggen = jieba.cut_for_search(title+text)
seglist = list(seggen)
for word in seglist:
db.store_word(word, url_index)
db.commit()
class TestDB(unittest.TestCase):
def setUp(self):
self.db = DB(directory=TEST_WRITE_ENV)
def tearDown(self):
if hasattr(self.db, 'env'):
try:
self.db.env.close()
except Exception as e:
print('Error for tearDown close', e)
if os.path.isdir(TEST_WRITE_ENV):
try:
shutil.rmtree(TEST_WRITE_ENV)
except PermissionError:
print('PermissionError whilst deleting test database- Using db.wipe(True)...')
time.sleep(1)
self.db.wipe(True)
shutil.rmtree(TEST_WRITE_ENV)
self.assertFalse(os.path.exists(TEST_WRITE_ENV), 'tearDown did not Delete Test Directory.')
def test_open(self):
"""Open a new DB, asserting the new directory data"""
self.db.open('test')
self.assertEqual(self.db.env.path(), TEST_WRITE_ENV)
self.assertTrue(os.path.exists(self.db.env.path()))
@patch('lmdb.open')
def test_make_directory(self, lmdb_open):
"""Open a new DB, asseting the correct library methods are used"""
self.db.open('test')
args = self.db._open_args(map_size=MAX_BYTES)
lmdb_open.assert_called_with(TEST_WRITE_ENV, **args)
def test_destroy(self):
db = self.db
db.open('test_wipe')
db.put('doggy', (1,2,3))
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.close()
db.open('test_wipe')
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.wipe()
db.open('test_wipe')
self.assertEqual(db.get('doggy'), None)
db.put('doggy', (1,2,3))
db.close()
db.open('test_wipe')
self.assertTupleEqual(db.get('doggy'), (1,2,3))
db.start = Mock(side_effect=db.start)
db.wipe(True)
db.start.assert_called()
self.assertEqual(db.get('doggy'), None)
def test_put_raise_error(self):
"""Add a key value and assert its existence"""
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
success = self.db.put(key, value)
self.assertTrue(success)
result = self.db.get(key)
self.assertEqual(result, value)
with self.assertRaises(TypeError):
self.db.put('apple', 'green', encode_key=False)
def test_put_without_save(self):
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
self.db._transaction_success = Mock()
success = self.db.put(key, value, save=False)
self.assertTrue(success)
self.assertEqual(self.db._transaction_success.call_count, 0)
def test_decode_with_decode_func(self):
class Foo:
def __str__(self):
return 'cake'
self.assertEqual(self.db.decode(Foo()), 'cake')
class Two:
def decode(self, encode):
return 1
def __str__(self):
return 'cake'
self.assertEqual(self.db.decode(Foo()), 'cake')
self.assertEqual(self.db.decode(Two()), 1)
self.assertEqual(self.db.decode(b'convert string'), 'convert string')
def test_translate(self):
key = b'window'
self.assertEqual(self.db.translate(key), 'window')
self.assertEqual(self.db.translate(b'!:str!:dog'), [b'str', b'dog'])
def test_delete(self):
"""Add a key with put(), then delete. More deletes should yield false."""
# Use put for an input.
self.db.open('test_put')
key, value = b'my_test_key', b'my test value'
self.db.put(key, value)
success = self.db.delete(key)
self.assertTrue(success)
success = self.db.delete(key)
# not thing to delete the second time
self.assertFalse(success)
success = self.db.delete(key)
self.assertFalse(success)
def test_keys(self):
return_value = [1,2,3]
self.db.iter = Mock(return_value=return_value)
result = self.db.keys(cake=True)
self.db.iter.assert_called_with(values=False,
dups=False,
convert=False,
cake=True)
self.assertCountEqual(result, return_value)
def test_values(self):
return_value = [1,2,3]
self.db.iter = Mock(return_value=return_value)
result = self.db.values(cake=True)
self.db.iter.assert_called_with(keys=False,
cake=True)
self.assertCountEqual(result, return_value)
def test_count(self):
db = self.db
db.open('test_count')
db.put('a', 'b')
db.put('c', 'c')
db.put('c', 'd')
self.assertEqual(db.count('a'), 1)
self.assertEqual(db.count('c'), 2)
def test_delete(self):
db = self.db
db.open('test_delete')
db.wipe()
db.put('a', 'b')
db.put('a', 'e')
db.put('b', 'c')
db.put('c', 'd')
db.put('c', 'e')
db.put('d', 'f')
keys = tuple(db.iter())
expected = (
('a', 'b'),
('a', 'e'),
('b', 'c'),
('c', 'd'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
result = db.delete('a')
self.assertTrue(result)
keys = tuple(db.iter())
expected = (
('b', 'c'),
('c', 'd'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
result = db.delete('c', value='d')
keys = tuple(db.iter())
expected = (
('b', 'c'),
('c', 'e'),
('d', 'f')
)
self.assertTupleEqual(keys, expected)
def test_get_last(self):
db = self.db
db.open('test_delete')
db.wipe()
db.put('a', 'b')
db.put('a', 'e')
db.put('b', 'c')
db.put('c', 'd')
db.put('c', 'e')
self.assertEqual(db.get('a'), 'b')
self.assertEqual(db.get('a', last=True), 'e')
self.assertEqual(db.get('c'), 'd')
self.assertEqual(db.get('c', last=True), 'e')
self.assertEqual(db.get('b'), 'c')
self.assertEqual(db.get('b', last=True), 'c')
self.assertNotEqual(db.get('b', last=True), 'a')
self.assertNotEqual(db.get('b'), 'a')
def test_pop(self):
self.db._cursor = Mock()
self.db.pop('apple')
self.db._cursor.pop.assert_called_with(b'apple')
def test_multiple_db(self):
db = self.db
db = DB(directory=TEST_WRITE_ENV)
db.open('test_multiple_1')
db.wipe()
# Has only 1
db.put('cake', 'moo')
self.assertEqual(len(tuple(db.iter())), 1)
db2 = DB(directory=TEST_WRITE_ENV)
db2.open('test_multiple_2')
db2.wipe()
# only wipe the seperate db.
self.assertEqual(len(tuple(db.iter())), 1)
self.assertEqual(len(tuple(db2.iter())), 0)
db.put('dog', 'blue')
db2.put('cat', 'blue')
# Did not wipe the first key from the first db
self.assertCountEqual(tuple(db.iter()), (('dog', 'blue',), ('cake', 'moo') ))
self.assertCountEqual(tuple(db2.iter()), (('cat', 'blue',), ))
@patch('lmdb.open')
def test_create_transaction_using(self, lmdb_open):
item = Mock()
begin_mock = Mock()
# self.db.open = Mock()
self.db.env = begin_mock
self.db._cursor = 'foo'
kw = dict(using=item, write=False)
self.db.create_transaction(**kw)
begin_mock.begin.assert_called_with(db=item, write=False)
self.assertEqual(self.db._cursor, None)
def test_commit(self):
"""put() a key and commit
"""
# First we assert the death of non persistent put() delete
self.db.open('test_commit')
key, value = b'commit_key', b'my test value'
success = self.db.put(key, value, save=True)
self.assertTrue(success)
success = self.db.delete(key, commit=False)
self.assertTrue(success)
result = self.db.get(key)
self.assertEqual(result, None)
# close and open-
self.db.close()
self.setUp()
self.db.open('test_commit')
# Should exist due to persistent write
result = self.db.get(key)
self.assertEqual(result, value)
# Now delete.
success = self.db.delete(key, commit=False)
self.assertTrue(success)
# store the change
self.db.commit()
# Delete and remake -
self.tearDown()
self.setUp()
self.db.open('test_commit')
# Should not exist due to commit()
result = self.db.get(key)
self.assertEqual(result, None)
def test_in_out_conversion(self):
self.db.open('test_commit')
result = self.db.put('apples', '[1,24,54,6,56]')
self.assertTrue(result)
result = self.db.get('apples')
self.db.put('apples', [1,24,54,6,56])
result = self.db.get('apples')
self.assertEqual(result, [1,24,54,6,56])
self.assertIsInstance(result, list)
def test_tuple_store(self):
self.db.open('test_commit')
self.db.put('nums', (1,2,3,4,5,6,7,8,))
self.assertIsInstance(self.db.get('nums'), tuple)
def test_list_store(self):
self.db.open('test_commit')
self.db.put('list', [1,2,3,4,5])
res = self.db.get('list')
self.assertIsInstance(self.db.get('list'), list)
def test_collect(self):
self.db.open('test_collect')
self.db.delete('collect')
self.db.put('collect', (1,2,3,4,5,))
self.db.put('collect', (5,4,3,2,1,))
result = self.db.collect('collect')
self.assertEqual(len(list(result)), 2)
def test_dict_store(self):
self.db.open('test_commit')
de = {'foo': 1, 'bar': 'two', 'three': True}
self.db.put('dict', de)
res = self.db.get('dict')
expected = {'foo': 1, 'bar': 'two', 'three': True}
self.assertDictEqual(res, expected)
def test___repr__(self):
db = DB(directory=TEST_WRITE_ENV, name='dave')
s= '<database.db.DB "dave">'
self.assertEqual(str(db), s)