def do_copy(self, line):
''' copy <source> <target>\tcopy data from source kvlite database to target kvlite database
<source> - reference name to source database
<target> - reference name to target database
for creating reference name, use `create` command'''
try:
source_ref, target_ref = [
param for param in line.split(' ') if param <> ''
][:2]
except ValueError:
print 'Error! Please specify <source> and <target>'
return
if source_ref not in self.__kvlite_colls:
print 'Error! The source reference is not created, please use `create` command'
return
if target_ref not in self.__kvlite_colls:
print 'Error! The target reference is not created, please use `create` command'
return
source = kvlite.open(self.__kvlite_colls[source_ref])
target = kvlite.open(self.__kvlite_colls[target_ref])
kvlite.copy(source, target)
source.close()
target.close()
def test_diff_serializers(self):
""" test for checking different serializers
create collection with one serializers and check that you can open items
without definition the serializer again
"""
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name="compressed_json")
collection.put(1, {"k1": "v1"})
collection.commit()
collection.close()
collection = kvlite.open(URI)
self.assertEqual(collection.get({"_key": 1}), ("1".zfill(kvlite.settings.KEY_LENGTH), {"k1": "v1"}))
collection.close()
def create_update_item(collection, k, v):
''' create or update item
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {
'error': 'The collection %s is not found' % name,
'status': 'NOT OK'
}
if v:
try:
collection.put(k, json.loads(v))
collection.commit()
collection.close()
return {
'status': 'OK',
}
except Exception, err:
return {
'status': 'Error',
'message': err,
}
def test_long_key(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
self.assertRaises(RuntimeError, collection.get, {"_key": "1" * 41})
self.assertRaises(RuntimeError, collection.put, "1" * 41, "long_key")
self.assertRaises(RuntimeError, collection.delete, "1" * 41)
collection.close()
def test_metadata(self):
""" test metadata
"""
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name="pickle")
self.assertEqual(
collection.meta, {"name": "kvlite_test", "kvlite-version": kvlite.__version__, "serializer": "pickle"}
)
collection.close()
def do_use(self, collection_name):
''' use <collection_name>\tuse the collection as the default (current) collection'''
if collection_name in self.__kvlite_colls:
self.prompt = '%s>' % collection_name
self.__current_coll_name = collection_name
self.__current_coll = kvlite.open(self.__kvlite_colls[self.__current_coll_name])
return
else:
print 'Error! Unknown collection: %s' % collection_name
def test_long_key(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
self.assertRaises(RuntimeError, collection.get, {'_key':'1'*41})
self.assertRaises(RuntimeError, collection.put, '1'*41, 'long_key')
self.assertRaises(RuntimeError, collection.delete, '1'*41)
collection.close()
def test_long_key(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
self.assertRaises(RuntimeError, collection.get, {'_key': '1' * 41})
self.assertRaises(RuntimeError, collection.put, '1' * 41, 'long_key')
self.assertRaises(RuntimeError, collection.delete, '1' * 41)
collection.close()
def test_sqlite_open(self):
collection = kvlite.open('sqlite://testdb.sqlite:kvlite_test')
collection.put('a',1)
self.assertEqual(collection.count,1)
self.assertEqual(collection.get('a'), ('a',1))
collection.delete('a')
self.assertEqual(collection.count,0)
self.assertEqual(collection.get('a'), (None,None))
collection.close()
def test_diff_key_length(self):
''' test for different key length
'''
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
collection.put(1, 'key-01')
self.assertEqual(collection.get({'_key': '1'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
self.assertEqual(collection.get({'_key': '01'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
self.assertEqual(collection.get({'_key': '001'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
collection.close()
def test_sqlite_open(self):
collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test')
collection.put('a',1)
self.assertEqual(collection.count,1)
self.assertEqual(collection.get('a'), ('a',1))
collection.delete('a')
self.assertEqual(collection.count,0)
self.assertEqual(collection.get('a'), (None,None))
collection.close()
def setUp(self):
self.collection = kvlite.open(URI + '.kvlite_test')
self.collection_name = 'kvlite_test'
self.manager = MysqlCollectionManager(URI)
if self.collection_name not in self.manager.collections():
self.manager.create(self.collection_name)
collection_class = self.manager.collection_class
self.collection = collection_class(self.manager, self.collection_name)
def test_diff_key_length(self):
""" test for different key length
"""
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
collection.put(1, "key-01")
self.assertEqual(collection.get({"_key": "1"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01"))
self.assertEqual(collection.get({"_key": "01"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01"))
self.assertEqual(collection.get({"_key": "001"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01"))
collection.close()
def __init__(self):
""" init
"""
if self.container:
self._container = kvlite.open(self.container)
else:
raise RuntimeError("Undefined container: %s", self.container)
self.urls_queue = list()
self._processed_urls = list()
def __init__(self):
''' init
'''
if self.container:
self._container = kvlite.open(self.container)
else:
raise RuntimeError('Undefined container: %s', self.container)
self.urls_queue = list()
self._processed_urls = list()
def do_use(self, collection_name):
''' use <collection_name>\tuse the collection as the default (current) collection'''
if collection_name in self.__kvlite_colls:
self.prompt = '%s>' % collection_name
self.__current_coll_name = collection_name
self.__current_coll = kvlite.open(
self.__kvlite_colls[self.__current_coll_name])
return
else:
print 'Error! Unknown collection: %s' % collection_name
def test_sqlite_open(self):
_key = kvlite.get_uuid(1)[0]
collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test')
collection.put(_key, 1)
self.assertEqual(collection.count, 1)
self.assertEqual(collection.get({'_key': _key}), (_key, 1))
collection.delete(_key)
self.assertEqual(collection.count, 0)
self.assertEqual(collection.get({'_key': _key}), (None, None))
collection.close()
def test_sqlite_open(self):
_key = kvlite.get_uuid(1)[0]
collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test')
collection.put(_key,1)
self.assertEqual(collection.count,1)
self.assertEqual(collection.get({'_key': _key}), (_key,1))
collection.delete(_key)
self.assertEqual(collection.count,0)
self.assertEqual(collection.get({'_key': _key}), (None,None))
collection.close()
def setUp(self):
self.collection = kvlite.open(URI + '.kvlite_test')
self.collection_name = 'kvlite_test'
self.manager = MysqlCollectionManager(URI)
if self.collection_name not in self.manager.collections():
self.manager.create(self.collection_name)
collection_class = self.manager.collection_class
self.collection = collection_class(
self.manager, self.collection_name)
def get_item(collection, k):
''' get item details
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'}
k, v = collection.get({'_key': k})
collection.close()
return { 'status': 'OK', 'item': {'key': k, 'value': v}, }
def test_metadata(self):
''' test metadata
'''
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name='pickle')
self.assertEqual(
collection.meta, {
'name': 'kvlite_test',
'kvlite-version': kvlite.__version__,
'serializer': 'pickle',
})
collection.close()
def test_put_get_delete_count_many(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
ks = [(collection.get_uuid(), "test_{}".format(i)) for i in xrange(100)]
collection.put(ks)
self.assertEqual(collection.count, 100)
for k in ks:
collection.delete(k[0])
self.assertEqual(collection.count, 0)
collection.commit()
collection.close()
def test_metadata(self):
''' test metadata
'''
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name='pickle')
self.assertEqual(
collection.meta,
{
'name': 'kvlite_test',
'kvlite-version': kvlite.__version__,
'serializer': 'pickle',
})
collection.close()
def test_copy(self):
''' test_copy
'''
COPIED_ITEMS = 1200
source_uri = self.URI.format(kvlite.utils.tmp_name())
source = kvlite.open(source_uri, serializer_name='pickle')
target_uri = self.URI.format(kvlite.utils.tmp_name())
target = kvlite.open(target_uri, serializer_name='pickle')
kv = [(k, 'value: %d' % k) for k in range(1,COPIED_ITEMS+1)]
source.put(kv)
source.commit()
kvlite.utils.copy(source, target)
self.assertEqual(target.count, COPIED_ITEMS)
source.close()
target.close()
def test_copy(self):
''' test_copy
'''
COPIED_ITEMS = 250
source_uri = self.URI.format(kvlite.utils.tmp_name())
source = kvlite.open(source_uri, serializer_name='pickle')
target_uri = self.URI.format(kvlite.utils.tmp_name())
target = kvlite.open(target_uri, serializer_name='pickle')
kv = [(k, 'value: %d' % k) for k in range(1, COPIED_ITEMS + 1)]
source.put(kv)
source.commit()
kvlite.utils.copy(source, target)
self.assertEqual(target.count, COPIED_ITEMS)
source.close()
target.close()
def do_use(self, collection_name):
""" use <collection_name>\tuse the collection as the default (current) collection"""
if self.__current_coll:
self.__current_coll.close()
self.__current_coll = None
if collection_name in self.__kvlite_colls:
self.prompt = "%s>" % collection_name
self.__current_coll_name = collection_name
self.__current_coll = kvlite.open(self.__kvlite_colls[self.__current_coll_name])
return
else:
print "Error! Unknown collection: %s" % collection_name
def test_put_get_delete_count_one(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
k = collection.get_uuid()
v = 'test_put_one'
collection.put(k, v)
self.assertEqual(collection.get({'_key': k}), (k, v))
self.assertEqual(collection.count, 1)
collection.delete(k)
self.assertEqual(collection.count, 0)
collection.commit()
collection.close()
def test_put_get_delete_count_one(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
k = collection.get_uuid()
v = "test_put_one"
collection.put(k, v)
self.assertEqual(collection.get({"_key": k}), (k, v))
self.assertEqual(collection.count, 1)
collection.delete(k)
self.assertEqual(collection.count, 0)
collection.commit()
collection.close()
def test_diff_key_length(self):
''' test for different key length
'''
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
collection.put(1, 'key-01')
self.assertEqual(collection.get({'_key': '1'}),
('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
self.assertEqual(collection.get({'_key': '01'}),
('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
self.assertEqual(collection.get({'_key': '001'}),
('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01'))
collection.close()
def test_put_get_delete_count_many(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
ks = [(collection.get_uuid(), 'test_{}'.format(i))
for i in xrange(100)]
collection.put(ks)
self.assertEqual(collection.count, 100)
for k in ks:
collection.delete(k[0])
self.assertEqual(collection.count, 0)
collection.commit()
collection.close()
def test_use_different_serializators_for_many(self):
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name="compressed_json")
collection.put(u"11", u"diffser1")
collection.put(u"22", u"diffser2")
collection.put(u"33", u"diffser3")
collection.commit()
collection.close()
# in version v0.6.3 you cannot re-define serializer in open() which was
# assigned during first collection creation
collection = kvlite.open(URI, serializer_name="pickle")
res = [(k, v) for k, v in collection]
self.assertEqual(
res,
[
("11".zfill(kvlite.settings.KEY_LENGTH), u"diffser1"),
("22".zfill(kvlite.settings.KEY_LENGTH), u"diffser2"),
("33".zfill(kvlite.settings.KEY_LENGTH), u"diffser3"),
],
)
collection.close()
def do_show(self, line):
''' show collections <details>\tlist of available collections'''
if line.startswith('collections'):
for coll in self.__kvlite_colls:
print 'Database: %s' % coll
print 'URI: %s' % self.__kvlite_colls[coll]
coll = kvlite.open(self.__kvlite_colls[coll])
for k, v in coll.meta.items():
print '%s: %s' % (k, v)
coll.close()
print
else:
print 'Unknown argument: %s' % line
def do_show(self, line):
""" show collections <details>\tlist of available collections"""
if line.startswith("collections"):
for coll in self.__kvlite_colls:
print "Database: %s" % coll
print "URI: %s" % self.__kvlite_colls[coll]
coll = kvlite.open(self.__kvlite_colls[coll])
for k, v in coll.meta.items():
print "%s: %s" % (k, v)
coll.close()
print
else:
print "Unknown argument: %s" % line
def do_copy(self, line):
""" copy <source> <target>\tcopy data from source kvlite database to target kvlite database
<source> - reference name to source database
<target> - reference name to target database
for creating reference name, use `create` command"""
try:
source_ref, target_ref = [param for param in line.split(" ") if param <> ""][:2]
except ValueError:
print "Error! Please specify <source> and <target>"
return
if source_ref not in self.__kvlite_colls:
print "Error! The source reference is not created, please use `create` command"
return
if target_ref not in self.__kvlite_colls:
print "Error! The target reference is not created, please use `create` command"
return
source = kvlite.open(self.__kvlite_colls[source_ref])
target = kvlite.open(self.__kvlite_colls[target_ref])
kvlite.copy(source, target)
source.close()
target.close()
def do_show(self, line):
''' show collections <details>\tlist of available collections'''
if line.startswith('collections'):
for coll in self.__kvlite_colls:
print 'Database: %s' % coll
print 'URI: %s' % self.__kvlite_colls[coll]
coll = kvlite.open(self.__kvlite_colls[coll])
for k,v in coll.meta.items():
print '%s: %s' % (k,v)
coll.close()
print
else:
print 'Unknown argument: %s' % line
def test_use_different_serializators_for_many(self):
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name='completed_json')
collection.put(u'11', u'diffser1')
collection.put(u'22', u'diffser2')
collection.put(u'33', u'diffser3')
collection.commit()
collection.close()
collection = kvlite.open(URI, serializer_name='pickle')
with self.assertRaises(RuntimeError):
res = [(k, v) for k, v in collection]
collection.put(u'11', u'diffser1')
collection.put(u'22', u'diffser2')
collection.put(u'33', u'diffser3')
collection.commit()
collection.close()
collection = kvlite.open(URI, serializer_name='completed_json')
with self.assertRaises(RuntimeError):
res = [(k, v) for k, v in collection]
collection.close()
def test_use_different_serializators_for_many(self):
URI = self.URI.format(kvlite.utils.tmp_name())
collection = kvlite.open(URI, serializer_name='completed_json')
collection.put(u'11', u'diffser1')
collection.put(u'22', u'diffser2')
collection.put(u'33', u'diffser3')
collection.commit()
collection.close()
collection = kvlite.open(URI, serializer_name='pickle')
with self.assertRaises(RuntimeError):
res = [(k,v) for k,v in collection]
collection.put(u'11', u'diffser1')
collection.put(u'22', u'diffser2')
collection.put(u'33', u'diffser3')
collection.commit()
collection.close()
collection = kvlite.open(URI, serializer_name='completed_json')
with self.assertRaises(RuntimeError):
res = [(k,v) for k,v in collection]
collection.close()
def delete_item(collection, k):
''' delete item by key
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'}
try:
collection.delete(k)
collection.commit()
collection.close()
return { 'status': 'OK', }
except:
return { 'status': 'Error', 'message': 'Cannot delete the item by key', }
def test_get_many_by_keys(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
kvs = [(collection.get_uuid(), "test") for _ in range(10)]
collection.put(kvs)
collection.commit()
# keys are in database
result = [kv for kv in collection.get({"_key": [kv[0] for kv in kvs[0:3]]})]
self.assertEqual(len(kvs[0:3]), len(result))
for res in result:
self.assertIn(res, kvs[0:3])
# keys are not in database
result = [kv for kv in collection.get({"_key": ["00" for kv in range(3)]})]
self.assertNotEqual(len(kvs[0:3]), len(result))
collection.close()
def test_pagination(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
PAGE_SIZE = 10
kvs = [(collection.get_uuid(), "test") for _ in range(100)]
collection.put(kvs)
# first page
result = [kv for kv in collection.get(offset=0, limit=PAGE_SIZE)]
self.assertEqual(len(result), len(kvs[0:PAGE_SIZE]))
for res in result:
self.assertIn(res, kvs[0:PAGE_SIZE])
# second page
result = [kv for kv in collection.get(offset=2 * PAGE_SIZE, limit=PAGE_SIZE)]
self.assertEqual(len(result), len(kvs[2 * PAGE_SIZE : 2 * PAGE_SIZE + PAGE_SIZE]))
for res in result:
self.assertIn(res, kvs[2 * PAGE_SIZE : 2 * PAGE_SIZE + PAGE_SIZE])
collection.close()
def get_page(name, page):
''' return collection data for specific page
'''
page = int(page)
try:
collection_uri = settings.COLLECTIONS[name]
except KeyError:
return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'}
collection = kvlite.open(collection_uri)
last_page = collection.count / settings.ITEMS_PER_PAGE
#last_page = int(ceil(collection.count / float(settings.ITEMS_PER_PAGE)))
data = [kv for kv in collection.get(offset=page * settings.ITEMS_PER_PAGE, limit=settings.ITEMS_PER_PAGE)]
collection.close()
return {
'status': 'OK',
'last_page': last_page,
'data': data,
}
def test_pagination(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
PAGE_SIZE = 10
kvs = [(collection.get_uuid(), 'test') for _ in range(100)]
collection.put(kvs)
# first page
result = [kv for kv in collection.get(offset=0, limit=PAGE_SIZE)]
self.assertEqual(len(result), len(kvs[0:PAGE_SIZE]))
for res in result:
self.assertIn(res, kvs[0:PAGE_SIZE])
# second page
result = [
kv for kv in collection.get(offset=2 * PAGE_SIZE, limit=PAGE_SIZE)
]
self.assertEqual(len(result),
len(kvs[2 * PAGE_SIZE:2 * PAGE_SIZE + PAGE_SIZE]))
for res in result:
self.assertIn(res, kvs[2 * PAGE_SIZE:2 * PAGE_SIZE + PAGE_SIZE])
collection.close()
def create_update_item(collection, k, v):
''' create or update item
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'}
if v:
try:
collection.put(k, json.loads(v))
collection.commit()
collection.close()
return { 'status': 'OK', }
except Exception, err:
return {
'status': 'Error',
'message': err,
}
def test_get_many_by_keys(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
kvs = [(collection.get_uuid(), 'test') for _ in range(10)]
collection.put(kvs)
collection.commit()
# keys are in database
result = [
kv for kv in collection.get({'_key': [kv[0] for kv in kvs[0:3]]})
]
self.assertEqual(len(kvs[0:3]), len(result))
for res in result:
self.assertIn(res, kvs[0:3])
# keys are not in database
result = [
kv for kv in collection.get({'_key': ['00' for kv in range(3)]})
]
self.assertNotEqual(len(kvs[0:3]), len(result))
collection.close()
def get_item(collection, k):
''' get item details
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {
'error': 'The collection %s is not found' % name,
'status': 'NOT OK'
}
k, v = collection.get({'_key': k})
collection.close()
return {
'status': 'OK',
'item': {
'key': k,
'value': v
},
}
def delete_item(collection, k):
''' delete item by key
'''
try:
collection_uri = settings.COLLECTIONS[collection]
collection = kvlite.open(collection_uri)
except KeyError:
return {
'error': 'The collection %s is not found' % name,
'status': 'NOT OK'
}
try:
collection.delete(k)
collection.commit()
collection.close()
return {
'status': 'OK',
}
except:
return {
'status': 'Error',
'message': 'Cannot delete the item by key',
}
def get_page(name, page):
''' return collection data for specific page
'''
page = int(page)
try:
collection_uri = settings.COLLECTIONS[name]
except KeyError:
return {
'error': 'The collection %s is not found' % name,
'status': 'NOT OK'
}
collection = kvlite.open(collection_uri)
last_page = collection.count / settings.ITEMS_PER_PAGE
#last_page = int(ceil(collection.count / float(settings.ITEMS_PER_PAGE)))
data = [
kv for kv in collection.get(offset=page * settings.ITEMS_PER_PAGE,
limit=settings.ITEMS_PER_PAGE)
]
collection.close()
return {
'status': 'OK',
'last_page': last_page,
'data': data,
}
def test_put_wrong_oldformat_kv(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
collection.put(collection.get_uuid(), 'value')
collection.close()
def test_absent_key(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
self.assertEqual(collection.get({'_key': u'a1b2c3'}), (None, None))
collection.close()
def test_get_uuid(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
uuids = [collection.get_uuid() for i in range(1000)]
self.assertEqual(len(set(uuids)), 1000)
collection.close()
def test_absent_key(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
self.assertEqual(collection.get({"_key": u"a1b2c3"}), (None, None))
collection.close()
def test_put_wrong_oldformat_kv(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
collection.put(collection.get_uuid(), "value")
collection.close()
def test_get_uuid(self):
collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name()))
uuids = [collection.get_uuid() for i in range(1000)]
self.assertEqual(len(set(uuids)), 1000)
collection.close()
