def do_copy(self, line): ''' copy <source> <target>\tcopy data from source kvlite database to target kvlite database <source> - reference name to source database <target> - reference name to target database for creating reference name, use `create` command''' try: source_ref, target_ref = [ param for param in line.split(' ') if param <> '' ][:2] except ValueError: print 'Error! Please specify <source> and <target>' return if source_ref not in self.__kvlite_colls: print 'Error! The source reference is not created, please use `create` command' return if target_ref not in self.__kvlite_colls: print 'Error! The target reference is not created, please use `create` command' return source = kvlite.open(self.__kvlite_colls[source_ref]) target = kvlite.open(self.__kvlite_colls[target_ref]) kvlite.copy(source, target) source.close() target.close()
def test_diff_serializers(self): """ test for checking different serializers create collection with one serializers and check that you can open items without definition the serializer again """ URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name="compressed_json") collection.put(1, {"k1": "v1"}) collection.commit() collection.close() collection = kvlite.open(URI) self.assertEqual(collection.get({"_key": 1}), ("1".zfill(kvlite.settings.KEY_LENGTH), {"k1": "v1"})) collection.close()
def create_update_item(collection, k, v): ''' create or update item ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return { 'error': 'The collection %s is not found' % name, 'status': 'NOT OK' } if v: try: collection.put(k, json.loads(v)) collection.commit() collection.close() return { 'status': 'OK', } except Exception, err: return { 'status': 'Error', 'message': err, }
def test_long_key(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) self.assertRaises(RuntimeError, collection.get, {"_key": "1" * 41}) self.assertRaises(RuntimeError, collection.put, "1" * 41, "long_key") self.assertRaises(RuntimeError, collection.delete, "1" * 41) collection.close()
def test_metadata(self): """ test metadata """ URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name="pickle") self.assertEqual( collection.meta, {"name": "kvlite_test", "kvlite-version": kvlite.__version__, "serializer": "pickle"} ) collection.close()
def do_use(self, collection_name): ''' use <collection_name>\tuse the collection as the default (current) collection''' if collection_name in self.__kvlite_colls: self.prompt = '%s>' % collection_name self.__current_coll_name = collection_name self.__current_coll = kvlite.open(self.__kvlite_colls[self.__current_coll_name]) return else: print 'Error! Unknown collection: %s' % collection_name
def test_long_key(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) self.assertRaises(RuntimeError, collection.get, {'_key':'1'*41}) self.assertRaises(RuntimeError, collection.put, '1'*41, 'long_key') self.assertRaises(RuntimeError, collection.delete, '1'*41) collection.close()
def test_long_key(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) self.assertRaises(RuntimeError, collection.get, {'_key': '1' * 41}) self.assertRaises(RuntimeError, collection.put, '1' * 41, 'long_key') self.assertRaises(RuntimeError, collection.delete, '1' * 41) collection.close()
def test_sqlite_open(self): collection = kvlite.open('sqlite://testdb.sqlite:kvlite_test') collection.put('a',1) self.assertEqual(collection.count,1) self.assertEqual(collection.get('a'), ('a',1)) collection.delete('a') self.assertEqual(collection.count,0) self.assertEqual(collection.get('a'), (None,None)) collection.close()
def test_diff_key_length(self): ''' test for different key length ''' collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) collection.put(1, 'key-01') self.assertEqual(collection.get({'_key': '1'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01')) self.assertEqual(collection.get({'_key': '01'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01')) self.assertEqual(collection.get({'_key': '001'}), ('1'.zfill(kvlite.settings.KEY_LENGTH), 'key-01')) collection.close()
def test_sqlite_open(self): collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test') collection.put('a',1) self.assertEqual(collection.count,1) self.assertEqual(collection.get('a'), ('a',1)) collection.delete('a') self.assertEqual(collection.count,0) self.assertEqual(collection.get('a'), (None,None)) collection.close()
def setUp(self): self.collection = kvlite.open(URI + '.kvlite_test') self.collection_name = 'kvlite_test' self.manager = MysqlCollectionManager(URI) if self.collection_name not in self.manager.collections(): self.manager.create(self.collection_name) collection_class = self.manager.collection_class self.collection = collection_class(self.manager, self.collection_name)
def test_diff_key_length(self): """ test for different key length """ collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) collection.put(1, "key-01") self.assertEqual(collection.get({"_key": "1"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01")) self.assertEqual(collection.get({"_key": "01"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01")) self.assertEqual(collection.get({"_key": "001"}), ("1".zfill(kvlite.settings.KEY_LENGTH), "key-01")) collection.close()
def __init__(self): """ init """ if self.container: self._container = kvlite.open(self.container) else: raise RuntimeError("Undefined container: %s", self.container) self.urls_queue = list() self._processed_urls = list()
def __init__(self): ''' init ''' if self.container: self._container = kvlite.open(self.container) else: raise RuntimeError('Undefined container: %s', self.container) self.urls_queue = list() self._processed_urls = list()
def do_use(self, collection_name): ''' use <collection_name>\tuse the collection as the default (current) collection''' if collection_name in self.__kvlite_colls: self.prompt = '%s>' % collection_name self.__current_coll_name = collection_name self.__current_coll = kvlite.open( self.__kvlite_colls[self.__current_coll_name]) return else: print 'Error! Unknown collection: %s' % collection_name
def test_sqlite_open(self): _key = kvlite.get_uuid(1)[0] collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test') collection.put(_key, 1) self.assertEqual(collection.count, 1) self.assertEqual(collection.get({'_key': _key}), (_key, 1)) collection.delete(_key) self.assertEqual(collection.count, 0) self.assertEqual(collection.get({'_key': _key}), (None, None)) collection.close()
def test_sqlite_open(self): _key = kvlite.get_uuid(1)[0] collection = kvlite.open('sqlite://tests/db/testdb.sqlite:kvlite_test') collection.put(_key,1) self.assertEqual(collection.count,1) self.assertEqual(collection.get({'_key': _key}), (_key,1)) collection.delete(_key) self.assertEqual(collection.count,0) self.assertEqual(collection.get({'_key': _key}), (None,None)) collection.close()
def setUp(self): self.collection = kvlite.open(URI + '.kvlite_test') self.collection_name = 'kvlite_test' self.manager = MysqlCollectionManager(URI) if self.collection_name not in self.manager.collections(): self.manager.create(self.collection_name) collection_class = self.manager.collection_class self.collection = collection_class( self.manager, self.collection_name)
def get_item(collection, k): ''' get item details ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'} k, v = collection.get({'_key': k}) collection.close() return { 'status': 'OK', 'item': {'key': k, 'value': v}, }
def test_metadata(self): ''' test metadata ''' URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name='pickle') self.assertEqual( collection.meta, { 'name': 'kvlite_test', 'kvlite-version': kvlite.__version__, 'serializer': 'pickle', }) collection.close()
def test_put_get_delete_count_many(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) ks = [(collection.get_uuid(), "test_{}".format(i)) for i in xrange(100)] collection.put(ks) self.assertEqual(collection.count, 100) for k in ks: collection.delete(k[0]) self.assertEqual(collection.count, 0) collection.commit() collection.close()
def test_copy(self): ''' test_copy ''' COPIED_ITEMS = 1200 source_uri = self.URI.format(kvlite.utils.tmp_name()) source = kvlite.open(source_uri, serializer_name='pickle') target_uri = self.URI.format(kvlite.utils.tmp_name()) target = kvlite.open(target_uri, serializer_name='pickle') kv = [(k, 'value: %d' % k) for k in range(1,COPIED_ITEMS+1)] source.put(kv) source.commit() kvlite.utils.copy(source, target) self.assertEqual(target.count, COPIED_ITEMS) source.close() target.close()
def test_copy(self): ''' test_copy ''' COPIED_ITEMS = 250 source_uri = self.URI.format(kvlite.utils.tmp_name()) source = kvlite.open(source_uri, serializer_name='pickle') target_uri = self.URI.format(kvlite.utils.tmp_name()) target = kvlite.open(target_uri, serializer_name='pickle') kv = [(k, 'value: %d' % k) for k in range(1, COPIED_ITEMS + 1)] source.put(kv) source.commit() kvlite.utils.copy(source, target) self.assertEqual(target.count, COPIED_ITEMS) source.close() target.close()
def do_use(self, collection_name): """ use <collection_name>\tuse the collection as the default (current) collection""" if self.__current_coll: self.__current_coll.close() self.__current_coll = None if collection_name in self.__kvlite_colls: self.prompt = "%s>" % collection_name self.__current_coll_name = collection_name self.__current_coll = kvlite.open(self.__kvlite_colls[self.__current_coll_name]) return else: print "Error! Unknown collection: %s" % collection_name
def test_put_get_delete_count_one(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) k = collection.get_uuid() v = 'test_put_one' collection.put(k, v) self.assertEqual(collection.get({'_key': k}), (k, v)) self.assertEqual(collection.count, 1) collection.delete(k) self.assertEqual(collection.count, 0) collection.commit() collection.close()
def test_put_get_delete_count_one(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) k = collection.get_uuid() v = "test_put_one" collection.put(k, v) self.assertEqual(collection.get({"_key": k}), (k, v)) self.assertEqual(collection.count, 1) collection.delete(k) self.assertEqual(collection.count, 0) collection.commit() collection.close()
def test_put_get_delete_count_many(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) ks = [(collection.get_uuid(), 'test_{}'.format(i)) for i in xrange(100)] collection.put(ks) self.assertEqual(collection.count, 100) for k in ks: collection.delete(k[0]) self.assertEqual(collection.count, 0) collection.commit() collection.close()
def test_use_different_serializators_for_many(self): URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name="compressed_json") collection.put(u"11", u"diffser1") collection.put(u"22", u"diffser2") collection.put(u"33", u"diffser3") collection.commit() collection.close() # in version v0.6.3 you cannot re-define serializer in open() which was # assigned during first collection creation collection = kvlite.open(URI, serializer_name="pickle") res = [(k, v) for k, v in collection] self.assertEqual( res, [ ("11".zfill(kvlite.settings.KEY_LENGTH), u"diffser1"), ("22".zfill(kvlite.settings.KEY_LENGTH), u"diffser2"), ("33".zfill(kvlite.settings.KEY_LENGTH), u"diffser3"), ], ) collection.close()
def do_show(self, line): ''' show collections <details>\tlist of available collections''' if line.startswith('collections'): for coll in self.__kvlite_colls: print 'Database: %s' % coll print 'URI: %s' % self.__kvlite_colls[coll] coll = kvlite.open(self.__kvlite_colls[coll]) for k, v in coll.meta.items(): print '%s: %s' % (k, v) coll.close() print else: print 'Unknown argument: %s' % line
def do_show(self, line): """ show collections <details>\tlist of available collections""" if line.startswith("collections"): for coll in self.__kvlite_colls: print "Database: %s" % coll print "URI: %s" % self.__kvlite_colls[coll] coll = kvlite.open(self.__kvlite_colls[coll]) for k, v in coll.meta.items(): print "%s: %s" % (k, v) coll.close() print else: print "Unknown argument: %s" % line
def do_copy(self, line): """ copy <source> <target>\tcopy data from source kvlite database to target kvlite database <source> - reference name to source database <target> - reference name to target database for creating reference name, use `create` command""" try: source_ref, target_ref = [param for param in line.split(" ") if param <> ""][:2] except ValueError: print "Error! Please specify <source> and <target>" return if source_ref not in self.__kvlite_colls: print "Error! The source reference is not created, please use `create` command" return if target_ref not in self.__kvlite_colls: print "Error! The target reference is not created, please use `create` command" return source = kvlite.open(self.__kvlite_colls[source_ref]) target = kvlite.open(self.__kvlite_colls[target_ref]) kvlite.copy(source, target) source.close() target.close()
def do_show(self, line): ''' show collections <details>\tlist of available collections''' if line.startswith('collections'): for coll in self.__kvlite_colls: print 'Database: %s' % coll print 'URI: %s' % self.__kvlite_colls[coll] coll = kvlite.open(self.__kvlite_colls[coll]) for k,v in coll.meta.items(): print '%s: %s' % (k,v) coll.close() print else: print 'Unknown argument: %s' % line
def test_use_different_serializators_for_many(self): URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name='completed_json') collection.put(u'11', u'diffser1') collection.put(u'22', u'diffser2') collection.put(u'33', u'diffser3') collection.commit() collection.close() collection = kvlite.open(URI, serializer_name='pickle') with self.assertRaises(RuntimeError): res = [(k, v) for k, v in collection] collection.put(u'11', u'diffser1') collection.put(u'22', u'diffser2') collection.put(u'33', u'diffser3') collection.commit() collection.close() collection = kvlite.open(URI, serializer_name='completed_json') with self.assertRaises(RuntimeError): res = [(k, v) for k, v in collection] collection.close()
def test_use_different_serializators_for_many(self): URI = self.URI.format(kvlite.utils.tmp_name()) collection = kvlite.open(URI, serializer_name='completed_json') collection.put(u'11', u'diffser1') collection.put(u'22', u'diffser2') collection.put(u'33', u'diffser3') collection.commit() collection.close() collection = kvlite.open(URI, serializer_name='pickle') with self.assertRaises(RuntimeError): res = [(k,v) for k,v in collection] collection.put(u'11', u'diffser1') collection.put(u'22', u'diffser2') collection.put(u'33', u'diffser3') collection.commit() collection.close() collection = kvlite.open(URI, serializer_name='completed_json') with self.assertRaises(RuntimeError): res = [(k,v) for k,v in collection] collection.close()
def delete_item(collection, k): ''' delete item by key ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'} try: collection.delete(k) collection.commit() collection.close() return { 'status': 'OK', } except: return { 'status': 'Error', 'message': 'Cannot delete the item by key', }
def test_get_many_by_keys(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) kvs = [(collection.get_uuid(), "test") for _ in range(10)] collection.put(kvs) collection.commit() # keys are in database result = [kv for kv in collection.get({"_key": [kv[0] for kv in kvs[0:3]]})] self.assertEqual(len(kvs[0:3]), len(result)) for res in result: self.assertIn(res, kvs[0:3]) # keys are not in database result = [kv for kv in collection.get({"_key": ["00" for kv in range(3)]})] self.assertNotEqual(len(kvs[0:3]), len(result)) collection.close()
def test_pagination(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) PAGE_SIZE = 10 kvs = [(collection.get_uuid(), "test") for _ in range(100)] collection.put(kvs) # first page result = [kv for kv in collection.get(offset=0, limit=PAGE_SIZE)] self.assertEqual(len(result), len(kvs[0:PAGE_SIZE])) for res in result: self.assertIn(res, kvs[0:PAGE_SIZE]) # second page result = [kv for kv in collection.get(offset=2 * PAGE_SIZE, limit=PAGE_SIZE)] self.assertEqual(len(result), len(kvs[2 * PAGE_SIZE : 2 * PAGE_SIZE + PAGE_SIZE])) for res in result: self.assertIn(res, kvs[2 * PAGE_SIZE : 2 * PAGE_SIZE + PAGE_SIZE]) collection.close()
def get_page(name, page): ''' return collection data for specific page ''' page = int(page) try: collection_uri = settings.COLLECTIONS[name] except KeyError: return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'} collection = kvlite.open(collection_uri) last_page = collection.count / settings.ITEMS_PER_PAGE #last_page = int(ceil(collection.count / float(settings.ITEMS_PER_PAGE))) data = [kv for kv in collection.get(offset=page * settings.ITEMS_PER_PAGE, limit=settings.ITEMS_PER_PAGE)] collection.close() return { 'status': 'OK', 'last_page': last_page, 'data': data, }
def test_pagination(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) PAGE_SIZE = 10 kvs = [(collection.get_uuid(), 'test') for _ in range(100)] collection.put(kvs) # first page result = [kv for kv in collection.get(offset=0, limit=PAGE_SIZE)] self.assertEqual(len(result), len(kvs[0:PAGE_SIZE])) for res in result: self.assertIn(res, kvs[0:PAGE_SIZE]) # second page result = [ kv for kv in collection.get(offset=2 * PAGE_SIZE, limit=PAGE_SIZE) ] self.assertEqual(len(result), len(kvs[2 * PAGE_SIZE:2 * PAGE_SIZE + PAGE_SIZE])) for res in result: self.assertIn(res, kvs[2 * PAGE_SIZE:2 * PAGE_SIZE + PAGE_SIZE]) collection.close()
def create_update_item(collection, k, v): ''' create or update item ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return {'error': 'The collection %s is not found' % name, 'status': 'NOT OK'} if v: try: collection.put(k, json.loads(v)) collection.commit() collection.close() return { 'status': 'OK', } except Exception, err: return { 'status': 'Error', 'message': err, }
def test_get_many_by_keys(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) kvs = [(collection.get_uuid(), 'test') for _ in range(10)] collection.put(kvs) collection.commit() # keys are in database result = [ kv for kv in collection.get({'_key': [kv[0] for kv in kvs[0:3]]}) ] self.assertEqual(len(kvs[0:3]), len(result)) for res in result: self.assertIn(res, kvs[0:3]) # keys are not in database result = [ kv for kv in collection.get({'_key': ['00' for kv in range(3)]}) ] self.assertNotEqual(len(kvs[0:3]), len(result)) collection.close()
def get_item(collection, k): ''' get item details ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return { 'error': 'The collection %s is not found' % name, 'status': 'NOT OK' } k, v = collection.get({'_key': k}) collection.close() return { 'status': 'OK', 'item': { 'key': k, 'value': v }, }
def delete_item(collection, k): ''' delete item by key ''' try: collection_uri = settings.COLLECTIONS[collection] collection = kvlite.open(collection_uri) except KeyError: return { 'error': 'The collection %s is not found' % name, 'status': 'NOT OK' } try: collection.delete(k) collection.commit() collection.close() return { 'status': 'OK', } except: return { 'status': 'Error', 'message': 'Cannot delete the item by key', }
def get_page(name, page): ''' return collection data for specific page ''' page = int(page) try: collection_uri = settings.COLLECTIONS[name] except KeyError: return { 'error': 'The collection %s is not found' % name, 'status': 'NOT OK' } collection = kvlite.open(collection_uri) last_page = collection.count / settings.ITEMS_PER_PAGE #last_page = int(ceil(collection.count / float(settings.ITEMS_PER_PAGE))) data = [ kv for kv in collection.get(offset=page * settings.ITEMS_PER_PAGE, limit=settings.ITEMS_PER_PAGE) ] collection.close() return { 'status': 'OK', 'last_page': last_page, 'data': data, }
def test_put_wrong_oldformat_kv(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) collection.put(collection.get_uuid(), 'value') collection.close()
def test_absent_key(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) self.assertEqual(collection.get({'_key': u'a1b2c3'}), (None, None)) collection.close()
def test_get_uuid(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) uuids = [collection.get_uuid() for i in range(1000)] self.assertEqual(len(set(uuids)), 1000) collection.close()
def test_absent_key(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) self.assertEqual(collection.get({"_key": u"a1b2c3"}), (None, None)) collection.close()
def test_put_wrong_oldformat_kv(self): collection = kvlite.open(self.URI.format(kvlite.utils.tmp_name())) collection.put(collection.get_uuid(), "value") collection.close()