def test_put_with_float_data_user_deserializer_none(self): """ Invoke put() for float data record with user deserializer None. """ if TestUserSerializer.skip_old_server is False: aerospike.set_serializer(serialize_function) else: aerospike.set_serializer(serialize_function_old_server) key = ('test', 'demo', 1) rec = {"pi": 3.14} res = TestUserSerializer.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 self.delete_keys.append(key) try: aerospike.set_deserializer(None) except e.ParamError as exception: assert exception.code == -2 assert exception.msg == "Parameter must be a callable"
def test_with_class_serializer_and_instance_serializer(self): """ Invoke put() for mixed data record with class and instance serializer. Verify that the instance serializer takes precedence """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) hostlist, user, password = TestBaseClass.get_hosts() method_config = { 'hosts': hostlist, 'serialization': (instance_serializer, instance_deserializer)} client = TestBaseClass.get_new_connection(method_config) response = client.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) _, _, bins = client.get(self.test_key) assert bins == {'normal': 1234, 'tuple': ( 'instance serialized', 'instance deserialized' ) } client.close()
def test_pos_put_user_serializer_no_deserializer(self): """ Invoke put() for float data record with user serializer is registered, but deserializer is not registered. """ key = ('test', 'demo', 'put_user_serializer') rec = {"pi": 3.14} def serialize_function(val): return pickle.dumps(val) aerospike.set_serializer(serialize_function) res = self.as_connection.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = self.as_connection.get(key) if self.skip_old_server is False: assert bins == {'pi': 3.14} else: assert bins == {'pi': bytearray(b'F3.1400000000000001\n.')} self.as_connection.remove(key)
def test_class_serializer_non_callable(self): """ Verify that calling unset_serializers actually removes the class serializer """ with pytest.raises(e.ParamError): aerospike.set_serializer(5)
def test_put_user_serializer_no_deserializer(self): """ Invoke put() for float data record with user serializer is registered, but deserializer is not registered. """ key = ('test', 'demo', 1) rec = {"pi": 3.14} def serialize_function(val): return marshal.dumps(val) aerospike.set_serializer(serialize_function) res = TestPut.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestPut.client.get(key) if TestPut.skip_old_server is False: assert bins == {'pi': 3.14} else: assert bins == {'pi': bytearray(b'g\x1f\x85\xebQ\xb8\x1e\t@')} self.delete_keys.append(key)
def test_with_class_serializer_and_instance_serializer_with_unset_serializer(self): """ Invoke put() for mixed data record with python serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = {"hosts": hostlist} if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ("test", "demo", 16) try: TestPythonSerializer.client.remove(key) except: pass rec = {"normal": 1234, "tuple": (1, 2, 3)} aerospike.unset_serializers() try: client.put(key, rec, serializer=aerospike.SERIALIZER_USER) except e.ClientError as exception: assert exception.code == -1 assert exception.msg == "No serializer callback registered"
def test_with_unset_serializer_python_serializer(self): """ Invoke put() for mixed data record with python serializer and calling unset_serializers """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = {"hosts": hostlist} if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ("test", "demo", 16) try: client.remove(key) except: pass rec = {"normal": 1234, "tuple": (1, 2, 3)} aerospike.unset_serializers() res = client.put(key, rec, serializer=aerospike.SERIALIZER_PYTHON) assert res == 0 _, _, bins = client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {"normal": 1234, "tuple": (1, 2, 3)} client.remove(key)
def test_with_class_serializer_and_instance_serializer(self): pytest.xfail(reason="Serialization of Bytes not functional") """ Invoke put() for mixed data record with class and instance serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = { "hosts": hostlist, "serialization": ((lambda v: marshal.dumps(v)), (lambda v: marshal.loads(v))), } if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ("test", "demo", 16) try: TestPythonSerializer.client.remove(key) except: pass rec = {"normal": 1234, "tuple": (1, 2, 3)} res = client.put(key, rec, serializer=aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {"normal": 1234, "tuple": (1, 2, 3)} client.remove(key)
def test_with_class_serializer_and_instance_serializer_with_unset_serializer( self): """ Invoke put() for mixed data record with python serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = {'hosts': hostlist} if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ('test', 'demo', 16) try: TestPythonSerializer.client.remove(key) except: pass rec = {'normal': 1234, 'tuple': (1, 2, 3)} aerospike.unset_serializers() try: client.put(key, rec, serializer=aerospike.SERIALIZER_USER) except e.ClientError as exception: assert exception.code == -1 assert exception.msg == "No serializer callback registered"
def test_with_unset_serializer_python_serializer(self): """ Invoke put() for mixed data record with python serializer and calling unset_serializers """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = {'hosts': hostlist} if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ('test', 'demo', 16) try: client.remove(key) except: pass rec = {'normal': 1234, 'tuple': (1, 2, 3)} aerospike.unset_serializers() res = client.put(key, rec, serializer=aerospike.SERIALIZER_PYTHON) assert res == 0 _, _, bins = client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} client.remove(key)
def test_with_class_serializer_and_instance_serializer(self): pytest.xfail(reason="Serialization of Bytes not functional") """ Invoke put() for mixed data record with class and instance serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) hostlist, user, password = TestBaseClass.get_hosts() method_config = { 'hosts': hostlist, 'serialization': ((lambda v: marshal.dumps(v)), (lambda v: marshal.loads(v))) } if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) key = ('test', 'demo', 16) try: TestPythonSerializer.client.remove(key) except: pass rec = {'normal': 1234, 'tuple': (1, 2, 3)} res = client.put(key, rec, serializer=aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} client.remove(key)
def test_put_with_mixeddata_client_serializer_deserializer_with_spec_in_put(self): pytest.xfail(reason="Need Python 2/3 compatible bytearray for strings") # Invoke put() for mixed data with class and instance serialziers # with a specification in put. Client one is called hostlist, user, password = TestBaseClass.get_hosts() method_config = {'hosts': hostlist, 'serialization': (client_serialize_function, client_deserialize_function)} if user is None and password is None: client = aerospike.client(method_config).connect() else: client = aerospike.client(method_config).connect(user, password) aerospike.set_serializer(serialize_function) aerospike.set_deserializer(deserialize_function) key = ('test', 'demo', 1) rec = { 'map': {"key": "asd';q;'1';", "pi": 3}, 'normal': 1234, 'special': '!@#@#$QSDAsd;as', 'list': ["nanslkdl", 1, bytearray("asd;as[d'as;d")], 'bytes': bytearray("asd;as[d'as;d"), 'nestedlist': ["nanslkdl", 1, bytearray("asd;as[d'as;d"), [1, bytearray("asd;as[d'as;d")]], 'nestedmap': { "key": "asd';q;'1';", "pi": 314, "nest": {"pi1": 312, "t": 1} }, } res = client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = client.get(key) assert bins == { 'map': {"key": "asd';q;'1';", "pi": 3}, 'normal': 1234, 'special': '!@#@#$QSDAsd;as', 'list': ["nanslkdl", 1, bytearray("asd;as[d'as;d", "utf-8")], 'bytes': bytearray("asd;as[d'as;d", "utf-8"), 'nestedlist': ["nanslkdl", 1, bytearray("asd;as[d'as;d", "utf-8"), [1, bytearray("asd;as[d'as;d", "utf-8")]], 'nestedmap': { "key": "asd';q;'1';", "pi": 314, "nest": {"pi1": 312, "t": 1} }, } client.close() self.delete_keys.append(key)
def test_builtin_with_class_serializer_and_instance_serializer(self): """ Invoke put() passing SERIALIZER_PYTHON and verify that it causes the language serializer to override instance and class serializers """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) hostlist, user, password = TestBaseClass.get_hosts() method_config = { 'hosts': hostlist, 'serialization': (instance_serializer, instance_deserializer)} client = TestBaseClass.get_new_connection(method_config) rec = {'normal': 1234, 'tuple': (1, 2, 3)} response = client.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_PYTHON) assert response == 0 _, _, bins = client.get(self.test_key) # The instance and class serializers would have mutated this, # So getting back what we put in means the language serializer ran assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} client.close()
def test_serializer_with_two_args(self): aerospike.set_serializer(serializer_two_arg) aerospike.set_deserializer(class_deserializer) with pytest.raises(e.ClientError): self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER)
def test_put_with_mixed_data_user_serializer(self): pytest.xfail(reason="Need Python 2/3 compatible bytearray for strings") """ Invoke put() for mixed data record with user serializer. """ key = ('test', 'demo', 1) if TestUserSerializer.skip_old_server is False: aerospike.set_serializer(serialize_function) aerospike.set_deserializer(deserialize_function) else: aerospike.set_serializer(serialize_function_old_server) aerospike.set_deserializer(deserialize_function_old_server) rec = { 'map': {"key": "asd';q;'1';", "pi": 3.14}, 'normal': 1234, 'special': '!@#@#$QSDAsd;as', 'list': ["nanslkdl", 1, bytes("asd;as[d'as;d", "utf-8")], 'bytes': bytes("asd;as[d'as;d", "utf-8"), 'nestedlist': ["nanslkdl", 1, bytes("asd;as[d'as;d", "utf-8"), [1, bytes("asd;as[d'as;d", "utf-8")]], 'nestedmap': { "key": "asd';q;'1';", "pi": 3.14, "nest": {"pi1": 3.12, "t": 1} }, } res = TestUserSerializer.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestUserSerializer.client.get(key) assert bins == { 'map': {"key": "asd';q;'1';", "pi": 3.14}, 'normal': 1234, 'special': '!@#@#$QSDAsd;as', 'list': ["nanslkdl", 1, bytes("asd;as[d'as;d", "utf-8")], 'bytes': bytes("asd;as[d'as;d", "utf-8"), 'nestedlist': ["nanslkdl", 1, bytes("asd;as[d'as;d", "utf-8"), [1, bytes("asd;as[d'as;d", "utf-8")]], 'nestedmap': {"key": "asd';q;'1';", "pi": 3.14, "nest": {"pi1": 3.12, "t": 1}}, } self.delete_keys.append(key)
def test_put_with_float_data_user_serializer_none(self): """ Invoke put() for float data record with user serializer. """ try: aerospike.set_serializer(None) except e.ParamError as exception: assert exception.code == -2 assert exception.msg == "Parameter must be a callable"
def test_only_setting_a_serializer(self): aerospike.set_serializer(class_serializer) self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) _, _, record = self.as_connection.get(self.test_key) # this should not have been deserialized with the class serializer # it should now have used the language default serializer assert record['tuple'] != ('class serialized', 'class deserialized')
def test_changing_deserializer_after_a_record_put(self): aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) aerospike.set_deserializer(instance_deserializer) _, _, record = self.as_connection.get(self.test_key) # this should not have been deserialized with the class serializer # it should now have used the instance deserializer assert record['tuple'] == ('class serialized', 'instance deserialized')
def test_deserializer_raises_error(self): # If the deserializer failed, we should get a bytearray # representation of the item aerospike.set_serializer(class_serializer) aerospike.set_deserializer(deserializer_error) self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) _, _, response = self.as_connection.get(self.test_key) assert response['normal'] == 1234 assert isinstance(response['tuple'], bytearray)
def test_unsetting_serializers_after_a_record_put(self): aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) aerospike.unset_serializers() _, _, record = self.as_connection.get(self.test_key) # this should not have been deserialized with the class serializer # it will have been deserialized by the class deserializer, # so this should be a deserialization of 'class serialized' assert record['tuple'] != ('class serialized', 'class deserialized')
def test_put_user_serializer_no_deserializer(self): """ Invoke put() for float data record with user serializer is registered, but deserializer is not registered. """ key = ('test', 'demo', 1) rec = {"pi": 3.14} def serialize_function(val): return pickle.dumps(val) response = aerospike.set_serializer(serialize_function) res = TestPut.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestPut.client.get(key) if TestPut.skip_old_server == False: assert bins == {'pi': 3.14} else: assert bins == {'pi': bytearray(b'F3.1400000000000001\n.')} self.delete_keys.append(key)
def test_put_with_no_serializer_arg_and_class_serializer_set(self): ''' Test that when no serializer arg is passed in, and no instance serializer has been set, and a class serializer has been set, the default language serializer is used. ''' aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) response = self.as_connection.put(self.test_key, self.mixed_record) _, _, bins = self.as_connection.get(self.test_key) # The class serializer would have altered our record, so we # We check that the data is the same as what we stored assert bins == {'normal': 1234, 'tuple': (1, 2, 3)}
def test_setting_serializer_is_a_per_rec_setting(self): aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) self.as_connection.put( ('test', 'demo', 'test_record_2'), self.mixed_record) _, _, record = self.as_connection.get( ('test', 'demo', 'test_record_2')) self.as_connection.remove(('test', 'demo', 'test_record_2')) # this should not have been serialized with the class serializer assert record == self.mixed_record
def test_builtin_with_class_serializer(self): """ Invoke put() for mixed data record with builtin serializer """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) response = self.as_connection.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_PYTHON) assert response == 0 _, _, bins = self.as_connection.get(self.test_key) # The class serializer would have altered our record, so we # We check that the data is the same as what we stored assert bins == {'normal': 1234, 'tuple': (1, 2, 3)}
def test_class_serializer_unset(self): """ Verify that calling unset_serializers actually removes the class serializer """ aerospike.set_serializer(json.dumps) aerospike.set_deserializer(json.loads) hostlist, user, password = TestBaseClass.get_hosts() method_config = { 'hosts': hostlist } client = TestBaseClass.get_new_connection(method_config) aerospike.unset_serializers() with pytest.raises(e.ClientError) as err_info: client.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) assert err_info.value.code == AerospikeStatus.AEROSPIKE_ERR_CLIENT
def test_put_with_bool_data_user_serializer(self): """ Invoke put() for bool data record with user serializer. """ aerospike.set_serializer(serialize_function) aerospike.set_deserializer(deserialize_function) key = ('test', 'demo', 1) rec = {'status': True} res = TestUserSerializer.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestUserSerializer.client.get(key) assert bins == {'status': True} self.delete_keys.append(key)
def test_with_class_serializer(self): """ Invoke put() for mixed data record with class serializer. It should get called when serializer is passed """ aerospike.set_serializer(class_serializer) aerospike.set_deserializer(class_deserializer) rec = {'normal': 1234, 'tuple': (1, 2, 3)} response = self.as_connection.put(self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_USER) assert response == 0 _, _, bins = self.as_connection.get(self.test_key) assert bins == { 'normal': 1234, 'tuple': ('class serialized', 'class deserialized') }
def test_with_unset_serializer_python_serializer(self): """ Invoke put() for mixed data record with python serializer and calling unset_serializers """ aerospike.set_serializer(json.dumps) aerospike.set_deserializer(json.loads) hostlist, user, password = TestBaseClass.get_hosts() method_config = { 'hosts': hostlist } client = TestBaseClass.get_new_connection(method_config) aerospike.unset_serializers() response = client.put( self.test_key, self.mixed_record, serializer=aerospike.SERIALIZER_PYTHON) _, _, bins = client.get(self.test_key) assert bins == {'normal': 1234, 'tuple': (1, 2, 3)} client.close()
def test_with_class_serializer(self): """ Invoke put() for mixed data record with class serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) key = ("test", "demo", 14) try: TestPythonSerializer.client.remove(key) except: pass rec = {"normal": 1234, "tuple": (1, 2, 3)} res = TestPythonSerializer.client.put(key, rec, serializer=aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestPythonSerializer.client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {"normal": 1234, "tuple": [1, 2, 3]} self.delete_keys.append(key)
def test_put_with_float_data_user_serializer(self): # Invoke put() for float data record with user serializer. if TestUserSerializer.skip_old_server is False: aerospike.set_serializer(serialize_function) aerospike.set_deserializer(deserialize_function) else: aerospike.set_serializer(serialize_function_old_server) aerospike.set_deserializer(deserialize_function_old_server) key = ('test', 'demo', 1) rec = {"pi": 3.14} res = TestUserSerializer.client.put(key, rec, {}, {}, aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestUserSerializer.client.get(key) assert bins == {'pi': 3.14} self.delete_keys.append(key)
def test_with_class_serializer(self): """ Invoke put() for mixed data record with class serializer. """ aerospike.set_serializer((lambda v: json.dumps(v))) aerospike.set_deserializer((lambda v: json.loads(v))) key = ('test', 'demo', 14) try: TestPythonSerializer.client.remove(key) except: pass rec = {'normal': 1234, 'tuple': (1, 2, 3)} res = TestPythonSerializer.client.put( key, rec, serializer=aerospike.SERIALIZER_USER) assert res == 0 _, _, bins = TestPythonSerializer.client.get(key) # tuples JSON-encode to a list, and we use this fact to check which # serializer ran: assert bins == {'normal': 1234, 'tuple': [1, 2, 3]} self.delete_keys.append(key)
# Python objects load objects_to_write = {} with open('lr_model', 'rb') as f: objects_to_write["lr_model"] = joblib.load(f) with open('count_vectorizer', 'rb') as f: objects_to_write["count_vectorizer"] = joblib.load(f) with open('tfidf_transformer', 'rb') as f: objects_to_write["tfidf_transformer"] = joblib.load(f) # Aerospike serializer for the matrices aerospike.set_serializer(joblib_serializer) # Create a client and connect it to the cluster try: client = aerospike.client(config).connect() except: import sys print("failed to connect to the cluster with", config['hosts']) sys.exit(1) # Records are addressable via a tuple of (namespace, set, key) # for k, v in objects_to_write.iteritems(): for k, v in objects_to_write.items(): key = (aerospike_namespace, aerospike_set, k) try: client.put(key, {'key': k, 'value': v})