def test_numpy_codec(self): a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264], dtype='float32') invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a == b).all()) # Rank 1, length 1 works: a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264], dtype='float32') mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a == b).all())
def test_numpy_codec(self): a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32') invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a==b).all()) # Rank 1, length 1 works: a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32') mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a==b).all()) # Rank 0 array raises Value Error because numpy tolist does not return a list a = np.array(3.14159265358979323846264,dtype='float32') invoke = Invocation() invoke.message = a self.assertRaises(ValueError, codec.outgoing, invoke)
def test_encode_dotdict(self): from interface.messages import process_dispatcher_schedule_process_in msg_obj = process_dispatcher_schedule_process_in() msg_obj.name = "process" msg_obj.configuration = {"process": {"property": "FOO"}} invoke = Invocation() invoke.message = msg_obj encode = EncodeInterceptor() mangled = encode.outgoing(invoke) msg_encoded1 = mangled.message received = encode.incoming(mangled) msg_rec1 = received.message msg_obj = process_dispatcher_schedule_process_in() msg_obj.name = "process" msg_obj.configuration = DotDict() msg_obj.configuration.process.property = "FOO" invoke = Invocation() invoke.message = msg_obj encode = EncodeInterceptor() mangled = encode.outgoing(invoke) msg_encoded2 = mangled.message received = encode.incoming(mangled) msg_rec2 = received.message self.assertEquals(msg_encoded1, msg_encoded2) self.assertIsInstance(msg_rec1["configuration"], dict) self.assertIsInstance(msg_rec2["configuration"], dict)
def test_numpy_codec(self): a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264], dtype='float32') invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a == b).all()) # Rank 1, length 1 works: a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264], dtype='float32') mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertTrue((a == b).all()) # Rank 0 array raises Value Error because numpy tolist does not return a list a = np.array(3.14159265358979323846264, dtype='float32') invoke = Invocation() invoke.message = a self.assertRaises(ValueError, codec.outgoing, invoke)
def test_encode_dotdict(self): from interface.messages import process_management_schedule_process_in msg_obj = process_management_schedule_process_in() msg_obj.name = "process" msg_obj.configuration = {"process": {"property": "FOO"}} invoke = Invocation() invoke.message = msg_obj encode = EncodeInterceptor() mangled = encode.outgoing(invoke) msg_encoded1 = mangled.message received = encode.incoming(mangled) msg_rec1 = received.message msg_obj = process_management_schedule_process_in() msg_obj.name = "process" msg_obj.configuration = DotDict() msg_obj.configuration.process.property = "FOO" invoke = Invocation() invoke.message = msg_obj encode = EncodeInterceptor() mangled = encode.outgoing(invoke) msg_encoded2 = mangled.message received = encode.incoming(mangled) msg_rec2 = received.message self.assertEquals(msg_encoded1, msg_encoded2) self.assertIsInstance(msg_rec1["configuration"], dict) self.assertIsInstance(msg_rec2["configuration"], dict)
def test_slice(self): a = slice(5, 20, 2) invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b)
def test_scalars(self): a = np.uint64(312) invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b)
def test_set(self): a = {1,2} invoke = Invocation() invoke.message = a encode = EncodeInterceptor() mangled = encode.outgoing(invoke) received = encode.incoming(mangled) b = received.message self.assertEquals(a,b)
def test_dtype(self): a = np.dtype("float32") invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b) a = np.dtype("object") invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b)
def test_dtype(self): a = np.dtype('float32') invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b) a = np.dtype('object') invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message self.assertEquals(a, b)
def test_numpy_encode(self): a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32') invoke = Invocation() invoke.message = a encode = EncodeInterceptor() mangled = encode.outgoing(invoke) received = encode.incoming(mangled) b = received.message self.assertTrue((a==b).all()) # Rank 1, length 1 works: a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32') mangled = encode.outgoing(invoke) received = encode.incoming(mangled) b = received.message self.assertTrue((a==b).all())
def test_dtype(self): a = np.dtype('float32') invoke = Invocation() invoke.message = a encode = EncodeInterceptor() mangled = encode.outgoing(invoke) received = encode.incoming(mangled) b = received.message self.assertEquals(a,b) a = np.dtype('object') invoke = Invocation() invoke.message = a encode = EncodeInterceptor() mangled = encode.outgoing(invoke) received = encode.incoming(mangled) b = received.message self.assertEquals(a,b)
def test_packed_numpy(self): a = np.array([(90, 8010, 3, 14112, 3.14159265358979323846264)], dtype="float32") invoke = Invocation() invoke.message = {"double stuffed": [a, a, a]} codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message c = b.get("double stuffed") for d in c: self.assertTrue((a == d).all())
def test_packed_numpy(self): a = np.array([(90,8010,3,14112,3.14159265358979323846264)],dtype='float32') invoke = Invocation() invoke.message = {'double stuffed':[a,a,a]} codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message c = b.get('double stuffed') for d in c: self.assertTrue((a==d).all())
def test_set(self): a = {'s': set([1, 2, 3]), 'l': [1, 2, 3], 't': (1, 2, 3)} invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message # We only get lists back - damn you msgpack! only_lists = {'s': set([1, 2, 3]), 'l': [1, 2, 3], 't': [1, 2, 3]} self.assertEquals(only_lists, b)
def test_set(self): a = {'s':set([1,2,3]),'l':[1,2,3],'t':(1,2,3)} invoke = Invocation() invoke.message = a codec = EncodeInterceptor() mangled = codec.outgoing(invoke) received = codec.incoming(mangled) b = received.message # We only get lists back - damn you msgpack! only_lists = {'s':set([1,2,3]),'l':[1,2,3],'t':[1,2,3]} self.assertEquals(only_lists,b)
def test_perf(self): _io_serializer = IonObjectSerializer() _io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry()) def time_serialize(test_obj, name="?", has_ion=False): with time_it(name + ", serialize"): os = _io_serializer.serialize(test_obj) with time_it(name + ", deserialize"): os2 = _io_deserializer.deserialize(os) count_objs(os) if has_ion: test_obj = os with time_it(name + ", json.dumps"): oj = json.dumps(test_obj) with time_it(name + ", json.loads"): o2 = json.loads(oj) log.info(" len(json): %s", len(oj)) with time_it(name + ", simplejson.dumps"): oj = simplejson.dumps(test_obj) with time_it(name + ", simplejson.loads"): o2 = simplejson.loads(oj) log.info(" len(simplejson): %s", len(oj)) with time_it(name + ", msgpack.packb"): o1 = msgpack.packb(test_obj) with time_it(name + ", msgpack.unpackb"): o2 = msgpack.unpackb(o1, use_list=1) log.info(" len(msgpack): %s", len(o1)) # with time_it(name + ", pickle.dumps"): # op = pickle.dumps(test_obj) # # with time_it(name + ", pickle.loads"): # o2 = pickle.loads(op) # log.info(" len(pickle): %s", len(op)) # # with time_it(name + ", cPickle.dumps"): # op = cPickle.dumps(test_obj) # # with time_it(name + ", cPickle.loads"): # o2 = cPickle.loads(op) # log.info(" len(cPickle): %s", len(op)) log.info("----------------") # Large nested with time_it("highly nested dict/list, create"): test_obj = create_test_object(4, 4, do_list=False, uvals=True, ukeys=True) time_serialize(test_obj, "highly nested dict/list") # Nested with time_it("nested dict/list, create"): test_obj = create_test_object(3, 40, do_list=True, uvals=False, ukeys=False) time_serialize(test_obj, "nested dict/list") # Large string #value = ''.join(random.choice(allowed_chars) for x in xrange(1460000)) value = ''.join(random.choice(allowed_chars) for x in xrange(500000)) time_serialize(value, "long string") # ION with time_it("create ion"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested", has_ion=True) from pyon.core.interceptor.interceptor import Invocation from pyon.core.interceptor.encode import EncodeInterceptor encode = EncodeInterceptor() invocation = Invocation() invocation.message = test_obj1 with time_it("ion object, encode"): encode.outgoing(invocation) with time_it("ion object, decode"): encode.incoming(invocation) count_objs(invocation.message) # ION with time_it("create ion unicode"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False, uvals=True, ukeys=True) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested unicode", has_ion=True) # Create objects with validation on with time_it("create ion calidated"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=True) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested validated", has_ion=True)
def test_perf(self): _io_serializer = IonObjectSerializer() _io_deserializer = IonObjectDeserializer( obj_registry=get_obj_registry()) def time_serialize(test_obj, name="?", has_ion=False): with time_it(name + ", serialize"): os = _io_serializer.serialize(test_obj) with time_it(name + ", deserialize"): os2 = _io_deserializer.deserialize(os) count_objs(os) if has_ion: test_obj = os with time_it(name + ", json.dumps"): oj = json.dumps(test_obj) with time_it(name + ", json.loads"): o2 = json.loads(oj) log.info(" len(json): %s", len(oj)) with time_it(name + ", simplejson.dumps"): oj = simplejson.dumps(test_obj) with time_it(name + ", simplejson.loads"): o2 = simplejson.loads(oj) log.info(" len(simplejson): %s", len(oj)) with time_it(name + ", msgpack.packb"): o1 = msgpack.packb(test_obj) with time_it(name + ", msgpack.unpackb"): o2 = msgpack.unpackb(o1, use_list=1) log.info(" len(msgpack): %s", len(o1)) # with time_it(name + ", pickle.dumps"): # op = pickle.dumps(test_obj) # # with time_it(name + ", pickle.loads"): # o2 = pickle.loads(op) # log.info(" len(pickle): %s", len(op)) # # with time_it(name + ", cPickle.dumps"): # op = cPickle.dumps(test_obj) # # with time_it(name + ", cPickle.loads"): # o2 = cPickle.loads(op) # log.info(" len(cPickle): %s", len(op)) log.info("----------------") # Large nested with time_it("highly nested dict/list, create"): test_obj = create_test_object(4, 4, do_list=False, uvals=True, ukeys=True) time_serialize(test_obj, "highly nested dict/list") # Nested with time_it("nested dict/list, create"): test_obj = create_test_object(3, 40, do_list=True, uvals=False, ukeys=False) time_serialize(test_obj, "nested dict/list") # Large string #value = ''.join(random.choice(allowed_chars) for x in xrange(1460000)) value = ''.join(random.choice(allowed_chars) for x in xrange(500000)) time_serialize(value, "long string") # ION with time_it("create ion"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested", has_ion=True) from pyon.core.interceptor.interceptor import Invocation from pyon.core.interceptor.encode import EncodeInterceptor encode = EncodeInterceptor() invocation = Invocation() invocation.message = test_obj1 with time_it("ion object, encode"): encode.outgoing(invocation) with time_it("ion object, decode"): encode.incoming(invocation) count_objs(invocation.message) # ION with time_it("create ion unicode"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False, uvals=True, ukeys=True) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested unicode", has_ion=True) # Create objects with validation on with time_it("create ion calidated"): test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=True) count_objs(test_obj1) time_serialize(test_obj1, "dict of ion nested validated", has_ion=True)