def test_numpy_codec(self):
a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a==b).all())
# Rank 1, length 1 works:
a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32')
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a==b).all())
# Rank 0 array raises Value Error because numpy tolist does not return a list
a = np.array(3.14159265358979323846264,dtype='float32')
invoke = Invocation()
invoke.message = a
self.assertRaises(ValueError, codec.outgoing, invoke)
def test_set(self):
a = {1,2}
invoke = Invocation()
invoke.message = a
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
received = encode.incoming(mangled)
b = received.message
self.assertEquals(a,b)
def test_slice(self):
a = slice(5, 20, 2)
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_scalars(self):
a = np.uint64(312)
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_slice(self):
a = slice(5, 20, 2)
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_set(self):
a = {1,2}
invoke = Invocation()
invoke.message = a
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
received = encode.incoming(mangled)
b = received.message
self.assertEquals(a,b)
def test_scalars(self):
a = np.uint64(312)
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_encode_dotdict(self):
from interface.messages import process_management_schedule_process_in
msg_obj = process_management_schedule_process_in()
msg_obj.name = "process"
msg_obj.configuration = {"process": {"property": "FOO"}}
invoke = Invocation()
invoke.message = msg_obj
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
msg_encoded1 = mangled.message
received = encode.incoming(mangled)
msg_rec1 = received.message
msg_obj = process_management_schedule_process_in()
msg_obj.name = "process"
msg_obj.configuration = DotDict()
msg_obj.configuration.process.property = "FOO"
invoke = Invocation()
invoke.message = msg_obj
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
msg_encoded2 = mangled.message
received = encode.incoming(mangled)
msg_rec2 = received.message
self.assertEquals(msg_encoded1, msg_encoded2)
self.assertIsInstance(msg_rec1["configuration"], dict)
self.assertIsInstance(msg_rec2["configuration"], dict)
def test_numpy_codec(self):
a = np.array([90,8010,3,14112,3.14159265358979323846264],dtype='float32')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a==b).all())
def test_packed_numpy(self):
a = np.array([(90, 8010, 3, 14112, 3.14159265358979323846264)], dtype="float32")
invoke = Invocation()
invoke.message = {"double stuffed": [a, a, a]}
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
c = b.get("double stuffed")
for d in c:
self.assertTrue((a == d).all())
def test_packed_numpy(self):
a = np.array([(90,8010,3,14112,3.14159265358979323846264)],dtype='float32')
invoke = Invocation()
invoke.message = {'double stuffed':[a,a,a]}
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
c = b.get('double stuffed')
for d in c:
self.assertTrue((a==d).all())
def test_numpy_codec(self):
a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264],
dtype='float32')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a == b).all())
def test_set(self):
a = {'s':set([1,2,3]),'l':[1,2,3],'t':(1,2,3)}
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
# We only get lists back - damn you msgpack!
only_lists = {'s':set([1,2,3]),'l':[1,2,3],'t':[1,2,3]}
self.assertEquals(only_lists,b)
def test_set(self):
a = {'s': set([1, 2, 3]), 'l': [1, 2, 3], 't': (1, 2, 3)}
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
# We only get lists back - damn you msgpack!
only_lists = {'s': set([1, 2, 3]), 'l': [1, 2, 3], 't': [1, 2, 3]}
self.assertEquals(only_lists, b)
def test_numpy_codec(self):
a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264],
dtype='float32')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a == b).all())
# Rank 1, length 1 works:
a = np.array([90, 8010, 3, 14112, 3.14159265358979323846264],
dtype='float32')
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertTrue((a == b).all())
# Rank 0 array raises Value Error because numpy tolist does not return a list
a = np.array(3.14159265358979323846264, dtype='float32')
invoke = Invocation()
invoke.message = a
self.assertRaises(ValueError, codec.outgoing, invoke)
def test_encode_dotdict(self):
from interface.messages import process_dispatcher_schedule_process_in
msg_obj = process_dispatcher_schedule_process_in()
msg_obj.name = "process"
msg_obj.configuration = {"process": {"property": "FOO"}}
invoke = Invocation()
invoke.message = msg_obj
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
msg_encoded1 = mangled.message
received = encode.incoming(mangled)
msg_rec1 = received.message
msg_obj = process_dispatcher_schedule_process_in()
msg_obj.name = "process"
msg_obj.configuration = DotDict()
msg_obj.configuration.process.property = "FOO"
invoke = Invocation()
invoke.message = msg_obj
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
msg_encoded2 = mangled.message
received = encode.incoming(mangled)
msg_rec2 = received.message
self.assertEquals(msg_encoded1, msg_encoded2)
self.assertIsInstance(msg_rec1["configuration"], dict)
self.assertIsInstance(msg_rec2["configuration"], dict)
def test_dtype(self):
a = np.dtype('float32')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
a = np.dtype('object')
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_dtype(self):
a = np.dtype("float32")
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
a = np.dtype("object")
invoke = Invocation()
invoke.message = a
codec = EncodeInterceptor()
mangled = codec.outgoing(invoke)
received = codec.incoming(mangled)
b = received.message
self.assertEquals(a, b)
def test_dtype(self):
a = np.dtype('float32')
invoke = Invocation()
invoke.message = a
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
received = encode.incoming(mangled)
b = received.message
self.assertEquals(a,b)
a = np.dtype('object')
invoke = Invocation()
invoke.message = a
encode = EncodeInterceptor()
mangled = encode.outgoing(invoke)
received = encode.incoming(mangled)
b = received.message
self.assertEquals(a,b)
def test_perf(self):
_io_serializer = IonObjectSerializer()
_io_deserializer = IonObjectDeserializer(
obj_registry=get_obj_registry())
def time_serialize(test_obj, name="?", has_ion=False):
with time_it(name + ", serialize"):
os = _io_serializer.serialize(test_obj)
with time_it(name + ", deserialize"):
os2 = _io_deserializer.deserialize(os)
count_objs(os)
if has_ion:
test_obj = os
with time_it(name + ", json.dumps"):
oj = json.dumps(test_obj)
with time_it(name + ", json.loads"):
o2 = json.loads(oj)
log.info(" len(json): %s", len(oj))
with time_it(name + ", simplejson.dumps"):
oj = simplejson.dumps(test_obj)
with time_it(name + ", simplejson.loads"):
o2 = simplejson.loads(oj)
log.info(" len(simplejson): %s", len(oj))
with time_it(name + ", msgpack.packb"):
o1 = msgpack.packb(test_obj)
with time_it(name + ", msgpack.unpackb"):
o2 = msgpack.unpackb(o1, use_list=1)
log.info(" len(msgpack): %s", len(o1))
# with time_it(name + ", pickle.dumps"):
# op = pickle.dumps(test_obj)
#
# with time_it(name + ", pickle.loads"):
# o2 = pickle.loads(op)
# log.info(" len(pickle): %s", len(op))
#
# with time_it(name + ", cPickle.dumps"):
# op = cPickle.dumps(test_obj)
#
# with time_it(name + ", cPickle.loads"):
# o2 = cPickle.loads(op)
# log.info(" len(cPickle): %s", len(op))
log.info("----------------")
# Large nested
with time_it("highly nested dict/list, create"):
test_obj = create_test_object(4,
4,
do_list=False,
uvals=True,
ukeys=True)
time_serialize(test_obj, "highly nested dict/list")
# Nested
with time_it("nested dict/list, create"):
test_obj = create_test_object(3,
40,
do_list=True,
uvals=False,
ukeys=False)
time_serialize(test_obj, "nested dict/list")
# Large string
#value = ''.join(random.choice(allowed_chars) for x in xrange(1460000))
value = ''.join(random.choice(allowed_chars) for x in xrange(500000))
time_serialize(value, "long string")
# ION
with time_it("create ion"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=False)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested", has_ion=True)
from pyon.core.interceptor.interceptor import Invocation
from pyon.core.interceptor.encode import EncodeInterceptor
encode = EncodeInterceptor()
invocation = Invocation()
invocation.message = test_obj1
with time_it("ion object, encode"):
encode.outgoing(invocation)
with time_it("ion object, decode"):
encode.incoming(invocation)
count_objs(invocation.message)
# ION
with time_it("create ion unicode"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=False,
uvals=True,
ukeys=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested unicode", has_ion=True)
# Create objects with validation on
with time_it("create ion calidated"):
test_obj1 = create_test_object(2,
200,
do_ion=True,
do_list=False,
do_dict=True,
obj_validate=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested validated", has_ion=True)
def test_perf(self):
_io_serializer = IonObjectSerializer()
_io_deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
def time_serialize(test_obj, name="?", has_ion=False):
with time_it(name + ", serialize"):
os = _io_serializer.serialize(test_obj)
with time_it(name + ", deserialize"):
os2 = _io_deserializer.deserialize(os)
count_objs(os)
if has_ion:
test_obj = os
with time_it(name + ", json.dumps"):
oj = json.dumps(test_obj)
with time_it(name + ", json.loads"):
o2 = json.loads(oj)
log.info(" len(json): %s", len(oj))
with time_it(name + ", simplejson.dumps"):
oj = simplejson.dumps(test_obj)
with time_it(name + ", simplejson.loads"):
o2 = simplejson.loads(oj)
log.info(" len(simplejson): %s", len(oj))
with time_it(name + ", msgpack.packb"):
o1 = msgpack.packb(test_obj)
with time_it(name + ", msgpack.unpackb"):
o2 = msgpack.unpackb(o1, use_list=1)
log.info(" len(msgpack): %s", len(o1))
# with time_it(name + ", pickle.dumps"):
# op = pickle.dumps(test_obj)
#
# with time_it(name + ", pickle.loads"):
# o2 = pickle.loads(op)
# log.info(" len(pickle): %s", len(op))
#
# with time_it(name + ", cPickle.dumps"):
# op = cPickle.dumps(test_obj)
#
# with time_it(name + ", cPickle.loads"):
# o2 = cPickle.loads(op)
# log.info(" len(cPickle): %s", len(op))
log.info("----------------")
# Large nested
with time_it("highly nested dict/list, create"):
test_obj = create_test_object(4, 4, do_list=False, uvals=True, ukeys=True)
time_serialize(test_obj, "highly nested dict/list")
# Nested
with time_it("nested dict/list, create"):
test_obj = create_test_object(3, 40, do_list=True, uvals=False, ukeys=False)
time_serialize(test_obj, "nested dict/list")
# Large string
#value = ''.join(random.choice(allowed_chars) for x in xrange(1460000))
value = ''.join(random.choice(allowed_chars) for x in xrange(500000))
time_serialize(value, "long string")
# ION
with time_it("create ion"):
test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested", has_ion=True)
from pyon.core.interceptor.interceptor import Invocation
from pyon.core.interceptor.encode import EncodeInterceptor
encode = EncodeInterceptor()
invocation = Invocation()
invocation.message = test_obj1
with time_it("ion object, encode"):
encode.outgoing(invocation)
with time_it("ion object, decode"):
encode.incoming(invocation)
count_objs(invocation.message)
# ION
with time_it("create ion unicode"):
test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=False, uvals=True, ukeys=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested unicode", has_ion=True)
# Create objects with validation on
with time_it("create ion calidated"):
test_obj1 = create_test_object(2, 200, do_ion=True, do_list=False, do_dict=True, obj_validate=True)
count_objs(test_obj1)
time_serialize(test_obj1, "dict of ion nested validated", has_ion=True)
