def test_pickle_threshold():
numpy = pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
A = numpy.ones((5, 5))
bufs = serialize_object(A, 1024)
assert len(bufs) == 1
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
A = numpy.ones((512, 512))
bufs = serialize_object(A, 1024)
assert len(bufs) == 2
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
def test_pickle_threshold():
import numpy
from numpy.testing.utils import assert_array_equal
A = numpy.ones((5, 5))
bufs = serialize_object(A, 1024)
nt.assert_equal(len(bufs), 1)
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
A = numpy.ones((512, 512))
bufs = serialize_object(A, 1024)
nt.assert_equal(len(bufs), 2)
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
def test_pickle_threshold():
numpy = pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
A = numpy.ones((5, 5))
bufs = serialize_object(A, 1024)
assert len(bufs) == 1
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
A = numpy.ones((512, 512))
bufs = serialize_object(A, 1024)
assert len(bufs) == 2
B, _ = deserialize_object(bufs)
assert_array_equal(A, B)
def test_list():
lis = [lambda x:x, 1]
bufs = serialize_object(lis)
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned, list)
l2, r = deserialize_object(bufs)
nt.assert_equal(l2[0](l2[1]), lis[0](lis[1]))
def test_list():
lis = [lambda x: x, 1]
bufs = serialize_object(lis)
canned = pickle.loads(bufs[0])
assert isinstance(canned, list)
l2, r = deserialize_object(bufs)
assert l2[0](l2[1]) == lis[0](lis[1])
def test_tuple():
tup = (lambda x: x, 1)
bufs = serialize_object(tup)
canned = pickle.loads(bufs[0])
assert isinstance(canned, tuple)
t2, r = deserialize_object(bufs)
assert t2[0](t2[1]) == tup[0](tup[1])
def test_tuple():
tup = (lambda x:x, 1)
bufs = serialize_object(tup)
canned = pickle.loads(bufs[0])
assert isinstance(canned, tuple)
t2, r = deserialize_object(bufs)
assert t2[0](t2[1]) == tup[0](tup[1])
def test_list():
lis = [lambda x:x, 1]
bufs = serialize_object(lis)
canned = pickle.loads(bufs[0])
assert isinstance(canned, list)
l2, r = deserialize_object(bufs)
assert l2[0](l2[1]) == lis[0](lis[1])
def test_tuple():
tup = (lambda x:x, 1)
bufs = serialize_object(tup)
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned, tuple)
t2, r = deserialize_object(bufs)
nt.assert_equal(t2[0](t2[1]), tup[0](tup[1]))
def test_roundtrip_buffered():
for obj in [dict(a=b"x" * 1025), b"hello" * 500, [b"hello" * 501, 1, 2, 3]]:
bufs = serialize_object(obj)
assert len(bufs) == 2
obj2, remainder = deserialize_object(bufs)
assert remainder == []
assert obj == obj2
def test_roundtrip_memoryview():
b = b'asdf' * 1025
view = memoryview(b)
bufs = serialize_object(view)
assert len(bufs) == 2
v2, remainder = deserialize_object(bufs)
assert remainder == []
assert v2.tobytes() == b
def test_namedtuple():
p = point(1, 2)
bufs = serialize_object(p)
canned = pickle.loads(bufs[0])
assert isinstance(canned, point)
p2, r = deserialize_object(bufs, globals())
assert p2.x == p.x
assert p2.y == p.y
def test_roundtrip_memoryview():
b = b'asdf' * 1025
view = memoryview(b)
bufs = serialize_object(view)
nt.assert_equal(len(bufs), 2)
v2, remainder = deserialize_object(bufs)
nt.assert_equal(remainder, [])
nt.assert_equal(v2.tobytes(), b)
def test_roundtrip_memoryview():
b = b'asdf' * 1025
view = memoryview(b)
bufs = serialize_object(view)
assert len(bufs) == 2
v2, remainder = deserialize_object(bufs)
assert remainder == []
assert v2.tobytes() == b
def test_namedtuple():
p = point(1,2)
bufs = serialize_object(p)
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned, point)
p2, r = deserialize_object(bufs, globals())
nt.assert_equal(p2.x, p.x)
nt.assert_equal(p2.y, p.y)
def test_namedtuple():
p = point(1,2)
bufs = serialize_object(p)
canned = pickle.loads(bufs[0])
assert isinstance(canned, point)
p2, r = deserialize_object(bufs, globals())
assert p2.x == p.x
assert p2.y == p.y
def test_class():
@interactive
class C(object):
a=5
bufs = serialize_object(dict(C=C))
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned['C'], CannedClass)
d, r = deserialize_object(bufs)
C2 = d['C']
nt.assert_equal(C2.a, C.a)
def test_class():
@interactive
class C(object):
a=5
bufs = serialize_object(dict(C=C))
canned = pickle.loads(bufs[0])
assert isinstance(canned['C'], CannedClass)
d, r = deserialize_object(bufs)
C2 = d['C']
assert C2.a == C.a
def test_roundtrip_buffered():
for obj in [
dict(a=b"x"*1025),
b"hello"*500,
[b"hello"*501, 1,2,3]
]:
bufs = serialize_object(obj)
assert len(bufs) == 2
obj2, remainder = deserialize_object(bufs)
assert remainder == []
assert obj == obj2
def test_roundtrip_buffered():
for obj in [
dict(a=b"x"*1025),
b"hello"*500,
[b"hello"*501, 1,2,3]
]:
bufs = serialize_object(obj)
nt.assert_equal(len(bufs), 2)
obj2, remainder = deserialize_object(bufs)
nt.assert_equal(remainder, [])
nt.assert_equal(obj, obj2)
def test_class_oldstyle():
@interactive
class C:
a = 5
bufs = serialize_object(dict(C=C))
canned = pickle.loads(bufs[0])
assert isinstance(canned['C'], CannedClass)
d, r = deserialize_object(bufs)
C2 = d['C']
assert C2.a == C.a
def test_numpy():
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
bufs = [memoryview(b) for b in bufs]
B, r = deserialize_object(bufs)
nt.assert_equal(r, [])
nt.assert_equal(A.shape, B.shape)
nt.assert_equal(A.dtype, B.dtype)
assert_array_equal(A,B)
def test_numpy():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
bufs = [memoryview(b) for b in bufs]
B, r = deserialize_object(bufs)
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A, B)
def test_numpy():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
bufs = [memoryview(b) for b in bufs]
B, r = deserialize_object(bufs)
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A,B)
def _queue_management_worker(self):
""" TODO: docstring """
logger.debug("[MTHREAD] queue management worker starting")
while True:
task_id, buf = self.incoming_q.get() # TODO: why does this hang?
msg = deserialize_object(buf)[0]
# TODO: handle exceptions
task_fut = self.tasks[task_id]
logger.debug("Got response for task id {}".format(task_id))
if "result" in msg:
task_fut.set_result(msg["result"])
elif "exception" in msg:
# TODO: handle exception
pass
elif 'exception' in msg:
logger.warning("Task: {} has returned with an exception")
try:
s, _ = deserialize_object(msg['exception'])
exception = ValueError(
"Remote exception description: {}".format(s))
task_fut.set_exception(exception)
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result nor exception")
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
def WorkQueueCollectorThread(collector_queue=multiprocessing.Queue(),
tasks={},
tasks_lock=threading.Lock(),
cancel_value=multiprocessing.Value('i', 1),
submit_process=None,
executor=None):
logger.debug("Starting Collector Thread")
continue_running = True
while continue_running:
if cancel_value.value == 0:
continue_running = False
continue
# The WorkQueue process that creates task has died
if not submit_process.is_alive() and cancel_value.value != 0:
raise ExecutorError(executor,
"Workqueue Submit Process is not alive")
# Get the result message from the collector_queue
try:
item = collector_queue.get(timeout=1)
except queue.Empty:
continue
parsl_tid = item["tid"]
received = item["result_received"]
# Obtain the future from the tasks dictionary
tasks_lock.acquire()
future = tasks[parsl_tid]
tasks_lock.release()
# Failed task
if received is False:
reason = item["reason"]
status = item["status"]
future.set_exception(AppFailure(reason, status))
# Successful task
else:
result = item["result"]
future_update, _ = deserialize_object(result["result"])
logger.debug("Updating Future for Parsl Task {}".format(parsl_tid))
if result["failure"] is False:
future.set_result(future_update)
else:
future.set_exception(RemoteExceptionWrapper(*future_update))
logger.debug("Exiting Collector Thread")
return
def test_numpy_in_seq():
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object((A,1,2,b'hello'))
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned[0], CannedArray)
tup, r = deserialize_object(bufs)
B = tup[0]
nt.assert_equal(r, [])
nt.assert_equal(A.shape, B.shape)
nt.assert_equal(A.dtype, B.dtype)
assert_array_equal(A,B)
def test_numpy_in_dict():
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(dict(a=A,b=1,c=range(20)))
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned['a'], CannedArray)
d, r = deserialize_object(bufs)
B = d['a']
nt.assert_equal(r, [])
nt.assert_equal(A.shape, B.shape)
nt.assert_equal(A.dtype, B.dtype)
assert_array_equal(A,B)
def test_numpy_in_seq():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object((A, 1, 2, b'hello'))
canned = pickle.loads(bufs[0])
assert isinstance(canned[0], CannedArray)
tup, r = deserialize_object(bufs)
B = tup[0]
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A, B)
def test_numpy_in_dict():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(dict(a=A,b=1,c=range(20)))
canned = pickle.loads(bufs[0])
assert isinstance(canned['a'], CannedArray)
d, r = deserialize_object(bufs)
B = d['a']
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A,B)
def test_numpy_in_dict():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(dict(a=A, b=1, c=range(20)))
canned = pickle.loads(bufs[0])
assert isinstance(canned['a'], CannedArray)
d, r = deserialize_object(bufs)
B = d['a']
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A, B)
def test_recarray():
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in [
[('f', float), ('s', '|S10')],
[('n', int), ('s', '|S1'), ('u', 'uint32')],
]:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
B, r = deserialize_object(bufs)
nt.assert_equal(r, [])
nt.assert_equal(A.shape, B.shape)
nt.assert_equal(A.dtype, B.dtype)
assert_array_equal(A,B)
def test_numpy_in_seq():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in DTYPES:
A = new_array(shape, dtype=dtype)
bufs = serialize_object((A,1,2,b'hello'))
canned = pickle.loads(bufs[0])
assert isinstance(canned[0], CannedArray)
tup, r = deserialize_object(bufs)
B = tup[0]
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A,B)
def collect_data(session, msg_frames):
"""Collect and deserialize messages"""
from ipyparallel import serialize
global LATEST_DATA
idents, msg = session.feed_identities(msg_frames)
try:
msg = session.deserialize(msg, content=True)
except Exception as e:
print(e)
return
if msg['header']['msg_type'] != 'data_message':
return
# show the contents of data messages:
data, remainder = serialize.deserialize_object(msg['buffers'])
LATEST_DATA[data['engine_id']] = data
def test_recarray():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in [
[('f', float), ('s', '|S10')],
[('n', int), ('s', '|S1'), ('u', 'uint32')],
]:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
B, r = deserialize_object(bufs)
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A,B)
def test_class_inheritance():
@interactive
class C(object):
a=5
@interactive
class D(C):
b=10
bufs = serialize_object(dict(D=D))
canned = pickle.loads(bufs[0])
assert isinstance(canned['D'], CannedClass)
d, r = deserialize_object(bufs)
D2 = d['D']
assert D2.a == D.a
assert D2.b == D.b
def collect_data(session, msg_frames):
"""Collect and deserialize messages"""
from ipyparallel import serialize
global LATEST_DATA
idents, msg = session.feed_identities(msg_frames)
try:
msg = session.deserialize(msg, content=True)
except Exception as e:
print(e)
return
if msg["header"]["msg_type"] != "data_message":
return
# show the contents of data messages:
data, remainder = serialize.deserialize_object(msg["buffers"])
LATEST_DATA[data["engine_id"]] = data
def test_class_inheritance():
@interactive
class C(object):
a=5
@interactive
class D(C):
b=10
bufs = serialize_object(dict(D=D))
canned = pickle.loads(bufs[0])
nt.assert_is_instance(canned['D'], CannedClass)
d, r = deserialize_object(bufs)
D2 = d['D']
nt.assert_equal(D2.a, D.a)
nt.assert_equal(D2.b, D.b)
def test_recarray():
pytest.importorskip('numpy')
from numpy.testing.utils import assert_array_equal
for shape in SHAPES:
for dtype in [
[('f', float), ('s', '|S10')],
[('n', int), ('s', '|S1'), ('u', 'uint32')],
]:
A = new_array(shape, dtype=dtype)
bufs = serialize_object(A)
B, r = deserialize_object(bufs)
assert r == []
assert A.shape == B.shape
assert A.dtype == B.dtype
assert_array_equal(A, B)
def test_class_inheritance():
@interactive
class C:
a = 5
@interactive
class D(C):
b = 10
bufs = serialize_object(dict(D=D))
canned = pickle.loads(bufs[0])
assert isinstance(canned['D'], CannedClass)
d, r = deserialize_object(bufs)
D2 = d['D']
assert D2.a == D.a
assert D2.b == D.b
def _queue_management_worker(self):
"""Listen to the queue for task status messages and handle them.
Depending on the message, tasks will be updated with results, exceptions,
or updates. It expects the following messages:
.. code:: python
{
"task_id" : <task_id>
"result" : serialized result object, if task succeeded
... more tags could be added later
}
{
"task_id" : <task_id>
"exception" : serialized exception object, on failure
}
We do not support these yet, but they could be added easily.
.. code:: python
{
"task_id" : <task_id>
"cpu_stat" : <>
"mem_stat" : <>
"io_stat" : <>
"started" : tstamp
}
The `None` message is a die request.
"""
logger.debug("[MTHREAD] queue management worker starting")
while not self.bad_state_is_set:
try:
msgs = self.incoming_q.get(timeout=1)
except queue.Empty:
logger.debug("[MTHREAD] queue empty")
# Timed out.
pass
except IOError as e:
logger.exception(
"[MTHREAD] Caught broken queue with exception code {}: {}".
format(e.errno, e))
return
except Exception as e:
logger.exception(
"[MTHREAD] Caught unknown exception: {}".format(e))
return
else:
if msgs is None:
logger.debug("[MTHREAD] Got None, exiting")
return
else:
for serialized_msg in msgs:
try:
msg = pickle.loads(serialized_msg)
tid = msg['task_id']
except pickle.UnpicklingError:
raise BadMessage(
"Message received could not be unpickled")
except Exception:
raise BadMessage(
"Message received does not contain 'task_id' field"
)
if tid == -1 and 'exception' in msg:
logger.warning(
"Executor shutting down due to exception from interchange"
)
exception, _ = deserialize_object(msg['exception'])
self.set_bad_state_and_fail_all(exception)
break
task_fut = self.tasks[tid]
if 'result' in msg:
result, _ = deserialize_object(msg['result'])
task_fut.set_result(result)
elif 'exception' in msg:
try:
s, _ = deserialize_object(msg['exception'])
# s should be a RemoteExceptionWrapper... so we can reraise it
if isinstance(s, RemoteExceptionWrapper):
try:
s.reraise()
except Exception as e:
task_fut.set_exception(e)
elif isinstance(s, Exception):
task_fut.set_exception(s)
else:
raise ValueError(
"Unknown exception-like type received: {}"
.format(type(s)))
except Exception as e:
# TODO could be a proper wrapped exception?
task_fut.set_exception(
DeserializationError(
"Received exception, but handling also threw an exception: {}"
.format(e)))
else:
raise BadMessage(
"Message received is neither result or exception"
)
if not self.is_alive:
break
logger.info("[MTHREAD] queue management worker finished")
def deserialize(self, msg):
"""inverse of serialize"""
return deserialize_object(msg)[0]
def runner(incoming_q, outgoing_q):
''' This is a function that mocks the Swift-T side. It listens on the the
incoming_q for tasks and posts returns on the outgoing_q
Args:
- incoming_q (Queue object) : The queue to listen on
- outgoing_q (Queue object) : Queue to post results on
The messages posted on the incoming_q will be of the form :
.. code:: python
{
"task_id" : <uuid.uuid4 string>,
"buffer" : serialized buffer containing the fn, args and kwargs
}
If ``None`` is received, the runner will exit.
Response messages should be of the form:
.. code:: python
{
"task_id" : <uuid.uuid4 string>,
"result" : serialized buffer containing result
"exception" : serialized exception object
}
On exiting the runner will post ``None`` to the outgoing_q
'''
logger.debug("[RUNNER] Starting")
def execute_task(bufs):
''' Deserialize the buf, and execute the task.
Returns the serialized result/exception
'''
all_names = dir(__builtins__)
user_ns = locals()
user_ns.update(
{'__builtins__': {k: getattr(__builtins__, k)
for k in all_names}})
f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False)
fname = getattr(f, '__name__', 'f')
prefix = "parsl_"
fname = prefix + "f"
argname = prefix + "args"
kwargname = prefix + "kwargs"
resultname = prefix + "result"
user_ns.update({
fname: f,
argname: args,
kwargname: kwargs,
resultname: resultname
})
code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname,
kwargname)
try:
print("[RUNNER] Executing : {0}".format(code))
exec(code, user_ns, user_ns)
except Exception as e:
logger.warning("Caught errors but will not handled %s", e)
raise e
else:
# print("Done : {0}".format(locals()))
print("[RUNNER] Result : {0}".format(user_ns.get(resultname)))
return user_ns.get(resultname)
while True:
try:
# Blocking wait on the queue
msg = incoming_q.get(block=True, timeout=10)
# logger.debug("[RUNNER] Got message : %s", msg)
except queue.Empty:
# Handle case where no items were on queue
logger.debug("[RUNNER] got nothing")
except IOError as ioerror:
logger.debug("[RUNNER] broken pipe, error: %s", ioerror)
try:
# Attempt to send a stop notification to the management thread
outgoing_q.put(None)
except Exception:
pass
break
except Exception as e:
logger.debug("[RUNNER] caught unknown exception : %s", e)
else:
# Handle received message
if not msg:
# Empty message is a die request
logger.debug("[RUNNER] Received exit request")
outgoing_q.put(None)
break
else:
# Received a valid message, handle it
logger.debug("[RUNNER] Got a valid task : %s", msg["task_id"])
try:
response_obj = execute_task(msg['buffer'])
response = {
"task_id": msg["task_id"],
"result": serialize_object(response_obj)
}
logger.warning("[RUNNER] Returing result : %s",
deserialize_object(response["result"]))
except Exception as e:
logger.debug("[RUNNER] Caught task exception")
response = {
"task_id": msg["task_id"],
"exception": serialize_object(e)
}
outgoing_q.put(response)
logger.debug("[RUNNER] Terminating")
def roundtrip(obj):
"""roundtrip an object through serialization"""
bufs = serialize_object(obj)
obj2, remainder = deserialize_object(bufs)
nt.assert_equals(remainder, [])
return obj2
def roundtrip(obj):
"""roundtrip an object through serialization"""
bufs = serialize_object(obj)
obj2, remainder = deserialize_object(bufs)
assert remainder == []
return obj2
