def test_pickle_threshold(): numpy = pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal A = numpy.ones((5, 5)) bufs = serialize_object(A, 1024) assert len(bufs) == 1 B, _ = deserialize_object(bufs) assert_array_equal(A, B) A = numpy.ones((512, 512)) bufs = serialize_object(A, 1024) assert len(bufs) == 2 B, _ = deserialize_object(bufs) assert_array_equal(A, B)
def test_pickle_threshold(): import numpy from numpy.testing.utils import assert_array_equal A = numpy.ones((5, 5)) bufs = serialize_object(A, 1024) nt.assert_equal(len(bufs), 1) B, _ = deserialize_object(bufs) assert_array_equal(A, B) A = numpy.ones((512, 512)) bufs = serialize_object(A, 1024) nt.assert_equal(len(bufs), 2) B, _ = deserialize_object(bufs) assert_array_equal(A, B)
def test_list(): lis = [lambda x:x, 1] bufs = serialize_object(lis) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned, list) l2, r = deserialize_object(bufs) nt.assert_equal(l2[0](l2[1]), lis[0](lis[1]))
def test_list(): lis = [lambda x: x, 1] bufs = serialize_object(lis) canned = pickle.loads(bufs[0]) assert isinstance(canned, list) l2, r = deserialize_object(bufs) assert l2[0](l2[1]) == lis[0](lis[1])
def test_tuple(): tup = (lambda x: x, 1) bufs = serialize_object(tup) canned = pickle.loads(bufs[0]) assert isinstance(canned, tuple) t2, r = deserialize_object(bufs) assert t2[0](t2[1]) == tup[0](tup[1])
def test_tuple(): tup = (lambda x:x, 1) bufs = serialize_object(tup) canned = pickle.loads(bufs[0]) assert isinstance(canned, tuple) t2, r = deserialize_object(bufs) assert t2[0](t2[1]) == tup[0](tup[1])
def test_list(): lis = [lambda x:x, 1] bufs = serialize_object(lis) canned = pickle.loads(bufs[0]) assert isinstance(canned, list) l2, r = deserialize_object(bufs) assert l2[0](l2[1]) == lis[0](lis[1])
def test_tuple(): tup = (lambda x:x, 1) bufs = serialize_object(tup) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned, tuple) t2, r = deserialize_object(bufs) nt.assert_equal(t2[0](t2[1]), tup[0](tup[1]))
def test_roundtrip_buffered(): for obj in [dict(a=b"x" * 1025), b"hello" * 500, [b"hello" * 501, 1, 2, 3]]: bufs = serialize_object(obj) assert len(bufs) == 2 obj2, remainder = deserialize_object(bufs) assert remainder == [] assert obj == obj2
def test_roundtrip_memoryview(): b = b'asdf' * 1025 view = memoryview(b) bufs = serialize_object(view) assert len(bufs) == 2 v2, remainder = deserialize_object(bufs) assert remainder == [] assert v2.tobytes() == b
def test_namedtuple(): p = point(1, 2) bufs = serialize_object(p) canned = pickle.loads(bufs[0]) assert isinstance(canned, point) p2, r = deserialize_object(bufs, globals()) assert p2.x == p.x assert p2.y == p.y
def test_roundtrip_memoryview(): b = b'asdf' * 1025 view = memoryview(b) bufs = serialize_object(view) nt.assert_equal(len(bufs), 2) v2, remainder = deserialize_object(bufs) nt.assert_equal(remainder, []) nt.assert_equal(v2.tobytes(), b)
def test_namedtuple(): p = point(1,2) bufs = serialize_object(p) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned, point) p2, r = deserialize_object(bufs, globals()) nt.assert_equal(p2.x, p.x) nt.assert_equal(p2.y, p.y)
def test_namedtuple(): p = point(1,2) bufs = serialize_object(p) canned = pickle.loads(bufs[0]) assert isinstance(canned, point) p2, r = deserialize_object(bufs, globals()) assert p2.x == p.x assert p2.y == p.y
def test_class(): @interactive class C(object): a=5 bufs = serialize_object(dict(C=C)) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned['C'], CannedClass) d, r = deserialize_object(bufs) C2 = d['C'] nt.assert_equal(C2.a, C.a)
def test_class(): @interactive class C(object): a=5 bufs = serialize_object(dict(C=C)) canned = pickle.loads(bufs[0]) assert isinstance(canned['C'], CannedClass) d, r = deserialize_object(bufs) C2 = d['C'] assert C2.a == C.a
def test_roundtrip_buffered(): for obj in [ dict(a=b"x"*1025), b"hello"*500, [b"hello"*501, 1,2,3] ]: bufs = serialize_object(obj) assert len(bufs) == 2 obj2, remainder = deserialize_object(bufs) assert remainder == [] assert obj == obj2
def test_roundtrip_buffered(): for obj in [ dict(a=b"x"*1025), b"hello"*500, [b"hello"*501, 1,2,3] ]: bufs = serialize_object(obj) nt.assert_equal(len(bufs), 2) obj2, remainder = deserialize_object(bufs) nt.assert_equal(remainder, []) nt.assert_equal(obj, obj2)
def test_class_oldstyle(): @interactive class C: a = 5 bufs = serialize_object(dict(C=C)) canned = pickle.loads(bufs[0]) assert isinstance(canned['C'], CannedClass) d, r = deserialize_object(bufs) C2 = d['C'] assert C2.a == C.a
def test_numpy(): from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) bufs = [memoryview(b) for b in bufs] B, r = deserialize_object(bufs) nt.assert_equal(r, []) nt.assert_equal(A.shape, B.shape) nt.assert_equal(A.dtype, B.dtype) assert_array_equal(A,B)
def test_numpy(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) bufs = [memoryview(b) for b in bufs] B, r = deserialize_object(bufs) assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A, B)
def test_numpy(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) bufs = [memoryview(b) for b in bufs] B, r = deserialize_object(bufs) assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A,B)
def _queue_management_worker(self): """ TODO: docstring """ logger.debug("[MTHREAD] queue management worker starting") while True: task_id, buf = self.incoming_q.get() # TODO: why does this hang? msg = deserialize_object(buf)[0] # TODO: handle exceptions task_fut = self.tasks[task_id] logger.debug("Got response for task id {}".format(task_id)) if "result" in msg: task_fut.set_result(msg["result"]) elif "exception" in msg: # TODO: handle exception pass elif 'exception' in msg: logger.warning("Task: {} has returned with an exception") try: s, _ = deserialize_object(msg['exception']) exception = ValueError( "Remote exception description: {}".format(s)) task_fut.set_exception(exception) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result nor exception") if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")
def WorkQueueCollectorThread(collector_queue=multiprocessing.Queue(), tasks={}, tasks_lock=threading.Lock(), cancel_value=multiprocessing.Value('i', 1), submit_process=None, executor=None): logger.debug("Starting Collector Thread") continue_running = True while continue_running: if cancel_value.value == 0: continue_running = False continue # The WorkQueue process that creates task has died if not submit_process.is_alive() and cancel_value.value != 0: raise ExecutorError(executor, "Workqueue Submit Process is not alive") # Get the result message from the collector_queue try: item = collector_queue.get(timeout=1) except queue.Empty: continue parsl_tid = item["tid"] received = item["result_received"] # Obtain the future from the tasks dictionary tasks_lock.acquire() future = tasks[parsl_tid] tasks_lock.release() # Failed task if received is False: reason = item["reason"] status = item["status"] future.set_exception(AppFailure(reason, status)) # Successful task else: result = item["result"] future_update, _ = deserialize_object(result["result"]) logger.debug("Updating Future for Parsl Task {}".format(parsl_tid)) if result["failure"] is False: future.set_result(future_update) else: future.set_exception(RemoteExceptionWrapper(*future_update)) logger.debug("Exiting Collector Thread") return
def test_numpy_in_seq(): from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object((A,1,2,b'hello')) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned[0], CannedArray) tup, r = deserialize_object(bufs) B = tup[0] nt.assert_equal(r, []) nt.assert_equal(A.shape, B.shape) nt.assert_equal(A.dtype, B.dtype) assert_array_equal(A,B)
def test_numpy_in_dict(): from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(dict(a=A,b=1,c=range(20))) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned['a'], CannedArray) d, r = deserialize_object(bufs) B = d['a'] nt.assert_equal(r, []) nt.assert_equal(A.shape, B.shape) nt.assert_equal(A.dtype, B.dtype) assert_array_equal(A,B)
def test_numpy_in_seq(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object((A, 1, 2, b'hello')) canned = pickle.loads(bufs[0]) assert isinstance(canned[0], CannedArray) tup, r = deserialize_object(bufs) B = tup[0] assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A, B)
def test_numpy_in_dict(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(dict(a=A,b=1,c=range(20))) canned = pickle.loads(bufs[0]) assert isinstance(canned['a'], CannedArray) d, r = deserialize_object(bufs) B = d['a'] assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A,B)
def test_numpy_in_dict(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object(dict(a=A, b=1, c=range(20))) canned = pickle.loads(bufs[0]) assert isinstance(canned['a'], CannedArray) d, r = deserialize_object(bufs) B = d['a'] assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A, B)
def test_recarray(): from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in [ [('f', float), ('s', '|S10')], [('n', int), ('s', '|S1'), ('u', 'uint32')], ]: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) B, r = deserialize_object(bufs) nt.assert_equal(r, []) nt.assert_equal(A.shape, B.shape) nt.assert_equal(A.dtype, B.dtype) assert_array_equal(A,B)
def test_numpy_in_seq(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in DTYPES: A = new_array(shape, dtype=dtype) bufs = serialize_object((A,1,2,b'hello')) canned = pickle.loads(bufs[0]) assert isinstance(canned[0], CannedArray) tup, r = deserialize_object(bufs) B = tup[0] assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A,B)
def collect_data(session, msg_frames): """Collect and deserialize messages""" from ipyparallel import serialize global LATEST_DATA idents, msg = session.feed_identities(msg_frames) try: msg = session.deserialize(msg, content=True) except Exception as e: print(e) return if msg['header']['msg_type'] != 'data_message': return # show the contents of data messages: data, remainder = serialize.deserialize_object(msg['buffers']) LATEST_DATA[data['engine_id']] = data
def test_recarray(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in [ [('f', float), ('s', '|S10')], [('n', int), ('s', '|S1'), ('u', 'uint32')], ]: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) B, r = deserialize_object(bufs) assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A,B)
def test_class_inheritance(): @interactive class C(object): a=5 @interactive class D(C): b=10 bufs = serialize_object(dict(D=D)) canned = pickle.loads(bufs[0]) assert isinstance(canned['D'], CannedClass) d, r = deserialize_object(bufs) D2 = d['D'] assert D2.a == D.a assert D2.b == D.b
def collect_data(session, msg_frames): """Collect and deserialize messages""" from ipyparallel import serialize global LATEST_DATA idents, msg = session.feed_identities(msg_frames) try: msg = session.deserialize(msg, content=True) except Exception as e: print(e) return if msg["header"]["msg_type"] != "data_message": return # show the contents of data messages: data, remainder = serialize.deserialize_object(msg["buffers"]) LATEST_DATA[data["engine_id"]] = data
def test_class_inheritance(): @interactive class C(object): a=5 @interactive class D(C): b=10 bufs = serialize_object(dict(D=D)) canned = pickle.loads(bufs[0]) nt.assert_is_instance(canned['D'], CannedClass) d, r = deserialize_object(bufs) D2 = d['D'] nt.assert_equal(D2.a, D.a) nt.assert_equal(D2.b, D.b)
def test_recarray(): pytest.importorskip('numpy') from numpy.testing.utils import assert_array_equal for shape in SHAPES: for dtype in [ [('f', float), ('s', '|S10')], [('n', int), ('s', '|S1'), ('u', 'uint32')], ]: A = new_array(shape, dtype=dtype) bufs = serialize_object(A) B, r = deserialize_object(bufs) assert r == [] assert A.shape == B.shape assert A.dtype == B.dtype assert_array_equal(A, B)
def test_class_inheritance(): @interactive class C: a = 5 @interactive class D(C): b = 10 bufs = serialize_object(dict(D=D)) canned = pickle.loads(bufs[0]) assert isinstance(canned['D'], CannedClass) d, r = deserialize_object(bufs) D2 = d['D'] assert D2.a == D.a assert D2.b == D.b
def _queue_management_worker(self): """Listen to the queue for task status messages and handle them. Depending on the message, tasks will be updated with results, exceptions, or updates. It expects the following messages: .. code:: python { "task_id" : <task_id> "result" : serialized result object, if task succeeded ... more tags could be added later } { "task_id" : <task_id> "exception" : serialized exception object, on failure } We do not support these yet, but they could be added easily. .. code:: python { "task_id" : <task_id> "cpu_stat" : <> "mem_stat" : <> "io_stat" : <> "started" : tstamp } The `None` message is a die request. """ logger.debug("[MTHREAD] queue management worker starting") while not self.bad_state_is_set: try: msgs = self.incoming_q.get(timeout=1) except queue.Empty: logger.debug("[MTHREAD] queue empty") # Timed out. pass except IOError as e: logger.exception( "[MTHREAD] Caught broken queue with exception code {}: {}". format(e.errno, e)) return except Exception as e: logger.exception( "[MTHREAD] Caught unknown exception: {}".format(e)) return else: if msgs is None: logger.debug("[MTHREAD] Got None, exiting") return else: for serialized_msg in msgs: try: msg = pickle.loads(serialized_msg) tid = msg['task_id'] except pickle.UnpicklingError: raise BadMessage( "Message received could not be unpickled") except Exception: raise BadMessage( "Message received does not contain 'task_id' field" ) if tid == -1 and 'exception' in msg: logger.warning( "Executor shutting down due to exception from interchange" ) exception, _ = deserialize_object(msg['exception']) self.set_bad_state_and_fail_all(exception) break task_fut = self.tasks[tid] if 'result' in msg: result, _ = deserialize_object(msg['result']) task_fut.set_result(result) elif 'exception' in msg: try: s, _ = deserialize_object(msg['exception']) # s should be a RemoteExceptionWrapper... so we can reraise it if isinstance(s, RemoteExceptionWrapper): try: s.reraise() except Exception as e: task_fut.set_exception(e) elif isinstance(s, Exception): task_fut.set_exception(s) else: raise ValueError( "Unknown exception-like type received: {}" .format(type(s))) except Exception as e: # TODO could be a proper wrapped exception? task_fut.set_exception( DeserializationError( "Received exception, but handling also threw an exception: {}" .format(e))) else: raise BadMessage( "Message received is neither result or exception" ) if not self.is_alive: break logger.info("[MTHREAD] queue management worker finished")
def deserialize(self, msg): """inverse of serialize""" return deserialize_object(msg)[0]
def runner(incoming_q, outgoing_q): ''' This is a function that mocks the Swift-T side. It listens on the the incoming_q for tasks and posts returns on the outgoing_q Args: - incoming_q (Queue object) : The queue to listen on - outgoing_q (Queue object) : Queue to post results on The messages posted on the incoming_q will be of the form : .. code:: python { "task_id" : <uuid.uuid4 string>, "buffer" : serialized buffer containing the fn, args and kwargs } If ``None`` is received, the runner will exit. Response messages should be of the form: .. code:: python { "task_id" : <uuid.uuid4 string>, "result" : serialized buffer containing result "exception" : serialized exception object } On exiting the runner will post ``None`` to the outgoing_q ''' logger.debug("[RUNNER] Starting") def execute_task(bufs): ''' Deserialize the buf, and execute the task. Returns the serialized result/exception ''' all_names = dir(__builtins__) user_ns = locals() user_ns.update( {'__builtins__': {k: getattr(__builtins__, k) for k in all_names}}) f, args, kwargs = unpack_apply_message(bufs, user_ns, copy=False) fname = getattr(f, '__name__', 'f') prefix = "parsl_" fname = prefix + "f" argname = prefix + "args" kwargname = prefix + "kwargs" resultname = prefix + "result" user_ns.update({ fname: f, argname: args, kwargname: kwargs, resultname: resultname }) code = "{0} = {1}(*{2}, **{3})".format(resultname, fname, argname, kwargname) try: print("[RUNNER] Executing : {0}".format(code)) exec(code, user_ns, user_ns) except Exception as e: logger.warning("Caught errors but will not handled %s", e) raise e else: # print("Done : {0}".format(locals())) print("[RUNNER] Result : {0}".format(user_ns.get(resultname))) return user_ns.get(resultname) while True: try: # Blocking wait on the queue msg = incoming_q.get(block=True, timeout=10) # logger.debug("[RUNNER] Got message : %s", msg) except queue.Empty: # Handle case where no items were on queue logger.debug("[RUNNER] got nothing") except IOError as ioerror: logger.debug("[RUNNER] broken pipe, error: %s", ioerror) try: # Attempt to send a stop notification to the management thread outgoing_q.put(None) except Exception: pass break except Exception as e: logger.debug("[RUNNER] caught unknown exception : %s", e) else: # Handle received message if not msg: # Empty message is a die request logger.debug("[RUNNER] Received exit request") outgoing_q.put(None) break else: # Received a valid message, handle it logger.debug("[RUNNER] Got a valid task : %s", msg["task_id"]) try: response_obj = execute_task(msg['buffer']) response = { "task_id": msg["task_id"], "result": serialize_object(response_obj) } logger.warning("[RUNNER] Returing result : %s", deserialize_object(response["result"])) except Exception as e: logger.debug("[RUNNER] Caught task exception") response = { "task_id": msg["task_id"], "exception": serialize_object(e) } outgoing_q.put(response) logger.debug("[RUNNER] Terminating")
def roundtrip(obj): """roundtrip an object through serialization""" bufs = serialize_object(obj) obj2, remainder = deserialize_object(bufs) nt.assert_equals(remainder, []) return obj2
def roundtrip(obj): """roundtrip an object through serialization""" bufs = serialize_object(obj) obj2, remainder = deserialize_object(bufs) assert remainder == [] return obj2