def test_tracker(self):
m = zmq.Frame(b'asdf', track=True)
self.assertFalse(m.tracker.done)
pm = zmq.MessageTracker(m)
self.assertFalse(pm.done)
del m
self.assertTrue(pm.done)
def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None):
"""
Partition a Python sequence and send the partitions to a set of engines.
"""
block = block if block is not None else self.block
track = track if track is not None else self.track
targets = targets if targets is not None else self.targets
mapObject = Map.dists[dist]()
nparts = len(targets)
msg_ids = []
trackers = []
for index, engineid in enumerate(targets):
partition = mapObject.getPartition(seq, index, nparts)
if flatten and len(partition) == 1:
ns = {key: partition[0]}
else:
ns = {key: partition}
r = self.push(ns, block=False, track=track, targets=engineid)
msg_ids.extend(r.msg_ids)
if track:
trackers.append(r._tracker)
if track:
tracker = zmq.MessageTracker(*trackers)
else:
tracker = None
r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker)
if block:
r.wait()
else:
return r
def test_tracker(self):
m = zmq.Message('asdf'.encode(), track=True)
self.assertFalse(m.done)
pm = zmq.MessageTracker(m)
self.assertFalse(pm.done)
del m
self.assertTrue(pm.done)
def test_tracker(self):
m = zmq.Message(asbytes('asdf'), track=True)
self.assertFalse(m.tracker.done)
pm = zmq.MessageTracker(m)
self.assertFalse(pm.done)
del m
self.assertTrue(pm.done)
def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None):
"""calls f(*args, **kwargs) on remote engines, returning the result.
This method sets all of `apply`'s flags via this View's attributes.
Parameters
----------
f : callable
args : list [default: empty]
kwargs : dict [default: empty]
targets : target list [default: self.targets]
where to run
block : bool [default: self.block]
whether to block
track : bool [default: self.track]
whether to ask zmq to track the message, for safe non-copying sends
Returns
-------
if self.block is False:
returns AsyncResult
else:
returns actual result of f(*args, **kwargs) on the engine(s)
This will be a list of self.targets is also a list (even length 1), or
the single result if self.targets is an integer engine id
"""
args = [] if args is None else args
kwargs = {} if kwargs is None else kwargs
block = self.block if block is None else block
track = self.track if track is None else track
targets = self.targets if targets is None else targets
_idents, _targets = self.client._build_targets(targets)
msg_ids = []
trackers = []
for ident in _idents:
msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track,
ident=ident)
if track:
trackers.append(msg['tracker'])
msg_ids.append(msg['header']['msg_id'])
if isinstance(targets, int):
msg_ids = msg_ids[0]
tracker = None if track is False else zmq.MessageTracker(*trackers)
ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets,
tracker=tracker, owner=True,
)
if block:
try:
return ar.get()
except KeyboardInterrupt:
pass
return ar
def test_tracker(self):
m = zmq.Frame(b'asdf', track=True)
self.assertFalse(m.tracker.done)
pm = zmq.MessageTracker(m)
self.assertFalse(pm.done)
del m
for i in range(10):
if pm.done:
break
time.sleep(0.1)
self.assertTrue(pm.done)
def test_multi_tracker(self):
m = zmq.Frame(b'asdf', track=True)
m2 = zmq.Frame(b'whoda', track=True)
mt = zmq.MessageTracker(m, m2)
self.assertFalse(m.tracker.done)
self.assertFalse(mt.done)
self.assertRaises(zmq.NotDone, mt.wait, 0.1)
del m
time.sleep(0.1)
self.assertRaises(zmq.NotDone, mt.wait, 0.1)
self.assertFalse(mt.done)
del m2
self.assertTrue(mt.wait() is None)
self.assertTrue(mt.done)
def test_multi_tracker(self):
m = zmq.Frame(b'asdf', copy=False, track=True)
m2 = zmq.Frame(b'whoda', copy=False, track=True)
mt = zmq.MessageTracker(m, m2)
self.assertFalse(m.tracker.done)
self.assertFalse(mt.done)
self.assertRaises(zmq.NotDone, mt.wait, 0.1)
del m
for i in range(3):
gc.collect()
self.assertRaises(zmq.NotDone, mt.wait, 0.1)
self.assertFalse(mt.done)
del m2
for i in range(3):
gc.collect()
assert mt.wait(0.1) is None
assert mt.done
def send(self, message, flags=0, copy=False, track=False):
if isinstance(message, unicode):
raise TypeError("Message must be in bytes, not an unicode Object")
if isinstance(message, Frame):
message = message.bytes
zmq_msg = ffi.new('zmq_msg_t*')
c_message = ffi.new('char[]', message)
rc = C.zmq_msg_init_size(zmq_msg, len(message))
C.memcpy(C.zmq_msg_data(zmq_msg), c_message, len(message))
rc = C.zmq_msg_send(zmq_msg, self._zmq_socket, flags)
C.zmq_msg_close(zmq_msg)
_check_rc(rc)
if track:
return zmq.MessageTracker()
def __init__(self, data, track=False):
try:
view(data)
except TypeError:
raise
self._data = data
if isinstance(data, unicode):
raise TypeError("Unicode objects not allowed. Only: str/bytes, " +
"buffer interfaces.")
self.more = False
self.tracker = None
self.closed = False
if track:
self.tracker = zmq.MessageTracker()
self.buffer = view(self.bytes)
def send(self, message, flags=0, copy=False, track=False):
if isinstance(message, unicode):
raise TypeError("Message must be in bytes, not an unicode Object")
if isinstance(message, Frame):
message = message.bytes
zmq_msg = ffi.new('zmq_msg_t*')
if not isinstance(message, bytes):
# cast any bufferable data to bytes via memoryview
message = memoryview(message).tobytes()
c_message = ffi.new('char[]', message)
rc = C.zmq_msg_init_size(zmq_msg, len(message))
_check_rc(rc)
C.memcpy(C.zmq_msg_data(zmq_msg), c_message, len(message))
_retry_sys_call(C.zmq_msg_send, zmq_msg, self._zmq_socket, flags)
rc2 = C.zmq_msg_close(zmq_msg)
_check_rc(rc2)
if track:
return zmq.MessageTracker()
def __init__(self, data=None, track=False, copy=None, copy_threshold=None):
self._failed_init = True
self.zmq_msg = ffi.cast('zmq_msg_t[1]',
C.malloc(ffi.sizeof("zmq_msg_t")))
# self.tracker should start finished
# except in the case where we are sharing memory with libzmq
if track:
self.tracker = zmq._FINISHED_TRACKER
if isinstance(data, unicode):
raise TypeError(
"Unicode objects not allowed. Only: bytes, buffer interfaces.")
if data is None:
rc = C.zmq_msg_init(self.zmq_msg)
_check_rc(rc)
self._failed_init = False
return
self._data = data
if type(data) is bytes:
# avoid unnecessary copy on .bytes access
self._bytes = data
self._buffer = memoryview(data)
c_data = ffi.from_buffer(self._buffer)
data_len_c = self._buffer.nbytes
if copy is None:
if copy_threshold and data_len_c < copy_threshold:
copy = True
else:
copy = False
if copy:
# copy message data instead of sharing memory
rc = C.zmq_msg_init_size(self.zmq_msg, data_len_c)
_check_rc(rc)
ffi.buffer(C.zmq_msg_data(self.zmq_msg),
data_len_c)[:] = self._buffer
self._failed_init = False
return
# Getting here means that we are doing a true zero-copy Frame,
# where libzmq and Python are sharing memory.
# Hook up garbage collection with MessageTracker and zmq_free_fn
# Event and MessageTracker for monitoring when zmq is done with data:
if track:
evt = Event()
self.tracker_event = evt
self.tracker = zmq.MessageTracker(evt)
# create the hint for zmq_free_fn
# two pointers: the zmq_gc context and a message to be sent to the zmq_gc PULL socket
# allows libzmq to signal to Python when it is done with Python-owned memory.
global zmq_gc
if zmq_gc is None:
from zmq.utils.garbage import gc as zmq_gc
# can't use ffi.new because it will be freed at the wrong time!
hint = ffi.cast("zhint[1]", C.malloc(ffi.sizeof("zhint")))
hint[0].id = zmq_gc.store(data, self.tracker_event)
if not zmq_gc._push_mutex:
zmq_gc._push_mutex = C.mutex_allocate()
hint[0].mutex = ffi.cast("mutex_t*", zmq_gc._push_mutex)
hint[0].sock = ffi.cast("void*", zmq_gc._push_socket.underlying)
# calls zmq_wrap_msg_init_data with the C.free_python_msg callback
rc = C.zmq_wrap_msg_init_data(
self.zmq_msg,
c_data,
data_len_c,
hint,
)
if rc != 0:
C.free(hint)
C.free(self.zmq_msg)
_check_rc(rc)
self._failed_init = False
# allow unicode
# disallow nan, because it's not actually valid JSON
json_packer = lambda obj: jsonapi.dumps(obj, default=date_default,
ensure_ascii=False, allow_nan=False,
)
json_unpacker = lambda s: jsonapi.loads(s)
pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL)
pickle_unpacker = pickle.loads
default_packer = json_packer
default_unpacker = json_unpacker
DELIM = b"<IDS|MSG>"
# singleton dummy tracker, which will always report as done
DONE = zmq.MessageTracker()
#-----------------------------------------------------------------------------
# Mixin tools for apps that use Sessions
#-----------------------------------------------------------------------------
def new_id():
"""Generate a new random id.
Avoids problematic runtime import in stdlib uuid on Python 2.
Returns
-------
id string (16 random bytes as hex-encoded text, chunks separated by '-')
"""
def _handle_sent(self, f):
"""Resolve sent Future, build MessageTracker"""
trackers = f.result()
trackers = [t for t in trackers if t is not None]
self._tracker = zmq.MessageTracker(*trackers)
self._sent_event.set()
from .futures import multi_future
from ipyparallel import error
from ipyparallel.util import _parse_date
from ipyparallel.util import compare_datetimes
from ipyparallel.util import progress
from ipyparallel.util import utcnow
def _raw_text(s):
display_pretty(s, raw=True)
_default = object()
# global empty tracker that's always done:
finished_tracker = zmq.MessageTracker()
@decorator
def check_ready(f, self, *args, **kwargs):
"""Check ready state prior to calling the method."""
self.wait(0)
if not self._ready:
raise TimeoutError("result not ready")
return f(self, *args, **kwargs)
_metadata_keys = []
# threading.TIMEOUT_MAX new in 3.2
_FOREVER = getattr(threading, 'TIMEOUT_MAX', int(1e6))
