def test_tracker(self): m = zmq.Frame(b'asdf', track=True) self.assertFalse(m.tracker.done) pm = zmq.MessageTracker(m) self.assertFalse(pm.done) del m self.assertTrue(pm.done)
def scatter(self, key, seq, dist='b', flatten=False, targets=None, block=None, track=None): """ Partition a Python sequence and send the partitions to a set of engines. """ block = block if block is not None else self.block track = track if track is not None else self.track targets = targets if targets is not None else self.targets mapObject = Map.dists[dist]() nparts = len(targets) msg_ids = [] trackers = [] for index, engineid in enumerate(targets): partition = mapObject.getPartition(seq, index, nparts) if flatten and len(partition) == 1: ns = {key: partition[0]} else: ns = {key: partition} r = self.push(ns, block=False, track=track, targets=engineid) msg_ids.extend(r.msg_ids) if track: trackers.append(r._tracker) if track: tracker = zmq.MessageTracker(*trackers) else: tracker = None r = AsyncResult(self.client, msg_ids, fname='scatter', targets=targets, tracker=tracker) if block: r.wait() else: return r
def test_tracker(self): m = zmq.Message('asdf'.encode(), track=True) self.assertFalse(m.done) pm = zmq.MessageTracker(m) self.assertFalse(pm.done) del m self.assertTrue(pm.done)
def test_tracker(self): m = zmq.Message(asbytes('asdf'), track=True) self.assertFalse(m.tracker.done) pm = zmq.MessageTracker(m) self.assertFalse(pm.done) del m self.assertTrue(pm.done)
def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, track=None): """calls f(*args, **kwargs) on remote engines, returning the result. This method sets all of `apply`'s flags via this View's attributes. Parameters ---------- f : callable args : list [default: empty] kwargs : dict [default: empty] targets : target list [default: self.targets] where to run block : bool [default: self.block] whether to block track : bool [default: self.track] whether to ask zmq to track the message, for safe non-copying sends Returns ------- if self.block is False: returns AsyncResult else: returns actual result of f(*args, **kwargs) on the engine(s) This will be a list of self.targets is also a list (even length 1), or the single result if self.targets is an integer engine id """ args = [] if args is None else args kwargs = {} if kwargs is None else kwargs block = self.block if block is None else block track = self.track if track is None else track targets = self.targets if targets is None else targets _idents, _targets = self.client._build_targets(targets) msg_ids = [] trackers = [] for ident in _idents: msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track, ident=ident) if track: trackers.append(msg['tracker']) msg_ids.append(msg['header']['msg_id']) if isinstance(targets, int): msg_ids = msg_ids[0] tracker = None if track is False else zmq.MessageTracker(*trackers) ar = AsyncResult(self.client, msg_ids, fname=getname(f), targets=_targets, tracker=tracker, owner=True, ) if block: try: return ar.get() except KeyboardInterrupt: pass return ar
def test_tracker(self): m = zmq.Frame(b'asdf', track=True) self.assertFalse(m.tracker.done) pm = zmq.MessageTracker(m) self.assertFalse(pm.done) del m for i in range(10): if pm.done: break time.sleep(0.1) self.assertTrue(pm.done)
def test_multi_tracker(self): m = zmq.Frame(b'asdf', track=True) m2 = zmq.Frame(b'whoda', track=True) mt = zmq.MessageTracker(m, m2) self.assertFalse(m.tracker.done) self.assertFalse(mt.done) self.assertRaises(zmq.NotDone, mt.wait, 0.1) del m time.sleep(0.1) self.assertRaises(zmq.NotDone, mt.wait, 0.1) self.assertFalse(mt.done) del m2 self.assertTrue(mt.wait() is None) self.assertTrue(mt.done)
def test_multi_tracker(self): m = zmq.Frame(b'asdf', copy=False, track=True) m2 = zmq.Frame(b'whoda', copy=False, track=True) mt = zmq.MessageTracker(m, m2) self.assertFalse(m.tracker.done) self.assertFalse(mt.done) self.assertRaises(zmq.NotDone, mt.wait, 0.1) del m for i in range(3): gc.collect() self.assertRaises(zmq.NotDone, mt.wait, 0.1) self.assertFalse(mt.done) del m2 for i in range(3): gc.collect() assert mt.wait(0.1) is None assert mt.done
def send(self, message, flags=0, copy=False, track=False): if isinstance(message, unicode): raise TypeError("Message must be in bytes, not an unicode Object") if isinstance(message, Frame): message = message.bytes zmq_msg = ffi.new('zmq_msg_t*') c_message = ffi.new('char[]', message) rc = C.zmq_msg_init_size(zmq_msg, len(message)) C.memcpy(C.zmq_msg_data(zmq_msg), c_message, len(message)) rc = C.zmq_msg_send(zmq_msg, self._zmq_socket, flags) C.zmq_msg_close(zmq_msg) _check_rc(rc) if track: return zmq.MessageTracker()
def __init__(self, data, track=False): try: view(data) except TypeError: raise self._data = data if isinstance(data, unicode): raise TypeError("Unicode objects not allowed. Only: str/bytes, " + "buffer interfaces.") self.more = False self.tracker = None self.closed = False if track: self.tracker = zmq.MessageTracker() self.buffer = view(self.bytes)
def send(self, message, flags=0, copy=False, track=False): if isinstance(message, unicode): raise TypeError("Message must be in bytes, not an unicode Object") if isinstance(message, Frame): message = message.bytes zmq_msg = ffi.new('zmq_msg_t*') if not isinstance(message, bytes): # cast any bufferable data to bytes via memoryview message = memoryview(message).tobytes() c_message = ffi.new('char[]', message) rc = C.zmq_msg_init_size(zmq_msg, len(message)) _check_rc(rc) C.memcpy(C.zmq_msg_data(zmq_msg), c_message, len(message)) _retry_sys_call(C.zmq_msg_send, zmq_msg, self._zmq_socket, flags) rc2 = C.zmq_msg_close(zmq_msg) _check_rc(rc2) if track: return zmq.MessageTracker()
def __init__(self, data=None, track=False, copy=None, copy_threshold=None): self._failed_init = True self.zmq_msg = ffi.cast('zmq_msg_t[1]', C.malloc(ffi.sizeof("zmq_msg_t"))) # self.tracker should start finished # except in the case where we are sharing memory with libzmq if track: self.tracker = zmq._FINISHED_TRACKER if isinstance(data, unicode): raise TypeError( "Unicode objects not allowed. Only: bytes, buffer interfaces.") if data is None: rc = C.zmq_msg_init(self.zmq_msg) _check_rc(rc) self._failed_init = False return self._data = data if type(data) is bytes: # avoid unnecessary copy on .bytes access self._bytes = data self._buffer = memoryview(data) c_data = ffi.from_buffer(self._buffer) data_len_c = self._buffer.nbytes if copy is None: if copy_threshold and data_len_c < copy_threshold: copy = True else: copy = False if copy: # copy message data instead of sharing memory rc = C.zmq_msg_init_size(self.zmq_msg, data_len_c) _check_rc(rc) ffi.buffer(C.zmq_msg_data(self.zmq_msg), data_len_c)[:] = self._buffer self._failed_init = False return # Getting here means that we are doing a true zero-copy Frame, # where libzmq and Python are sharing memory. # Hook up garbage collection with MessageTracker and zmq_free_fn # Event and MessageTracker for monitoring when zmq is done with data: if track: evt = Event() self.tracker_event = evt self.tracker = zmq.MessageTracker(evt) # create the hint for zmq_free_fn # two pointers: the zmq_gc context and a message to be sent to the zmq_gc PULL socket # allows libzmq to signal to Python when it is done with Python-owned memory. global zmq_gc if zmq_gc is None: from zmq.utils.garbage import gc as zmq_gc # can't use ffi.new because it will be freed at the wrong time! hint = ffi.cast("zhint[1]", C.malloc(ffi.sizeof("zhint"))) hint[0].id = zmq_gc.store(data, self.tracker_event) if not zmq_gc._push_mutex: zmq_gc._push_mutex = C.mutex_allocate() hint[0].mutex = ffi.cast("mutex_t*", zmq_gc._push_mutex) hint[0].sock = ffi.cast("void*", zmq_gc._push_socket.underlying) # calls zmq_wrap_msg_init_data with the C.free_python_msg callback rc = C.zmq_wrap_msg_init_data( self.zmq_msg, c_data, data_len_c, hint, ) if rc != 0: C.free(hint) C.free(self.zmq_msg) _check_rc(rc) self._failed_init = False
# allow unicode # disallow nan, because it's not actually valid JSON json_packer = lambda obj: jsonapi.dumps(obj, default=date_default, ensure_ascii=False, allow_nan=False, ) json_unpacker = lambda s: jsonapi.loads(s) pickle_packer = lambda o: pickle.dumps(squash_dates(o), PICKLE_PROTOCOL) pickle_unpacker = pickle.loads default_packer = json_packer default_unpacker = json_unpacker DELIM = b"<IDS|MSG>" # singleton dummy tracker, which will always report as done DONE = zmq.MessageTracker() #----------------------------------------------------------------------------- # Mixin tools for apps that use Sessions #----------------------------------------------------------------------------- def new_id(): """Generate a new random id. Avoids problematic runtime import in stdlib uuid on Python 2. Returns ------- id string (16 random bytes as hex-encoded text, chunks separated by '-') """
def _handle_sent(self, f): """Resolve sent Future, build MessageTracker""" trackers = f.result() trackers = [t for t in trackers if t is not None] self._tracker = zmq.MessageTracker(*trackers) self._sent_event.set()
from .futures import multi_future from ipyparallel import error from ipyparallel.util import _parse_date from ipyparallel.util import compare_datetimes from ipyparallel.util import progress from ipyparallel.util import utcnow def _raw_text(s): display_pretty(s, raw=True) _default = object() # global empty tracker that's always done: finished_tracker = zmq.MessageTracker() @decorator def check_ready(f, self, *args, **kwargs): """Check ready state prior to calling the method.""" self.wait(0) if not self._ready: raise TimeoutError("result not ready") return f(self, *args, **kwargs) _metadata_keys = [] # threading.TIMEOUT_MAX new in 3.2 _FOREVER = getattr(threading, 'TIMEOUT_MAX', int(1e6))