def __getitem__(self, key): """getitem returns result value(s) if keyed by int/slice, or metadata if key is str. """ if isinstance(key, int): return error.collect_exceptions([self._result[key]], self._fname)[0] elif isinstance(key, slice): return error.collect_exceptions(self._result[key], self._fname) elif isinstance(key, basestring): values = [ md[key] for md in self._metadata ] if self._single_result: return values[0] else: return values else: raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
def wait(self, timeout=-1): """Wait until the result is available or until `timeout` seconds pass. This method always returns None. """ if self._ready: return self._ready = self._client.wait(self.msg_ids, timeout) if self._ready: try: results = map(self._client.results.get, self.msg_ids) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception, e: self._exception = e self._success = False else: self._success = True finally:
def wait(self, timeout=-1): """Wait until the result is available or until `timeout` seconds pass. This method always returns None. """ if self._ready: self._wait_for_outputs(timeout) return self._ready = self._client.wait(self.msg_ids, timeout) if self._ready: try: results = list(map(self._client.results.get, self.msg_ids)) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception as e: self._exception = e self._success = False else: self._success = True finally: if timeout is None or timeout < 0: # cutoff infinite wait at 10s timeout = 10 self._wait_for_outputs(timeout) if self.owner: self._metadata = [self._client.metadata.pop(mid) for mid in self.msg_ids] [self._client.results.pop(mid) for mid in self.msg_ids]
def wait(self, timeout=-1): """Wait until the result is available or until `timeout` seconds pass. This method always returns None. """ if self._ready: self._wait_for_outputs(timeout) return self._ready = self._client.wait(self.msg_ids, timeout) if self._ready: try: results = list(map(self._client.results.get, self.msg_ids)) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception as e: self._exception = e self._success = False else: self._success = True finally: if timeout is None or timeout < 0: # cutoff infinite wait at 10s timeout = 10 self._wait_for_outputs(timeout)
def __getitem__(self, key): """getitem returns result value(s) if keyed by int/slice, or metadata if key is str. """ if isinstance(key, int): self._check_ready() return error.collect_exceptions([self._result[key]], self._fname)[0] elif isinstance(key, slice): self._check_ready() return error.collect_exceptions(self._result[key], self._fname) elif isinstance(key, string_types): # metadata proxy *does not* require that results are done self.wait(0) values = [ md[key] for md in self._metadata ] if self._single_result: return values[0] else: return values else: raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
def __getitem__(self, key): """getitem returns result value(s) if keyed by int/slice, or metadata if key is str. """ if isinstance(key, int): self._check_ready() return error.collect_exceptions([self._result[key]], self._fname)[0] elif isinstance(key, slice): self._check_ready() return error.collect_exceptions(self._result[key], self._fname) elif isinstance(key, str): # metadata proxy *does not* require that results are done self.wait(0) values = [ md[key] for md in self._metadata ] if self._single_result: return values[0] else: return values else: raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
def wait(self, timeout=-1): """wait for result to complete.""" start = time.time() if self._ready: return local_ids = [ msg_id for msg_id in self.msg_ids if msg_id in self._client.outstanding ] local_ready = self._client.wait(local_ids, timeout) if local_ready: remote_ids = [ msg_id for msg_id in self.msg_ids if msg_id not in self._client.results ] if not remote_ids: self._ready = True else: rdict = self._client.result_status(remote_ids, status_only=False) pending = rdict['pending'] while pending and (timeout < 0 or time.time() < start + timeout): rdict = self._client.result_status(remote_ids, status_only=False) pending = rdict['pending'] if pending: time.sleep(0.1) if not pending: self._ready = True if self._ready: try: results = list(map(self._client.results.get, self.msg_ids)) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception as e: self._exception = e self._success = False else: self._success = True finally: self._metadata = list( map(self._client.metadata.get, self.msg_ids))
def wait(self, timeout=-1): """wait for result to complete.""" start = time.time() if self._ready: return local_ids = [m for m in self.msg_ids if m in self._client.outstanding] local_ready = self._client.wait(local_ids, timeout) if local_ready: remote_ids = [ m for m in self.msg_ids if m not in self._client.results] if not remote_ids: self._ready = True else: rdict = self._client.result_status( remote_ids, status_only=False) pending = rdict['pending'] while pending and (timeout < 0 or time.time() < start + timeout): rdict = self._client.result_status( remote_ids, status_only=False) pending = rdict['pending'] if pending: time.sleep(0.1) if not pending: self._ready = True if self._ready: try: results = list(map(self._client.results.get, self.msg_ids)) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception as e: self._exception = e self._success = False else: self._success = True finally: self._metadata = [self._client.metadata[mid] for mid in self.msg_ids] if self.owner: [self._client.metadata.pop(mid) for mid in self.msg_ids] [self._client.results.pop(mid) for mid in self.msg_ids]
def wait(self, timeout=-1): """wait for result to complete.""" start = time.time() if self._ready: return local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids) local_ready = self._client.wait(local_ids, timeout) if local_ready: remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids) if not remote_ids: self._ready = True else: rdict = self._client.result_status(remote_ids, status_only=False) pending = rdict['pending'] while pending and (timeout < 0 or time.time() < start+timeout): rdict = self._client.result_status(remote_ids, status_only=False) pending = rdict['pending'] if pending: time.sleep(0.1) if not pending: self._ready = True if self._ready: try: results = map(self._client.results.get, self.msg_ids) self._result = results if self._single_result: r = results[0] if isinstance(r, Exception): raise r else: results = error.collect_exceptions(results, self._fname) self._result = self._reconstruct_result(results) except Exception, e: self._exception = e self._success = False else: self._success = True finally:
def result_status(self, msg_ids, status_only=True): """Check on the status of the result(s) of the apply request with `msg_ids`. If status_only is False, then the actual results will be retrieved, else only the status of the results will be checked. Parameters ---------- msg_ids : list of msg_ids if int: Passed as index to self.history for convenience. status_only : bool (default: True) if False: Retrieve the actual results of completed tasks. Returns ------- results : dict There will always be the keys 'pending' and 'completed', which will be lists of msg_ids that are incomplete or complete. If `status_only` is False, then completed results will be keyed by their `msg_id`. """ if not isinstance(msg_ids, (list,tuple)): msg_ids = [msg_ids] theids = [] for msg_id in msg_ids: if isinstance(msg_id, int): msg_id = self.history[msg_id] if not isinstance(msg_id, basestring): raise TypeError("msg_ids must be str, not %r"%msg_id) theids.append(msg_id) completed = [] local_results = {} # comment this block out to temporarily disable local shortcut: for msg_id in theids: if msg_id in self.results: completed.append(msg_id) local_results[msg_id] = self.results[msg_id] theids.remove(msg_id) if theids: # some not locally cached content = dict(msg_ids=theids, status_only=status_only) msg = self.session.send(self._query_socket, "result_request", content=content) zmq.select([self._query_socket], [], []) idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) buffers = msg['buffers'] else: content = dict(completed=[],pending=[]) content['completed'].extend(completed) if status_only: return content failures = [] # load cached results into result: content.update(local_results) # update cache with results: for msg_id in sorted(theids): if msg_id in content['completed']: rec = content[msg_id] parent = rec['header'] header = rec['result_header'] rcontent = rec['result_content'] iodict = rec['io'] if isinstance(rcontent, str): rcontent = self.session.unpack(rcontent) md = self.metadata[msg_id] md.update(self._extract_metadata(header, parent, rcontent)) md.update(iodict) if rcontent['status'] == 'ok': res,buffers = util.unserialize_object(buffers) else: print rcontent res = self._unwrap_exception(rcontent) failures.append(res) self.results[msg_id] = res content[msg_id] = res if len(theids) == 1 and failures: raise failures[0] error.collect_exceptions(failures, "result_status") return content
def result_status(self, msg_ids, status_only=True): """Check on the status of the result(s) of the apply request with `msg_ids`. If status_only is False, then the actual results will be retrieved, else only the status of the results will be checked. Parameters ---------- msg_ids : list of msg_ids if int: Passed as index to self.history for convenience. status_only : bool (default: True) if False: Retrieve the actual results of completed tasks. Returns ------- results : dict There will always be the keys 'pending' and 'completed', which will be lists of msg_ids that are incomplete or complete. If `status_only` is False, then completed results will be keyed by their `msg_id`. """ if not isinstance(msg_ids, (list, tuple)): msg_ids = [msg_ids] theids = [] for msg_id in msg_ids: if isinstance(msg_id, int): msg_id = self.history[msg_id] if not isinstance(msg_id, basestring): raise TypeError("msg_ids must be str, not %r" % msg_id) theids.append(msg_id) completed = [] local_results = {} # comment this block out to temporarily disable local shortcut: for msg_id in theids: if msg_id in self.results: completed.append(msg_id) local_results[msg_id] = self.results[msg_id] theids.remove(msg_id) if theids: # some not locally cached content = dict(msg_ids=theids, status_only=status_only) msg = self.session.send(self._query_socket, "result_request", content=content) zmq.select([self._query_socket], [], []) idents, msg = self.session.recv(self._query_socket, zmq.NOBLOCK) if self.debug: pprint(msg) content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) buffers = msg['buffers'] else: content = dict(completed=[], pending=[]) content['completed'].extend(completed) if status_only: return content failures = [] # load cached results into result: content.update(local_results) # update cache with results: for msg_id in sorted(theids): if msg_id in content['completed']: rec = content[msg_id] parent = rec['header'] header = rec['result_header'] rcontent = rec['result_content'] iodict = rec['io'] if isinstance(rcontent, str): rcontent = self.session.unpack(rcontent) md = self.metadata[msg_id] md.update(self._extract_metadata(header, parent, rcontent)) md.update(iodict) if rcontent['status'] == 'ok': res, buffers = util.unserialize_object(buffers) else: print rcontent res = self._unwrap_exception(rcontent) failures.append(res) self.results[msg_id] = res content[msg_id] = res if len(theids) == 1 and failures: raise failures[0] error.collect_exceptions(failures, "result_status") return content