Beispiel #1
0
 def __getitem__(self, key):
     """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
     """
     if isinstance(key, int):
         return error.collect_exceptions([self._result[key]], self._fname)[0]
     elif isinstance(key, slice):
         return error.collect_exceptions(self._result[key], self._fname)
     elif isinstance(key, basestring):
         values = [ md[key] for md in self._metadata ]
         if self._single_result:
             return values[0]
         else:
             return values
     else:
         raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
Beispiel #2
0
 def __getitem__(self, key):
     """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
     """
     if isinstance(key, int):
         return error.collect_exceptions([self._result[key]], self._fname)[0]
     elif isinstance(key, slice):
         return error.collect_exceptions(self._result[key], self._fname)
     elif isinstance(key, basestring):
         values = [ md[key] for md in self._metadata ]
         if self._single_result:
             return values[0]
         else:
             return values
     else:
         raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
Beispiel #3
0
    def wait(self, timeout=-1):
        """Wait until the result is available or until `timeout` seconds pass.

        This method always returns None.
        """
        if self._ready:
            return
        self._ready = self._client.wait(self.msg_ids, timeout)
        if self._ready:
            try:
                results = map(self._client.results.get, self.msg_ids)
                self._result = results
                if self._single_result:
                    r = results[0]
                    if isinstance(r, Exception):
                        raise r
                else:
                    results = error.collect_exceptions(results, self._fname)
                self._result = self._reconstruct_result(results)
            except Exception, e:
                self._exception = e
                self._success = False
            else:
                self._success = True
            finally:
Beispiel #4
0
    def wait(self, timeout=-1):
        """Wait until the result is available or until `timeout` seconds pass.

        This method always returns None.
        """
        if self._ready:
            self._wait_for_outputs(timeout)
            return
        self._ready = self._client.wait(self.msg_ids, timeout)
        if self._ready:
            try:
                results = list(map(self._client.results.get, self.msg_ids))
                self._result = results
                if self._single_result:
                    r = results[0]
                    if isinstance(r, Exception):
                        raise r
                else:
                    results = error.collect_exceptions(results, self._fname)
                self._result = self._reconstruct_result(results)
            except Exception as e:
                self._exception = e
                self._success = False
            else:
                self._success = True
            finally:
                if timeout is None or timeout < 0:
                    # cutoff infinite wait at 10s
                    timeout = 10
                self._wait_for_outputs(timeout)
                
                if self.owner:
                    
                    self._metadata = [self._client.metadata.pop(mid) for mid in self.msg_ids]
                    [self._client.results.pop(mid) for mid in self.msg_ids]
    def wait(self, timeout=-1):
        """Wait until the result is available or until `timeout` seconds pass.

        This method always returns None.
        """
        if self._ready:
            return
        self._ready = self._client.wait(self.msg_ids, timeout)
        if self._ready:
            try:
                results = map(self._client.results.get, self.msg_ids)
                self._result = results
                if self._single_result:
                    r = results[0]
                    if isinstance(r, Exception):
                        raise r
                else:
                    results = error.collect_exceptions(results, self._fname)
                self._result = self._reconstruct_result(results)
            except Exception, e:
                self._exception = e
                self._success = False
            else:
                self._success = True
            finally:
Beispiel #6
0
    def wait(self, timeout=-1):
        """Wait until the result is available or until `timeout` seconds pass.

        This method always returns None.
        """
        if self._ready:
            self._wait_for_outputs(timeout)
            return
        self._ready = self._client.wait(self.msg_ids, timeout)
        if self._ready:
            try:
                results = list(map(self._client.results.get, self.msg_ids))
                self._result = results
                if self._single_result:
                    r = results[0]
                    if isinstance(r, Exception):
                        raise r
                else:
                    results = error.collect_exceptions(results, self._fname)
                self._result = self._reconstruct_result(results)
            except Exception as e:
                self._exception = e
                self._success = False
            else:
                self._success = True
            finally:
                if timeout is None or timeout < 0:
                    # cutoff infinite wait at 10s
                    timeout = 10
                self._wait_for_outputs(timeout)
Beispiel #7
0
 def __getitem__(self, key):
     """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
     """
     if isinstance(key, int):
         self._check_ready()
         return error.collect_exceptions([self._result[key]], self._fname)[0]
     elif isinstance(key, slice):
         self._check_ready()
         return error.collect_exceptions(self._result[key], self._fname)
     elif isinstance(key, string_types):
         # metadata proxy *does not* require that results are done
         self.wait(0)
         values = [ md[key] for md in self._metadata ]
         if self._single_result:
             return values[0]
         else:
             return values
     else:
         raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
Beispiel #8
0
 def __getitem__(self, key):
     """getitem returns result value(s) if keyed by int/slice, or metadata if key is str.
     """
     if isinstance(key, int):
         self._check_ready()
         return error.collect_exceptions([self._result[key]], self._fname)[0]
     elif isinstance(key, slice):
         self._check_ready()
         return error.collect_exceptions(self._result[key], self._fname)
     elif isinstance(key, str):
         # metadata proxy *does not* require that results are done
         self.wait(0)
         values = [ md[key] for md in self._metadata ]
         if self._single_result:
             return values[0]
         else:
             return values
     else:
         raise TypeError("Invalid key type %r, must be 'int','slice', or 'str'"%type(key))
 def wait(self, timeout=-1):
     """wait for result to complete."""
     start = time.time()
     if self._ready:
         return
     local_ids = [
         msg_id for msg_id in self.msg_ids
         if msg_id in self._client.outstanding
     ]
     local_ready = self._client.wait(local_ids, timeout)
     if local_ready:
         remote_ids = [
             msg_id for msg_id in self.msg_ids
             if msg_id not in self._client.results
         ]
         if not remote_ids:
             self._ready = True
         else:
             rdict = self._client.result_status(remote_ids,
                                                status_only=False)
             pending = rdict['pending']
             while pending and (timeout < 0
                                or time.time() < start + timeout):
                 rdict = self._client.result_status(remote_ids,
                                                    status_only=False)
                 pending = rdict['pending']
                 if pending:
                     time.sleep(0.1)
             if not pending:
                 self._ready = True
     if self._ready:
         try:
             results = list(map(self._client.results.get, self.msg_ids))
             self._result = results
             if self._single_result:
                 r = results[0]
                 if isinstance(r, Exception):
                     raise r
             else:
                 results = error.collect_exceptions(results, self._fname)
             self._result = self._reconstruct_result(results)
         except Exception as e:
             self._exception = e
             self._success = False
         else:
             self._success = True
         finally:
             self._metadata = list(
                 map(self._client.metadata.get, self.msg_ids))
Beispiel #10
0
 def wait(self, timeout=-1):
     """wait for result to complete."""
     start = time.time()
     if self._ready:
         return
     local_ids = [m for m in self.msg_ids if m in self._client.outstanding]
     local_ready = self._client.wait(local_ids, timeout)
     if local_ready:
         remote_ids = [
             m for m in self.msg_ids if m not in self._client.results]
         if not remote_ids:
             self._ready = True
         else:
             rdict = self._client.result_status(
                 remote_ids, status_only=False)
             pending = rdict['pending']
             while pending and (timeout < 0 or time.time() < start + timeout):
                 rdict = self._client.result_status(
                     remote_ids, status_only=False)
                 pending = rdict['pending']
                 if pending:
                     time.sleep(0.1)
             if not pending:
                 self._ready = True
     if self._ready:
         try:
             results = list(map(self._client.results.get, self.msg_ids))
             self._result = results
             if self._single_result:
                 r = results[0]
                 if isinstance(r, Exception):
                     raise r
             else:
                 results = error.collect_exceptions(results, self._fname)
             self._result = self._reconstruct_result(results)
         except Exception as e:
             self._exception = e
             self._success = False
         else:
             self._success = True
         finally:
             self._metadata = [self._client.metadata[mid]
                               for mid in self.msg_ids]
             if self.owner:
                 [self._client.metadata.pop(mid) for mid in self.msg_ids]
                 [self._client.results.pop(mid) for mid in self.msg_ids]
Beispiel #11
0
 def wait(self, timeout=-1):
     """wait for result to complete."""
     start = time.time()
     if self._ready:
         return
     local_ids = filter(lambda msg_id: msg_id in self._client.outstanding, self.msg_ids)
     local_ready = self._client.wait(local_ids, timeout)
     if local_ready:
         remote_ids = filter(lambda msg_id: msg_id not in self._client.results, self.msg_ids)
         if not remote_ids:
             self._ready = True
         else:
             rdict = self._client.result_status(remote_ids, status_only=False)
             pending = rdict['pending']
             while pending and (timeout < 0 or time.time() < start+timeout):
                 rdict = self._client.result_status(remote_ids, status_only=False)
                 pending = rdict['pending']
                 if pending:
                     time.sleep(0.1)
             if not pending:
                 self._ready = True
     if self._ready:
         try:
             results = map(self._client.results.get, self.msg_ids)
             self._result = results
             if self._single_result:
                 r = results[0]
                 if isinstance(r, Exception):
                     raise r
             else:
                 results = error.collect_exceptions(results, self._fname)
             self._result = self._reconstruct_result(results)
         except Exception, e:
             self._exception = e
             self._success = False
         else:
             self._success = True
         finally:
Beispiel #12
0
 def result_status(self, msg_ids, status_only=True):
     """Check on the status of the result(s) of the apply request with `msg_ids`.
     
     If status_only is False, then the actual results will be retrieved, else
     only the status of the results will be checked.
     
     Parameters
     ----------
     
     msg_ids : list of msg_ids
         if int:
             Passed as index to self.history for convenience.
     status_only : bool (default: True)
         if False:
             Retrieve the actual results of completed tasks.
     
     Returns
     -------
     
     results : dict
         There will always be the keys 'pending' and 'completed', which will
         be lists of msg_ids that are incomplete or complete. If `status_only`
         is False, then completed results will be keyed by their `msg_id`.
     """
     if not isinstance(msg_ids, (list,tuple)):
         msg_ids = [msg_ids]
         
     theids = []
     for msg_id in msg_ids:
         if isinstance(msg_id, int):
             msg_id = self.history[msg_id]
         if not isinstance(msg_id, basestring):
             raise TypeError("msg_ids must be str, not %r"%msg_id)
         theids.append(msg_id)
     
     completed = []
     local_results = {}
     
     # comment this block out to temporarily disable local shortcut:
     for msg_id in theids:
         if msg_id in self.results:
             completed.append(msg_id)
             local_results[msg_id] = self.results[msg_id]
             theids.remove(msg_id)
     
     if theids: # some not locally cached
         content = dict(msg_ids=theids, status_only=status_only)
         msg = self.session.send(self._query_socket, "result_request", content=content)
         zmq.select([self._query_socket], [], [])
         idents,msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
         if self.debug:
             pprint(msg)
         content = msg['content']
         if content['status'] != 'ok':
             raise self._unwrap_exception(content)
         buffers = msg['buffers']
     else:
         content = dict(completed=[],pending=[])
     
     content['completed'].extend(completed)
     
     if status_only:
         return content
     
     failures = []
     # load cached results into result:
     content.update(local_results)
     # update cache with results:
     for msg_id in sorted(theids):
         if msg_id in content['completed']:
             rec = content[msg_id]
             parent = rec['header']
             header = rec['result_header']
             rcontent = rec['result_content']
             iodict = rec['io']
             if isinstance(rcontent, str):
                 rcontent = self.session.unpack(rcontent)
             
             md = self.metadata[msg_id]
             md.update(self._extract_metadata(header, parent, rcontent))
             md.update(iodict)
             
             if rcontent['status'] == 'ok':
                 res,buffers = util.unserialize_object(buffers)
             else:
                 print rcontent
                 res = self._unwrap_exception(rcontent)
                 failures.append(res)
             
             self.results[msg_id] = res
             content[msg_id] = res
     
     if len(theids) == 1 and failures:
             raise failures[0]
     
     error.collect_exceptions(failures, "result_status")
     return content
Beispiel #13
0
    def result_status(self, msg_ids, status_only=True):
        """Check on the status of the result(s) of the apply request with `msg_ids`.
        
        If status_only is False, then the actual results will be retrieved, else
        only the status of the results will be checked.
        
        Parameters
        ----------
        
        msg_ids : list of msg_ids
            if int:
                Passed as index to self.history for convenience.
        status_only : bool (default: True)
            if False:
                Retrieve the actual results of completed tasks.
        
        Returns
        -------
        
        results : dict
            There will always be the keys 'pending' and 'completed', which will
            be lists of msg_ids that are incomplete or complete. If `status_only`
            is False, then completed results will be keyed by their `msg_id`.
        """
        if not isinstance(msg_ids, (list, tuple)):
            msg_ids = [msg_ids]

        theids = []
        for msg_id in msg_ids:
            if isinstance(msg_id, int):
                msg_id = self.history[msg_id]
            if not isinstance(msg_id, basestring):
                raise TypeError("msg_ids must be str, not %r" % msg_id)
            theids.append(msg_id)

        completed = []
        local_results = {}

        # comment this block out to temporarily disable local shortcut:
        for msg_id in theids:
            if msg_id in self.results:
                completed.append(msg_id)
                local_results[msg_id] = self.results[msg_id]
                theids.remove(msg_id)

        if theids:  # some not locally cached
            content = dict(msg_ids=theids, status_only=status_only)
            msg = self.session.send(self._query_socket,
                                    "result_request",
                                    content=content)
            zmq.select([self._query_socket], [], [])
            idents, msg = self.session.recv(self._query_socket, zmq.NOBLOCK)
            if self.debug:
                pprint(msg)
            content = msg['content']
            if content['status'] != 'ok':
                raise self._unwrap_exception(content)
            buffers = msg['buffers']
        else:
            content = dict(completed=[], pending=[])

        content['completed'].extend(completed)

        if status_only:
            return content

        failures = []
        # load cached results into result:
        content.update(local_results)
        # update cache with results:
        for msg_id in sorted(theids):
            if msg_id in content['completed']:
                rec = content[msg_id]
                parent = rec['header']
                header = rec['result_header']
                rcontent = rec['result_content']
                iodict = rec['io']
                if isinstance(rcontent, str):
                    rcontent = self.session.unpack(rcontent)

                md = self.metadata[msg_id]
                md.update(self._extract_metadata(header, parent, rcontent))
                md.update(iodict)

                if rcontent['status'] == 'ok':
                    res, buffers = util.unserialize_object(buffers)
                else:
                    print rcontent
                    res = self._unwrap_exception(rcontent)
                    failures.append(res)

                self.results[msg_id] = res
                content[msg_id] = res

        if len(theids) == 1 and failures:
            raise failures[0]

        error.collect_exceptions(failures, "result_status")
        return content