def run_in_executor(self, executor, callback, *args): if (coroutines.iscoroutine(callback) or coroutines.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used with run_in_executor()") if isinstance(callback, events.Handle): assert not args assert not isinstance(callback, events.TimerHandle) if callback._cancelled: f = futures.Future(loop=self) f.set_result(None) return f callback, args = callback._callback, callback._args if executor is None: executor = self._default_executor if executor is None: executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS) self._default_executor = executor return futures.wrap_future(executor.submit(callback, *args), loop=self)
def _create_connection_transport(self, sock, protocol_factory, ssl, server_hostname): protocol = protocol_factory() waiter = futures.Future(loop=self) if ssl: sslcontext = None if isinstance(ssl, bool) else ssl transport = self._make_ssl_transport( sock, protocol, sslcontext, waiter, server_side=False, server_hostname=server_hostname) else: transport = self._make_socket_transport(sock, protocol, waiter) yield from waiter return transport, protocol
def run(self, funcs): if self._workers: raise RuntimeError("Can not start another `run` with %s" " existing workers" % (len(self._workers))) self._queue = compat_queue.Queue() self._death.clear() futs = [] for i in compat_range(0, self._max_workers): w = threading.Thread(target=_chained_worker, args=(i + 1, self._death, self._queue, futs)) w.daemon = True w.start() self._workers.append(w) for func in funcs: fut = futures.Future() futs.append(fut) self._queue.put((func, fut)) return futs
def run_thread_and_sync(self): """Start the server in a separate thread and wait for notification that it has successfuly started""" # create future so that we can sync with the IO loop running inside the # thread we started sync = futures.Future() self.thread = threading.Thread(target=self._start_loop_thread, args=(sync, )) self.thread.start() # wait for the loop to be stated futures.wait([sync]) self.log.info('test server thread started') return self
def wait_for(fut, timeout, *, loop=None): """Wait for the single Future or coroutine to complete, with timeout. Coroutine will be wrapped in Task. Returns result of the Future or coroutine. When a timeout occurs, it cancels the task and raises TimeoutError. To avoid the task cancellation, wrap it in shield(). If the wait is cancelled, the task is also cancelled. This function is a coroutine. """ if loop is None: loop = events.get_event_loop() if timeout is None: return (yield from fut) waiter = futures.Future(loop=loop) timeout_handle = loop.call_later(timeout, _release_waiter, waiter) cb = functools.partial(_release_waiter, waiter) fut = ensure_future(fut, loop=loop) fut.add_done_callback(cb) try: # wait until the future completes or the timeout try: yield from waiter except futures.CancelledError: fut.remove_done_callback(cb) fut.cancel() raise if fut.done(): return fut.result() else: fut.remove_done_callback(cb) fut.cancel() raise futures.TimeoutError() finally: timeout_handle.cancel()
def mock_submit(unused_fn, chunk_size: int, hard_limit: Optional[int] = None): target_valid = chunk_size target_total = int((1 + failure_rate) * chunk_size) if hard_limit is not None and target_total > hard_limit: target_valid = ceil(chunk_size * (hard_limit / target_total)) target_total = hard_limit target_invalid = target_total - target_valid lines = [ GenText(text=f'line-{i}', valid=(i >= target_invalid)) for i in range(target_total) ] mock_future = futures.Future() mock_future.set_result((chunk_size, lines, target_invalid)) return mock_future
def run_coroutine_threadsafe(coro, loop): """Submit a coroutine object to a given event loop. Return a concurrent_futures.Future to access the result. """ if not coroutines.iscoroutine(coro): raise TypeError('A coroutine object is required') future = concurrent_futures.Future() def callback(): try: futures._chain_future(ensure_future(coro, loop=loop), future) except Exception as exc: if future.set_running_or_notify_cancel(): future.set_exception(exc) raise loop.call_soon_threadsafe(callback) return future
def create_specific_server(self, client_id: str) -> SpecificServer: """ Create, but not start a SpecificServer for a given client. This method must be called once per client. """ with self.server_lock: assert ( self.servers.get(client_id) is None ), f"Server already created for Client: {client_id}" port = self._get_unused_port() server = SpecificServer( port=port, process_handle_future=futures.Future(), channel=ray._private.utils.init_grpc_channel( f"127.0.0.1:{port}", options=GRPC_OPTIONS ), ) self.servers[client_id] = server return server
def submit(self, fun, *args, additional_setup_lines=[], **kwargs): """Submit a job to the pool.""" fut = futures.Future() # Start the job. workerid = random_string() funcser = cloudpickle.dumps((fun, args, kwargs), True) with open(INFILE_FMT % workerid, 'wb') as f: f.write(funcser) jobid = self._start(workerid, additional_setup_lines) if self.debug: print("job submitted: %i" % jobid, file=sys.stderr) # Thread will wait for it to finish. self.wait_thread.wait(OUTFILE_FMT % workerid, jobid) with self.jobs_lock: self.jobs[jobid] = (fut, workerid) return fut
def submit(self, fun, *args, **kwargs): """Submit a job to the pool.""" fut = futures.Future() # Start the job. workerid = random_string() funcser = serialization.serialize((fun, args, kwargs), True) with open(INFILE_FMT % workerid, 'w') as f: f.write(funcser) jobid = self._start(workerid) if self.debug: print >> sys.stderr, "job submitted: %i" % jobid # Thread will wait for it to finish. self.wait_thread.wait(OUTFILE_FMT % workerid, jobid) with self.jobs_lock: self.jobs[jobid] = (fut, workerid) return fut
def _wait(fs, timeout, return_when, loop): """Internal helper for wait() and _wait_for(). The fs argument must be a collection of Futures. """ assert fs, 'Set of Futures is empty.' waiter = futures.Future(loop=loop) timeout_handle = None if timeout is not None: timeout_handle = loop.call_later(timeout, _release_waiter, waiter) counter = len(fs) def _on_completion(f): nonlocal counter counter -= 1 if (counter <= 0 or return_when == FIRST_COMPLETED or return_when == FIRST_EXCEPTION and (not f.cancelled() and f.exception() is not None)): if timeout_handle is not None: timeout_handle.cancel() if not waiter.done(): waiter.set_result(None) for f in fs: f.add_done_callback(_on_completion) try: yield from waiter finally: if timeout_handle is not None: timeout_handle.cancel() done, pending = set(), set() for f in fs: f.remove_done_callback(_on_completion) if f.done(): done.add(f) else: pending.add(f) return done, pending
def _dchain(pool, futures, fn): result = cf.Future() bnum = len(futures) def callback(_): nonlocal bnum bnum -= 1 if bnum > 0: return try: results = [f.result() for f in futures] temp = pool.submit(fn, *results) copy = lambda _: _copy_future_state(temp, result) temp.add_done_callback(copy) except: result.cancel() raise for f in futures: f.add_done_callback(callback) return result
def test_futures_added_is_triggered_and_pipelined(self): inc1 = mock.MagicMock() inc1.side_effect = inc embedded = futures.Future() embedded.add_done_callback(inc1) unit = PipedFuture() unit.add_done_callback(inc1) unit.add_done_future(embedded) unit.add_done_callback(inc1) unit.set_result(0) self.assertEqual(unit.result(), 2) # initial result is the pipeline result until that point, final result # is still its initial result since it's a standard Future instance. self.assertEqual(embedded.result(), 1) inc1.assert_has_calls([mock.call(unit), mock.call(embedded), mock.call(unit)])
def _batch_write_async(self, context, write_request_list): future_result = futures.Future() batch_future = self._execute_async(self._storage_driver.batch_write, context, write_request_list) def callback(res): try: res.result() unprocessed_items = () except NotImplementedError: unprocessed_items = self._batch_write_in_emulation_mode( context, write_request_list) except Exception: LOG.exception("Can't process batch write request") unprocessed_items = write_request_list future_result.set_result(unprocessed_items) batch_future.add_done_callback(callback) return future_result
def send_message(self, servers_id, cmd, args=(), kwds={}): """Send a message to (possibly) multiple servers. The same message is sent to all servers. """ loop = ioloop.IOLoop.instance() if type(servers_id) not in (tuple, list): servers_id = [servers_id] future_list = [] for server_id in servers_id: future = futures.Future() self.futures[server_id] = future future_list.append(future) loop.add_callback(self.client_instance.send, server_address=server_id, cmd=cmd, args=args, kwds=kwds) results = [] for future in future_list: results.append(future.result(timeout=self.timeout)) statuses, cmds, args_answers, kwds_answers = zip(*results) # # Check the reply status # for status, args_answer, server_id in zip(statuses, args_answers, servers_id): if status != gs.MSG_STATUS_OK: raise gs.MSG_EXCEPTION_MAP[status]( "Server {} raised Exception:\n{}".format( server_id, args_answer[0])) return args_answers, kwds_answers
def call(*args, **kwargs): f = futures.Future() def set_exception(s): try: f.set_exception(translate_status(s)) except futures.InvalidStateError: # pytype: disable=module-attr # Call could have been already canceled by the user. pass canceller = self._client.AsyncPyCall(method, list(args), kwargs, f.set_result, set_exception, self._wait_for_ready, self._call_timeout, self._compress) def done_callback(f): if f.cancelled(): canceller.Cancel() f.add_done_callback(done_callback) return f
def do(self, _f, *args, **kwargs): fut = futures.Future() g_self = None def _wrapper(): nonlocal g_self g_self = greenlet.getcurrent() try: result = _f(*args, **kwargs) except BaseException as e: fut.set_exception(e) else: fut.set_result(result) self.loop.call_soon_threadsafe( asyncio.async, green.coroutine(_wrapper)()) try: return fut.result() except BaseException as e: self.loop.call_soon_threadsafe(g_self.throw, e) raise
def test_execute_get_batch(self, mock_get_item, mock_repo_get, mock_validate_table_is_active, mock_validate_table_schema): future = futures.Future() future.set_result(True) mock_get_item.return_value = future context = mock.Mock(tenant='fake_tenant') table_name = 'fake_table' request_list = [ models.GetItemRequest( table_name, { 'id': models.AttributeValue('N', 1), 'str': models.AttributeValue('S', 'str1'), }, None, True), models.GetItemRequest( table_name, { 'id': models.AttributeValue('N', 1), 'str': models.AttributeValue('S', 'str2'), }, None, True) ] expected_get = [ mock.call(context, req.table_name, req.key_attribute_map, req.attributes_to_get, req.consistent) for req in request_list ] storage_manager = simple_impl.SimpleStorageManager( None, table_info_repo.TableInfoRepository()) result, unprocessed_items = storage_manager.execute_get_batch( context, request_list) mock_get_item.has_calls(expected_get) self.assertEqual(unprocessed_items, [])
def request(self, method, params=None): """Send a JSON RPC request to the client. Args: method (str): The method name of the message to send params (any): The payload of the message Returns: Future that will resolve once a response has been received """ msg_id = self._id_generator() log.debug("Sending request with id %s: %s %s", msg_id, method, params) message = {"jsonrpc": JSONRPC_VERSION, "id": msg_id, "method": method} if params is not None: message["params"] = params request_future = futures.Future() request_future.add_done_callback(self._cancel_callback(msg_id)) self._server_request_futures[msg_id] = request_future self._consumer(message) return request_future
def wait_for(fut, timeout, *, loop=None): """Wait for the single Future or coroutine to complete, with timeout. Coroutine will be wrapped in Task. Returns result of the Future or coroutine. When a timeout occurs, it cancels the task and raises TimeoutError. To avoid the task cancellation, wrap it in shield(). Usage: result = yield from asyncio.wait_for(fut, 10.0) """ if loop is None: loop = events.get_event_loop() if timeout is None: return (yield from fut) waiter = futures.Future(loop=loop) timeout_handle = loop.call_later(timeout, _release_waiter, waiter, False) cb = functools.partial(_release_waiter, waiter, True) fut = async(fut, loop=loop) fut.add_done_callback(cb) try: if (yield from waiter): return fut.result() else: fut.remove_done_callback(cb) fut.cancel() raise futures.TimeoutError() finally: timeout_handle.cancel()
def __init__(self, frame_id, expression): from concurrent import futures self.frame_id = frame_id self.expression = expression self.future = futures.Future()
def submit(self, function): future = futures.Future() future.set_result(function()) return future
def gather(*coros_or_futures, loop=None, return_exceptions=False): """Return a future aggregating results from the given coroutines or futures. All futures must share the same event loop. If all the tasks are done successfully, the returned future's result is the list of results (in the order of the original sequence, not necessarily the order of results arrival). If *return_exceptions* is True, exceptions in the tasks are treated the same as successful results, and gathered in the result list; otherwise, the first raised exception will be immediately propagated to the returned future. Cancellation: if the outer Future is cancelled, all children (that have not completed yet) are also cancelled. If any child is cancelled, this is treated as if it raised CancelledError -- the outer Future is *not* cancelled in this case. (This is to prevent the cancellation of one child to cause other children to be cancelled.) """ if not coros_or_futures: outer = futures.Future(loop=loop) outer.set_result([]) return outer arg_to_fut = {} for arg in set(coros_or_futures): if not isinstance(arg, futures.Future): fut = ensure_future(arg, loop=loop) if loop is None: loop = fut._loop # The caller cannot control this future, the "destroy pending task" # warning should not be emitted. fut._log_destroy_pending = False else: fut = arg if loop is None: loop = fut._loop elif fut._loop is not loop: raise ValueError("futures are tied to different event loops") arg_to_fut[arg] = fut children = [arg_to_fut[arg] for arg in coros_or_futures] nchildren = len(children) outer = _GatheringFuture(children, loop=loop) nfinished = 0 results = [None] * nchildren def _done_callback(i, fut): nonlocal nfinished if outer.done(): if not fut.cancelled(): # Mark exception retrieved. fut.exception() return if fut.cancelled(): res = futures.CancelledError() if not return_exceptions: outer.set_exception(res) return elif fut._exception is not None: res = fut.exception() # Mark exception retrieved. if not return_exceptions: outer.set_exception(res) return else: res = fut._result results[i] = res nfinished += 1 if nfinished == nchildren: outer.set_result(results) for i, fut in enumerate(children): fut.add_done_callback(functools.partial(_done_callback, i)) return outer
def _new_future(self): return futures.Future(loop=_FakeEventLoop())
def test_tb_logger_abandoned(self, m_log): fut = futures.Future(loop=self.loop) del fut self.assertFalse(m_log.error.called)
def test_initial_state(self): f = futures.Future(loop=self.loop) self.assertFalse(f.cancelled()) self.assertFalse(f.done()) f.cancel() self.assertTrue(f.cancelled())
def test_wrap_future_future(self): f1 = futures.Future(loop=self.loop) f2 = futures.wrap_future(f1) self.assertIs(f1, f2)
def test_tb_logger_exception_result_retrieved(self, m_log): fut = futures.Future(loop=self.loop) fut.set_exception(RuntimeError('boom')) self.assertRaises(RuntimeError, fut.result) del fut self.assertFalse(m_log.error.called)
def test_tb_logger_exception_unretrieved(self, m_log): fut = futures.Future(loop=self.loop) fut.set_exception(RuntimeError('boom')) del fut test_utils.run_briefly(self.loop) self.assertTrue(m_log.error.called)
def test_tb_logger_result_retrieved(self, m_log): fut = futures.Future(loop=self.loop) fut.set_result(42) fut.result() del fut self.assertFalse(m_log.error.called)