def run_in_executor(self, executor, callback, *args):
if (coroutines.iscoroutine(callback)
or coroutines.iscoroutinefunction(callback)):
raise TypeError("coroutines cannot be used with run_in_executor()")
if isinstance(callback, events.Handle):
assert not args
assert not isinstance(callback, events.TimerHandle)
if callback._cancelled:
f = futures.Future(loop=self)
f.set_result(None)
return f
callback, args = callback._callback, callback._args
if executor is None:
executor = self._default_executor
if executor is None:
executor = concurrent.futures.ThreadPoolExecutor(_MAX_WORKERS)
self._default_executor = executor
return futures.wrap_future(executor.submit(callback, *args), loop=self)
def _create_connection_transport(self, sock, protocol_factory, ssl,
server_hostname):
protocol = protocol_factory()
waiter = futures.Future(loop=self)
if ssl:
sslcontext = None if isinstance(ssl, bool) else ssl
transport = self._make_ssl_transport(
sock,
protocol,
sslcontext,
waiter,
server_side=False,
server_hostname=server_hostname)
else:
transport = self._make_socket_transport(sock, protocol, waiter)
yield from waiter
return transport, protocol
def run(self, funcs):
if self._workers:
raise RuntimeError("Can not start another `run` with %s"
" existing workers" % (len(self._workers)))
self._queue = compat_queue.Queue()
self._death.clear()
futs = []
for i in compat_range(0, self._max_workers):
w = threading.Thread(target=_chained_worker,
args=(i + 1, self._death, self._queue, futs))
w.daemon = True
w.start()
self._workers.append(w)
for func in funcs:
fut = futures.Future()
futs.append(fut)
self._queue.put((func, fut))
return futs
def run_thread_and_sync(self):
"""Start the server in a separate thread and wait for notification that it has
successfuly started"""
# create future so that we can sync with the IO loop running inside the
# thread we started
sync = futures.Future()
self.thread = threading.Thread(target=self._start_loop_thread,
args=(sync, ))
self.thread.start()
# wait for the loop to be stated
futures.wait([sync])
self.log.info('test server thread started')
return self
def wait_for(fut, timeout, *, loop=None):
"""Wait for the single Future or coroutine to complete, with timeout.
Coroutine will be wrapped in Task.
Returns result of the Future or coroutine. When a timeout occurs,
it cancels the task and raises TimeoutError. To avoid the task
cancellation, wrap it in shield().
If the wait is cancelled, the task is also cancelled.
This function is a coroutine.
"""
if loop is None:
loop = events.get_event_loop()
if timeout is None:
return (yield from fut)
waiter = futures.Future(loop=loop)
timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
cb = functools.partial(_release_waiter, waiter)
fut = ensure_future(fut, loop=loop)
fut.add_done_callback(cb)
try:
# wait until the future completes or the timeout
try:
yield from waiter
except futures.CancelledError:
fut.remove_done_callback(cb)
fut.cancel()
raise
if fut.done():
return fut.result()
else:
fut.remove_done_callback(cb)
fut.cancel()
raise futures.TimeoutError()
finally:
timeout_handle.cancel()
def mock_submit(unused_fn,
chunk_size: int,
hard_limit: Optional[int] = None):
target_valid = chunk_size
target_total = int((1 + failure_rate) * chunk_size)
if hard_limit is not None and target_total > hard_limit:
target_valid = ceil(chunk_size * (hard_limit / target_total))
target_total = hard_limit
target_invalid = target_total - target_valid
lines = [
GenText(text=f'line-{i}', valid=(i >= target_invalid))
for i in range(target_total)
]
mock_future = futures.Future()
mock_future.set_result((chunk_size, lines, target_invalid))
return mock_future
def run_coroutine_threadsafe(coro, loop):
"""Submit a coroutine object to a given event loop.
Return a concurrent_futures.Future to access the result.
"""
if not coroutines.iscoroutine(coro):
raise TypeError('A coroutine object is required')
future = concurrent_futures.Future()
def callback():
try:
futures._chain_future(ensure_future(coro, loop=loop), future)
except Exception as exc:
if future.set_running_or_notify_cancel():
future.set_exception(exc)
raise
loop.call_soon_threadsafe(callback)
return future
def create_specific_server(self, client_id: str) -> SpecificServer:
"""
Create, but not start a SpecificServer for a given client. This
method must be called once per client.
"""
with self.server_lock:
assert (
self.servers.get(client_id) is None
), f"Server already created for Client: {client_id}"
port = self._get_unused_port()
server = SpecificServer(
port=port,
process_handle_future=futures.Future(),
channel=ray._private.utils.init_grpc_channel(
f"127.0.0.1:{port}", options=GRPC_OPTIONS
),
)
self.servers[client_id] = server
return server
def submit(self, fun, *args, additional_setup_lines=[], **kwargs):
"""Submit a job to the pool."""
fut = futures.Future()
# Start the job.
workerid = random_string()
funcser = cloudpickle.dumps((fun, args, kwargs), True)
with open(INFILE_FMT % workerid, 'wb') as f:
f.write(funcser)
jobid = self._start(workerid, additional_setup_lines)
if self.debug:
print("job submitted: %i" % jobid, file=sys.stderr)
# Thread will wait for it to finish.
self.wait_thread.wait(OUTFILE_FMT % workerid, jobid)
with self.jobs_lock:
self.jobs[jobid] = (fut, workerid)
return fut
def submit(self, fun, *args, **kwargs):
"""Submit a job to the pool."""
fut = futures.Future()
# Start the job.
workerid = random_string()
funcser = serialization.serialize((fun, args, kwargs), True)
with open(INFILE_FMT % workerid, 'w') as f:
f.write(funcser)
jobid = self._start(workerid)
if self.debug:
print >> sys.stderr, "job submitted: %i" % jobid
# Thread will wait for it to finish.
self.wait_thread.wait(OUTFILE_FMT % workerid, jobid)
with self.jobs_lock:
self.jobs[jobid] = (fut, workerid)
return fut
def _wait(fs, timeout, return_when, loop):
"""Internal helper for wait() and _wait_for().
The fs argument must be a collection of Futures.
"""
assert fs, 'Set of Futures is empty.'
waiter = futures.Future(loop=loop)
timeout_handle = None
if timeout is not None:
timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
counter = len(fs)
def _on_completion(f):
nonlocal counter
counter -= 1
if (counter <= 0 or
return_when == FIRST_COMPLETED or
return_when == FIRST_EXCEPTION and (not f.cancelled() and
f.exception() is not None)):
if timeout_handle is not None:
timeout_handle.cancel()
if not waiter.done():
waiter.set_result(None)
for f in fs:
f.add_done_callback(_on_completion)
try:
yield from waiter
finally:
if timeout_handle is not None:
timeout_handle.cancel()
done, pending = set(), set()
for f in fs:
f.remove_done_callback(_on_completion)
if f.done():
done.add(f)
else:
pending.add(f)
return done, pending
def _dchain(pool, futures, fn):
result = cf.Future()
bnum = len(futures)
def callback(_):
nonlocal bnum
bnum -= 1
if bnum > 0:
return
try:
results = [f.result() for f in futures]
temp = pool.submit(fn, *results)
copy = lambda _: _copy_future_state(temp, result)
temp.add_done_callback(copy)
except:
result.cancel()
raise
for f in futures:
f.add_done_callback(callback)
return result
def test_futures_added_is_triggered_and_pipelined(self):
inc1 = mock.MagicMock()
inc1.side_effect = inc
embedded = futures.Future()
embedded.add_done_callback(inc1)
unit = PipedFuture()
unit.add_done_callback(inc1)
unit.add_done_future(embedded)
unit.add_done_callback(inc1)
unit.set_result(0)
self.assertEqual(unit.result(), 2)
# initial result is the pipeline result until that point, final result
# is still its initial result since it's a standard Future instance.
self.assertEqual(embedded.result(), 1)
inc1.assert_has_calls([mock.call(unit),
mock.call(embedded),
mock.call(unit)])
def _batch_write_async(self, context, write_request_list):
future_result = futures.Future()
batch_future = self._execute_async(self._storage_driver.batch_write,
context, write_request_list)
def callback(res):
try:
res.result()
unprocessed_items = ()
except NotImplementedError:
unprocessed_items = self._batch_write_in_emulation_mode(
context, write_request_list)
except Exception:
LOG.exception("Can't process batch write request")
unprocessed_items = write_request_list
future_result.set_result(unprocessed_items)
batch_future.add_done_callback(callback)
return future_result
def send_message(self, servers_id, cmd, args=(), kwds={}):
"""Send a message to (possibly) multiple servers.
The same message is sent to all servers.
"""
loop = ioloop.IOLoop.instance()
if type(servers_id) not in (tuple, list):
servers_id = [servers_id]
future_list = []
for server_id in servers_id:
future = futures.Future()
self.futures[server_id] = future
future_list.append(future)
loop.add_callback(self.client_instance.send,
server_address=server_id,
cmd=cmd,
args=args,
kwds=kwds)
results = []
for future in future_list:
results.append(future.result(timeout=self.timeout))
statuses, cmds, args_answers, kwds_answers = zip(*results)
#
# Check the reply status
#
for status, args_answer, server_id in zip(statuses, args_answers,
servers_id):
if status != gs.MSG_STATUS_OK:
raise gs.MSG_EXCEPTION_MAP[status](
"Server {} raised Exception:\n{}".format(
server_id, args_answer[0]))
return args_answers, kwds_answers
def call(*args, **kwargs):
f = futures.Future()
def set_exception(s):
try:
f.set_exception(translate_status(s))
except futures.InvalidStateError: # pytype: disable=module-attr
# Call could have been already canceled by the user.
pass
canceller = self._client.AsyncPyCall(method, list(args), kwargs,
f.set_result, set_exception,
self._wait_for_ready,
self._call_timeout,
self._compress)
def done_callback(f):
if f.cancelled():
canceller.Cancel()
f.add_done_callback(done_callback)
return f
def do(self, _f, *args, **kwargs):
fut = futures.Future()
g_self = None
def _wrapper():
nonlocal g_self
g_self = greenlet.getcurrent()
try:
result = _f(*args, **kwargs)
except BaseException as e:
fut.set_exception(e)
else:
fut.set_result(result)
self.loop.call_soon_threadsafe(
asyncio.async, green.coroutine(_wrapper)())
try:
return fut.result()
except BaseException as e:
self.loop.call_soon_threadsafe(g_self.throw, e)
raise
def test_execute_get_batch(self, mock_get_item, mock_repo_get,
mock_validate_table_is_active,
mock_validate_table_schema):
future = futures.Future()
future.set_result(True)
mock_get_item.return_value = future
context = mock.Mock(tenant='fake_tenant')
table_name = 'fake_table'
request_list = [
models.GetItemRequest(
table_name, {
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str1'),
}, None, True),
models.GetItemRequest(
table_name, {
'id': models.AttributeValue('N', 1),
'str': models.AttributeValue('S', 'str2'),
}, None, True)
]
expected_get = [
mock.call(context, req.table_name, req.key_attribute_map,
req.attributes_to_get, req.consistent)
for req in request_list
]
storage_manager = simple_impl.SimpleStorageManager(
None, table_info_repo.TableInfoRepository())
result, unprocessed_items = storage_manager.execute_get_batch(
context, request_list)
mock_get_item.has_calls(expected_get)
self.assertEqual(unprocessed_items, [])
def request(self, method, params=None):
"""Send a JSON RPC request to the client.
Args:
method (str): The method name of the message to send
params (any): The payload of the message
Returns:
Future that will resolve once a response has been received
"""
msg_id = self._id_generator()
log.debug("Sending request with id %s: %s %s", msg_id, method, params)
message = {"jsonrpc": JSONRPC_VERSION, "id": msg_id, "method": method}
if params is not None:
message["params"] = params
request_future = futures.Future()
request_future.add_done_callback(self._cancel_callback(msg_id))
self._server_request_futures[msg_id] = request_future
self._consumer(message)
return request_future
def wait_for(fut, timeout, *, loop=None):
"""Wait for the single Future or coroutine to complete, with timeout.
Coroutine will be wrapped in Task.
Returns result of the Future or coroutine. When a timeout occurs,
it cancels the task and raises TimeoutError. To avoid the task
cancellation, wrap it in shield().
Usage:
result = yield from asyncio.wait_for(fut, 10.0)
"""
if loop is None:
loop = events.get_event_loop()
if timeout is None:
return (yield from fut)
waiter = futures.Future(loop=loop)
timeout_handle = loop.call_later(timeout, _release_waiter, waiter, False)
cb = functools.partial(_release_waiter, waiter, True)
fut = async(fut, loop=loop)
fut.add_done_callback(cb)
try:
if (yield from waiter):
return fut.result()
else:
fut.remove_done_callback(cb)
fut.cancel()
raise futures.TimeoutError()
finally:
timeout_handle.cancel()
def __init__(self, frame_id, expression):
from concurrent import futures
self.frame_id = frame_id
self.expression = expression
self.future = futures.Future()
def submit(self, function):
future = futures.Future()
future.set_result(function())
return future
def gather(*coros_or_futures, loop=None, return_exceptions=False):
"""Return a future aggregating results from the given coroutines
or futures.
All futures must share the same event loop. If all the tasks are
done successfully, the returned future's result is the list of
results (in the order of the original sequence, not necessarily
the order of results arrival). If *return_exceptions* is True,
exceptions in the tasks are treated the same as successful
results, and gathered in the result list; otherwise, the first
raised exception will be immediately propagated to the returned
future.
Cancellation: if the outer Future is cancelled, all children (that
have not completed yet) are also cancelled. If any child is
cancelled, this is treated as if it raised CancelledError --
the outer Future is *not* cancelled in this case. (This is to
prevent the cancellation of one child to cause other children to
be cancelled.)
"""
if not coros_or_futures:
outer = futures.Future(loop=loop)
outer.set_result([])
return outer
arg_to_fut = {}
for arg in set(coros_or_futures):
if not isinstance(arg, futures.Future):
fut = ensure_future(arg, loop=loop)
if loop is None:
loop = fut._loop
# The caller cannot control this future, the "destroy pending task"
# warning should not be emitted.
fut._log_destroy_pending = False
else:
fut = arg
if loop is None:
loop = fut._loop
elif fut._loop is not loop:
raise ValueError("futures are tied to different event loops")
arg_to_fut[arg] = fut
children = [arg_to_fut[arg] for arg in coros_or_futures]
nchildren = len(children)
outer = _GatheringFuture(children, loop=loop)
nfinished = 0
results = [None] * nchildren
def _done_callback(i, fut):
nonlocal nfinished
if outer.done():
if not fut.cancelled():
# Mark exception retrieved.
fut.exception()
return
if fut.cancelled():
res = futures.CancelledError()
if not return_exceptions:
outer.set_exception(res)
return
elif fut._exception is not None:
res = fut.exception() # Mark exception retrieved.
if not return_exceptions:
outer.set_exception(res)
return
else:
res = fut._result
results[i] = res
nfinished += 1
if nfinished == nchildren:
outer.set_result(results)
for i, fut in enumerate(children):
fut.add_done_callback(functools.partial(_done_callback, i))
return outer
def _new_future(self):
return futures.Future(loop=_FakeEventLoop())
def test_tb_logger_abandoned(self, m_log):
fut = futures.Future(loop=self.loop)
del fut
self.assertFalse(m_log.error.called)
def test_initial_state(self):
f = futures.Future(loop=self.loop)
self.assertFalse(f.cancelled())
self.assertFalse(f.done())
f.cancel()
self.assertTrue(f.cancelled())
def test_wrap_future_future(self):
f1 = futures.Future(loop=self.loop)
f2 = futures.wrap_future(f1)
self.assertIs(f1, f2)
def test_tb_logger_exception_result_retrieved(self, m_log):
fut = futures.Future(loop=self.loop)
fut.set_exception(RuntimeError('boom'))
self.assertRaises(RuntimeError, fut.result)
del fut
self.assertFalse(m_log.error.called)
def test_tb_logger_exception_unretrieved(self, m_log):
fut = futures.Future(loop=self.loop)
fut.set_exception(RuntimeError('boom'))
del fut
test_utils.run_briefly(self.loop)
self.assertTrue(m_log.error.called)
def test_tb_logger_result_retrieved(self, m_log):
fut = futures.Future(loop=self.loop)
fut.set_result(42)
fut.result()
del fut
self.assertFalse(m_log.error.called)
