def test_yield_expression_result(self): @coroutine def yield_expression_coroutine(): for i in range(3): x = yield asyncio.sleep(0, result=i) self.assertEqual(x, i) asyncio.get_event_loop().run_until_complete(yield_expression_coroutine())
def test_yield_expression_result(self): @coroutine def yield_expression_coroutine(): for i in range(3): x = yield asyncio.sleep(0, result=i) self.assertEqual(x, i) asyncio.get_event_loop().run_until_complete(yield_expression_coroutine())
def test_returning_coroutine(self):
@coroutine
def returning_coroutine(loop=None):
yield asyncio.sleep(0, loop=loop)
coroutine_return('success')
loop = asyncio.get_event_loop()
self.assertEqual(
'success',
asyncio.get_event_loop().run_until_complete(
returning_coroutine(loop=loop)))
def testChildWatcher(self):
true_binary = find_binary("true")
self.assertNotEqual(true_binary, None)
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
try:
try:
asyncio.set_child_watcher(None)
except NotImplementedError:
pass
else:
self.assertTrue(False)
args_tuple = ('hello', 'world')
loop = asyncio.get_event_loop()
future = loop.create_future()
def callback(pid, returncode, *args):
future.set_result((pid, returncode, args))
with asyncio.get_child_watcher() as watcher:
pids = spawn([true_binary], returnpid=True)
watcher.add_child_handler(pids[0], callback, *args_tuple)
self.assertEqual(
loop.run_until_complete(future),
(pids[0], os.EX_OK, args_tuple))
finally:
asyncio.set_event_loop_policy(initial_policy)
def _do_test(self, read_end, write_end):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = asyncio.get_event_loop()
read_end = os.fdopen(read_end, 'rb', 0)
write_end = os.fdopen(write_end, 'wb', 0)
try:
def reader_callback():
if not reader_callback.called.done():
reader_callback.called.set_result(None)
reader_callback.called = loop.create_future()
loop.add_reader(read_end.fileno(), reader_callback)
# Allow the loop to check for IO events, and assert
# that our future is still not done.
loop.run_until_complete(asyncio.sleep(0, loop=loop))
self.assertFalse(reader_callback.called.done())
# Demonstrate that the callback is called afer the
# other end of the pipe has been closed.
write_end.close()
loop.run_until_complete(reader_callback.called)
finally:
loop.remove_reader(read_end.fileno())
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
async def run(self):
self.expected = getattr(self, "expected", None) or {"returncode": 0}
if self.debug:
fd_pipes = {}
pr = None
pw = None
else:
pr, pw = os.pipe()
fd_pipes = {1: pw, 2: pw}
pr = open(pr, "rb", 0)
proc = AsyncFunction(
scheduler=asyncio.get_event_loop(),
target=self._subprocess,
args=(self.args, self.cwd, self.env, self.expected, self.debug),
fd_pipes=fd_pipes,
)
proc.start()
if pw is not None:
os.close(pw)
await proc.async_wait()
if pr is None:
stdio = None
else:
stdio = await _reader(pr)
self.result = {
"stdio": stdio,
"result": proc.result,
}
def test_returning_coroutine(self):
@coroutine
def returning_coroutine():
yield asyncio.sleep(0)
coroutine_return('success')
self.assertEqual('success',
asyncio.get_event_loop().run_until_complete(returning_coroutine()))
def _run_test(self, test):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
try:
test(asyncio.get_event_loop())
finally:
asyncio.set_event_loop_policy(initial_policy)
def test_raising_coroutine(self):
class TestException(Exception):
pass
@coroutine
def raising_coroutine(loop=None):
yield asyncio.sleep(0, loop=loop)
raise TestException('exception')
loop = asyncio.get_event_loop()
self.assertRaises(TestException, loop.run_until_complete,
raising_coroutine(loop=loop))
def test_raising_coroutine(self):
class TestException(Exception):
pass
@coroutine
def raising_coroutine():
yield asyncio.sleep(0)
raise TestException('exception')
self.assertRaises(TestException,
asyncio.get_event_loop().run_until_complete, raising_coroutine())
def test_cancelled_coroutine(self):
@coroutine
def cancelled_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
yield loop.create_future()
loop = asyncio.get_event_loop()
future = cancelled_coroutine(loop=loop)
loop.call_soon(future.cancel)
self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
def check(self, **kwargs):
"""Perform profile dependant dependency checks
@param pkg: Package in which we check (object).
@param ebuild: Ebuild which we check (object).
@returns: dictionary
"""
ebuild = kwargs.get("ebuild").get()
pkg = kwargs.get("pkg").get()
ebuild.unknown_pkgs, ebuild.baddepsyntax = _depend_checks(
ebuild,
pkg,
self.portdb,
self.qatracker,
self.repo_metadata,
self.repo_settings.qadata,
)
relevant_profiles = []
for keyword, arch, groups in _gen_arches(ebuild, self.options,
self.repo_settings,
self.profiles):
if arch not in self.profiles:
# A missing profile will create an error further down
# during the KEYWORDS verification.
continue
if self.include_arches is not None:
if arch not in self.include_arches:
continue
for prof in self.profiles[arch]:
if self.include_profiles is not None:
if prof.sub_path not in self.include_profiles:
continue
relevant_profiles.append((keyword, groups, prof))
relevant_profiles.sort(key=sort_key)
ebuild.relevant_profiles = relevant_profiles
if self.options.jobs <= 1:
for task in self._iter_tasks(None, None, ebuild, pkg):
task, results = task
for result in results:
self._check_result(task, result)
loop = asyncio.get_event_loop()
loop.run_until_complete(self._async_check(loop, **kwargs))
return False
def test_cancelled_coroutine(self): @coroutine def cancelled_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: yield loop.create_future() loop = asyncio.get_event_loop() future = cancelled_coroutine(loop=loop) loop.call_soon(future.cancel) self.assertRaises(asyncio.CancelledError, loop.run_until_complete, future)
def testEventLoopInForkTestCase(self): initial_policy = asyncio.get_event_loop_policy() if not isinstance(initial_policy, DefaultEventLoopPolicy): asyncio.set_event_loop_policy(DefaultEventLoopPolicy()) try: loop = asyncio.get_event_loop() fork_exitcode = loop.create_future() # Make async_main fork while the loop is running, which would # trigger https://bugs.python.org/issue22087 with asyncio's # default event loop policy. loop.call_soon(async_main, fork_exitcode) assert loop.run_until_complete(fork_exitcode) == os.EX_OK finally: asyncio.set_event_loop_policy(initial_policy)
async def _check_call(self, cmd):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
"""
p = SpawnProcess(args=cmd,
scheduler=asyncio.get_event_loop(),
**self._spawn_kwargs)
p.start()
if await p.async_wait() != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def reader(input_file, loop=None):
"""
Asynchronously read a binary input file.
@param input_file: binary input file
@type input_file: file
@param loop: event loop
@type loop: EventLoop
@return: bytes
@rtype: asyncio.Future (or compatible)
"""
loop = loop or asyncio.get_event_loop()
loop = getattr(loop, '_asyncio_wrapper', loop)
future = loop.create_future()
_Reader(future, input_file, loop)
return future
def test_cancelled_future(self): """ When a coroutine raises CancelledError, the coroutine's future is cancelled. """ @coroutine def cancelled_future_coroutine(loop=None): loop = asyncio._wrap_loop(loop) while True: future = loop.create_future() loop.call_soon(future.cancel) yield future loop = asyncio.get_event_loop() future = loop.run_until_complete(asyncio.wait([cancelled_future_coroutine()]))[0].pop() self.assertTrue(future.cancelled())
def test_cancelled_future(self):
"""
When a coroutine raises CancelledError, the coroutine's
future is cancelled.
"""
@coroutine
def cancelled_future_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
future = loop.create_future()
loop.call_soon(future.cancel)
yield future
loop = asyncio.get_event_loop()
future = loop.run_until_complete(
asyncio.wait([cancelled_future_coroutine()]))[0].pop()
self.assertTrue(future.cancelled())
def async_main(fork_exitcode, loop=None): loop = loop or asyncio.get_event_loop() # Since python2.7 does not support Process.sentinel, use Pipe to # monitor for process exit. parent_conn, child_conn = multiprocessing.Pipe() def eof_callback(proc): loop.remove_reader(parent_conn.fileno()) parent_conn.close() proc.join() fork_exitcode.set_result(proc.exitcode) proc = multiprocessing.Process(target=fork_main, args=(parent_conn, child_conn)) loop.add_reader(parent_conn.fileno(), eof_callback, proc) proc.start() child_conn.close()
def _do_test(self, read_end, write_end):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
loop = asyncio.get_event_loop()
read_end = os.fdopen(read_end, 'rb', 0)
write_end = os.fdopen(write_end, 'wb', 0)
try:
def writer_callback():
if not writer_callback.called.done():
writer_callback.called.set_result(None)
writer_callback.called = loop.create_future()
_set_nonblocking(write_end.fileno())
loop.add_writer(write_end.fileno(), writer_callback)
# Fill up the pipe, so that no writer callbacks should be
# received until the state has changed.
while True:
try:
os.write(write_end.fileno(), 512 * b'0')
except EnvironmentError as e:
if e.errno != errno.EAGAIN:
raise
break
# Allow the loop to check for IO events, and assert
# that our future is still not done.
loop.run_until_complete(asyncio.sleep(0, loop=loop))
self.assertFalse(writer_callback.called.done())
# Demonstrate that the callback is called afer the
# other end of the pipe has been closed.
read_end.close()
loop.run_until_complete(writer_callback.called)
finally:
loop.remove_writer(write_end.fileno())
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
async def _check_call(self, cmd, privileged=False):
"""
Run cmd and raise RepoStorageException on failure.
@param cmd: command to executre
@type cmd: list
@param privileged: run with maximum privileges
@type privileged: bool
"""
if privileged:
kwargs = dict(fd_pipes=self._spawn_kwargs.get('fd_pipes'))
else:
kwargs = self._spawn_kwargs
p = SpawnProcess(args=cmd,
scheduler=asyncio.get_event_loop(),
**kwargs)
p.start()
if await p.async_wait() != os.EX_OK:
raise RepoStorageException('command exited with status {}: {}'.\
format(p.returncode, ' '.join(cmd)))
def test_catching_coroutine(self):
class TestException(Exception):
pass
@coroutine
def catching_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
loop.call_soon(future.set_exception, TestException('exception'))
try:
yield future
except TestException:
self.assertTrue(True)
else:
self.assertTrue(False)
coroutine_return('success')
loop = asyncio.get_event_loop()
self.assertEqual(
'success', loop.run_until_complete(catching_coroutine(loop=loop)))
def test_add_done_callback(self):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
try:
loop = asyncio.get_event_loop()
f1 = loop.create_future()
f2 = loop.create_future()
f1.add_done_callback(f2.set_result)
loop.call_soon(lambda: f1.set_result(None))
loop.run_until_complete(f1)
self.assertEqual(f1.done(), True)
# This proves that done callbacks of f1 are executed before
# loop.run_until_complete(f1) returns, which is how asyncio's
# default event loop behaves.
self.assertEqual(f2.done(), True)
finally:
asyncio.set_event_loop_policy(initial_policy)
def test_catching_coroutine(self):
class TestException(Exception):
pass
@coroutine
def catching_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
future = loop.create_future()
loop.call_soon(future.set_exception, TestException('exception'))
try:
yield future
except TestException:
self.assertTrue(True)
else:
self.assertTrue(False)
coroutine_return('success')
loop = asyncio.get_event_loop()
self.assertEqual('success',
loop.run_until_complete(catching_coroutine(loop=loop)))
def test_cancelled_coroutine(self):
"""
Verify that a coroutine can handle (and reraise) asyncio.CancelledError
in order to perform any necessary cleanup. Note that the
asyncio.CancelledError will only be thrown in the coroutine if there's
an opportunity (yield) before the generator raises StopIteration.
"""
loop = asyncio.get_event_loop()
ready_for_exception = loop.create_future()
exception_in_coroutine = loop.create_future()
@coroutine
def cancelled_coroutine(loop=None):
loop = asyncio._wrap_loop(loop)
while True:
task = loop.create_future()
try:
ready_for_exception.set_result(None)
yield task
except BaseException as e:
# Since python3.8, asyncio.CancelledError inherits
# from BaseException.
task.done() or task.cancel()
exception_in_coroutine.set_exception(e)
raise
else:
exception_in_coroutine.set_result(None)
future = cancelled_coroutine(loop=loop)
loop.run_until_complete(ready_for_exception)
future.cancel()
self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
future)
self.assertRaises(asyncio.CancelledError, loop.run_until_complete,
exception_in_coroutine)
def fork_main(parent_conn, child_conn): parent_conn.close() loop = asyncio.get_event_loop() # This fails with python's default event loop policy, # see https://bugs.python.org/issue22087. loop.run_until_complete(asyncio.sleep(0.1))
async def _test_content_db(self, emirrordist_cmd, env, layouts, content_db,
distfiles, settings, portdb):
# Simulate distfile digest change for ContentDB.
emdisopts = types.SimpleNamespace(content_db=content_db,
distfiles=settings["DISTDIR"])
with EmirrordistConfig(emdisopts, portdb,
asyncio.get_event_loop()) as emdisconf:
# Copy revisions from bar to foo.
for revision_key in emdisconf.content_db["filename:{}".format(
"bar")]:
emdisconf.content_db.add(
DistfileName("foo", digests=dict(revision_key)))
# Copy revisions from foo to bar.
for revision_key in emdisconf.content_db["filename:{}".format(
"foo")]:
emdisconf.content_db.add(
DistfileName("bar", digests=dict(revision_key)))
content_db_state = dict(emdisconf.content_db.items())
self.assertEqual(content_db_state,
dict(emdisconf.content_db.items()))
self.assertEqual(
[
k[len("filename:"):]
for k in content_db_state if k.startswith("filename:")
],
["bar", "foo"],
)
self.assertEqual(content_db_state["filename:foo"],
content_db_state["filename:bar"])
self.assertEqual(len(content_db_state["filename:foo"]), 2)
for k in distfiles:
try:
os.unlink(os.path.join(settings["DISTDIR"], k))
except OSError:
pass
proc = await asyncio.create_subprocess_exec(*emirrordist_cmd, env=env)
self.assertEqual(await proc.wait(), 0)
for k in distfiles:
with open(
os.path.join(settings["DISTDIR"], layouts[0].get_path(k)),
"rb") as f:
self.assertEqual(f.read(), distfiles[k])
with EmirrordistConfig(emdisopts, portdb,
asyncio.get_event_loop()) as emdisconf:
self.assertEqual(content_db_state,
dict(emdisconf.content_db.items()))
# Verify that remove works as expected
filename = [
filename for filename in distfiles if filename == "foo"
][0]
self.assertTrue(bool(filename.digests))
emdisconf.content_db.remove(filename)
# foo should still have a content revision corresponding to bar's content.
self.assertEqual(
[
k[len("filename:"):]
for k in emdisconf.content_db if k.startswith("filename:")
],
["bar", "foo"],
)
self.assertEqual(len(emdisconf.content_db["filename:foo"]), 1)
self.assertEqual(
len([
revision_key
for revision_key in emdisconf.content_db["filename:foo"]
if not filename.digests_equal(
DistfileName(
"foo",
digests=dict(revision_key),
))
]),
1,
)
# bar should still have a content revision corresponding to foo's content.
self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2)
self.assertEqual(
len([
revision_key
for revision_key in emdisconf.content_db["filename:bar"]
if filename.digests_equal(
DistfileName(
"bar",
digests=dict(revision_key),
))
]),
1,
)
# remove the foo which refers to bar's content
bar = [filename for filename in distfiles if filename == "bar"][0]
foo_remaining = DistfileName("foo", digests=bar.digests)
emdisconf.content_db.remove(foo_remaining)
self.assertEqual(
[
k[len("filename:"):]
for k in emdisconf.content_db if k.startswith("filename:")
],
["bar"],
)
self.assertRaises(KeyError, emdisconf.content_db.__getitem__,
"filename:foo")
# bar should still have a content revision corresponding to foo's content.
self.assertEqual(len(emdisconf.content_db["filename:bar"]), 2)
def test_method_coroutine(self):
class Cubby:
_empty = object()
def __init__(self, loop):
self._loop = loop
self._value = self._empty
self._waiters = []
def _notify(self):
waiters = self._waiters
self._waiters = []
for waiter in waiters:
waiter.cancelled() or waiter.set_result(None)
def _wait(self):
waiter = self._loop.create_future()
self._waiters.append(waiter)
return waiter
@coroutine
def read(self, loop=None):
while self._value is self._empty:
yield self._wait()
value = self._value
self._value = self._empty
self._notify()
coroutine_return(value)
@coroutine
def write(self, value, loop=None):
while self._value is not self._empty:
yield self._wait()
self._value = value
self._notify()
@coroutine
def writer_coroutine(cubby, values, sentinel, loop=None):
for value in values:
yield cubby.write(value, loop=loop)
yield cubby.write(sentinel, loop=loop)
@coroutine
def reader_coroutine(cubby, sentinel, loop=None):
results = []
while True:
result = yield cubby.read(loop=loop)
if result == sentinel:
break
results.append(result)
coroutine_return(results)
loop = asyncio.get_event_loop()
cubby = Cubby(loop)
values = list(range(3))
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
reader = asyncio.ensure_future(reader_coroutine(cubby, None,
loop=loop),
loop=loop)
loop.run_until_complete(asyncio.wait([writer, reader], loop=loop))
self.assertEqual(reader.result(), values)
# Test decoration of coroutine methods and functions for
# synchronous usage, allowing coroutines to smoothly
# blend with synchronous code.
sync_cubby = _sync_methods(cubby, loop=loop)
sync_reader = _sync_decorator(reader_coroutine, loop=loop)
writer = asyncio.ensure_future(writer_coroutine(cubby,
values,
None,
loop=loop),
loop=loop)
self.assertEqual(sync_reader(cubby, None), values)
self.assertTrue(writer.done())
for i in range(3):
sync_cubby.write(i)
self.assertEqual(sync_cubby.read(), i)
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values)
def test_method_coroutine(self): class Cubby(object): _empty = object() def __init__(self, loop): self._loop = loop self._value = self._empty self._waiters = [] def _notify(self): waiters = self._waiters self._waiters = [] for waiter in waiters: waiter.cancelled() or waiter.set_result(None) def _wait(self): waiter = self._loop.create_future() self._waiters.append(waiter) return waiter @coroutine def read(self): while self._value is self._empty: yield self._wait() value = self._value self._value = self._empty self._notify() coroutine_return(value) @coroutine def write(self, value): while self._value is not self._empty: yield self._wait() self._value = value self._notify() @coroutine def writer_coroutine(cubby, values, sentinel): for value in values: yield cubby.write(value) yield cubby.write(sentinel) @coroutine def reader_coroutine(cubby, sentinel): results = [] while True: result = yield cubby.read() if result == sentinel: break results.append(result) coroutine_return(results) loop = asyncio.get_event_loop() cubby = Cubby(loop) values = list(range(3)) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) reader = asyncio.ensure_future(reader_coroutine(cubby, None), loop=loop) loop.run_until_complete(asyncio.wait([writer, reader])) self.assertEqual(reader.result(), values) # Test decoration of coroutine methods and functions for # synchronous usage, allowing coroutines to smoothly # blend with synchronous code. sync_cubby = _sync_methods(cubby, loop=loop) sync_reader = _sync_decorator(reader_coroutine, loop=loop) writer = asyncio.ensure_future(writer_coroutine(cubby, values, None), loop=loop) self.assertEqual(sync_reader(cubby, None), values) self.assertTrue(writer.done()) for i in range(3): sync_cubby.write(i) self.assertEqual(sync_cubby.read(), i)
