def waitForEvent(emitter: EventEmitter, eventName: str, # noqa: C901
predicate: Callable[[Any], bool], timeout: float,
loop: asyncio.AbstractEventLoop) -> Awaitable:
"""Wait for an event emitted from the emitter."""
promise = loop.create_future()
def resolveCallback(target: Any) -> None:
promise.set_result(target)
def rejectCallback(exception: Exception) -> None:
promise.set_exception(exception)
async def timeoutTimer() -> None:
await asyncio.sleep(timeout / 1000)
rejectCallback(
TimeoutError('Timeout exceeded while waiting for event'))
def _listener(target: Any) -> None:
if not predicate(target):
return
cleanup()
resolveCallback(target)
listener = addEventListener(emitter, eventName, _listener)
if timeout:
eventTimeout = loop.create_task(timeoutTimer())
def cleanup() -> None:
removeEventListeners([listener])
if timeout:
eventTimeout.cancel()
return promise
def run_until_complete(coro: Awaitable, loop: AbstractEventLoop=None):
"""Run a task through to completion.
The ``.run_until_complete()`` method on asyncio event loop objects does not
finish tasks when it receives a SIGINT/CTRL-C. The method simply raises a
``KeyboardInterrupt`` exception and this usually results in warnings about
unfinished tasks plus some "event loop closed" ``RuntimeError`` exceptions
in pending tasks.
This is a really annoying default behavior and this function aims at
replacing that behavior with something that ensures the task actually runs
through to completion. When the ``KeyboardInterrupt`` exception is caught,
the task is canceled and resumed to give it a chance to clean up properly.
.. versionadded:: 0.4
.. versionchanged:: 0.5 Can now be called with a ``asyncio.Task`` argument.
"""
loop = loop or asyncio.get_event_loop()
if isinstance(coro, asyncio.Task):
task = coro
else:
task = loop.create_task(coro)
try:
loop.run_until_complete(task)
except KeyboardInterrupt:
task.cancel()
try:
loop.run_until_complete(task)
except asyncio.CancelledError:
return None
return task.result()
def start(
loop: asyncio.AbstractEventLoop, framerate: int = DEFAULT_FRAMERATE
) -> asyncio.Future:
return asyncio.gather(
loop.create_task(_ui_refresh(loop, framerate)),
loop.create_task(_process_inputs(loop)),
loop=loop,
)
def init_requirements(self, loop: asyncio.AbstractEventLoop):
getattr(super(), "init_requirements")(loop)
loop.run_until_complete(
self.connect_to_stream()
)
# Save to app object to be accessible in request handlers
self._server_instance.app.stream = self._stream
def set_timeout(task: asyncio.Task, timeout: [float, int], loop: asyncio.AbstractEventLoop = None, timeout_cancel=True):
assert isinstance(timeout, (float, int))
if loop is None:
loop = get_running_loop()
now_time = loop.time()
out_time = now_time + timeout
if timeout_cancel:
if timeout <= 0:
task.cancel()
return
unset_timeout(task)
handle = loop.call_at(out_time, task.cancel)
setattr(task, _MODULE_TIMEOUT_HANDLE, handle)
setattr(task, _MODULE_TIMEOUT, out_time)
async def get_address(self, loop: asyncio.AbstractEventLoop) -> SocketAddress:
server = ThriftServer(Handler(), port=0)
serve_task = loop.create_task(server.serve())
addy = await server.get_address()
server.stop()
await serve_task
return addy
def __init__(self, method: str, url: URL, *,
writer: 'asyncio.Task[None]',
continue100: Optional['asyncio.Future[bool]'],
timer: BaseTimerContext,
request_info: RequestInfo,
traces: List['Trace'],
loop: asyncio.AbstractEventLoop,
session: 'ClientSession') -> None:
assert isinstance(url, URL)
self.method = method
self.cookies = SimpleCookie()
self._real_url = url
self._url = url.with_fragment(None)
self._body = None # type: Any
self._writer = writer # type: Optional[asyncio.Task[None]]
self._continue = continue100 # None by default
self._closed = True
self._history = () # type: Tuple[ClientResponse, ...]
self._request_info = request_info
self._timer = timer if timer is not None else TimerNoop()
self._cache = {} # type: Dict[str, Any]
self._traces = traces
self._loop = loop
# store a reference to session #1985
self._session = session # type: Optional[ClientSession]
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
async def _scan(targets: Set[str],
ports: Set[int],
config: DockerScanModel,
loop: asyncio.AbstractEventLoop):
max_concurrency = asyncio.BoundedSemaphore(int(config.concurrency),
loop=loop)
results = []
tasks = []
for target in targets:
for port in ports:
await max_concurrency.acquire()
tasks.append(loop.create_task(_check_ports(
target,
port,
loop,
max_concurrency,
results,
config
)))
await asyncio.wait(tasks, loop=loop)
return results
async def total(loop: asyncio.AbstractEventLoop):
"""
asdf asd f asd f asd f asdf
.. note:
asdfasdfas df as df
>>> a = 1
>>> int(a)
1
@param loop: asdfasdf as df asdf
:param loop:
:type loop:
:return:
:rtype:
"""
t = []
sem = asyncio.Semaphore(10, loop=loop)
for x in range(1000):
await sem.acquire()
t.append(loop.create_task(hello(x, sem)))
return t
def __init__(
self,
name: str,
offset: Position,
size: Dimensions,
loop: asyncio.AbstractEventLoop,
actions: Dict[UIActionEnum, actions.Action] = None,
) -> None:
self.name = name
self.size = size
self.offset = offset
self.loop = loop
self.busy = False
self.children = set() # type: Set[UIComponent]
self.actions = actions if actions is not None else {}
for action in self.actions.values():
action.set_source(self)
self.parent = None # type: UIComponent
if loop is not None:
qsize = self.ACTION_QUEUE_SIZE
self._action_queue = asyncio.Queue(qsize) # type: asyncio.Queue
self._action_runner = loop.create_task(self._process_actions())
self._action_queue_get = None # type: asyncio.Future
self._current_action = None # type: UIActionEnum
self._nw_corner = offset - Position(1, 1)
self._se_corner = Position(
x=self.offset.x + self.size.width - 1,
y=self.offset.y + self.size.height - 1,
)
def run(*tasks: Awaitable, loop: asyncio.AbstractEventLoop=asyncio.get_event_loop()):
"""Helper to run tasks in the event loop
:param tasks: Tasks to run in the event loop.
:param loop: The event loop.
"""
futures = [asyncio.ensure_future(task, loop=loop) for task in tasks]
return loop.run_until_complete(asyncio.gather(*futures))
def __init__(self, *, loop: ALoop):
self._subscription_waiters = [] # type: List[Future]
self._assignment_waiters = [] # type: List[Future]
self._loop = loop # type: ALoop
# Fetch contexts
self._fetch_count = 0
self._last_fetch_ended = loop.time()
async def start_lightsd_connection(
loop: asyncio.AbstractEventLoop,
lightsd_url: str,
refresh_delay_s: float = DEFAULT_REFRESH_DELAY,
) -> None:
global _refresh_task, lightsd
lightsd = await lightsc.create_async_lightsd_connection(lightsd_url)
_refresh_task = loop.create_task(_poll(loop, refresh_delay_s))
def current_task(loop: asyncio.AbstractEventLoop) -> asyncio.Task:
if PY_37:
task = asyncio.current_task(loop=loop) # type: ignore
else:
task = asyncio.Task.current_task(loop=loop)
if task is None:
# this should be removed, tokio must use register_task and family API
if hasattr(loop, 'current_task'):
task = loop.current_task() # type: ignore
return task
def _cancel_all_tasks(loop: asyncio.AbstractEventLoop) -> None:
to_cancel = all_tasks(loop)
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(
asyncio.gather(*to_cancel, loop=loop, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler({
'message': 'unhandled exception during asyncio.run() shutdown',
'exception': task.exception(),
'task': task,
})
def __init__(self, connector: 'BaseConnector',
key: 'ConnectionKey',
protocol: ResponseHandler,
loop: asyncio.AbstractEventLoop) -> None:
self._key = key
self._connector = connector
self._loop = loop
self._protocol = protocol # type: Optional[ResponseHandler]
self._callbacks = [] # type: List[Callable[[], None]]
if loop.get_debug():
self._source_traceback = traceback.extract_stack(sys._getframe(1))
def get_left_time(task: asyncio.Task = None, loop: asyncio.AbstractEventLoop = None):
if loop is None:
loop = get_running_loop()
if task is None:
task = current_task()
out_time = getattr(task, _MODULE_TIMEOUT, None)
if not out_time:
return sys.maxsize
now_time = loop.time()
left_time = out_time - now_time
if left_time < 0:
return 0
return left_time
def cli(
loop: AbstractEventLoop,
aiohttp_client: Callable[[web.Application], Awaitable[TestClient]],
) -> TestClient:
app = init_app(
db_dsn=settings.DB_DSN_TEST,
secret_key="NQLSo4kyVKvWRDeo4tP_z25GPK4pN4vvrb14zv4SXI8=",
session_cookie_name="sessionid",
)
client = aiohttp_client(app)
app.router.add_get('/__test__/identity', _get_user_identity)
return loop.run_until_complete(client)
def test_get_cut_v2(
self,
caplog: pytest.LogCaptureFixture,
temp_event_loop: asyncio.AbstractEventLoop,
):
caplog.set_level(logging.DEBUG)
mock_reader = self.MockAsyncReader(TEST_V2_DATA1_EXACT[0:20])
reslt = temp_event_loop.run_until_complete(get_proxy(mock_reader))
assert isinstance(reslt, ProxyData)
assert not reslt.valid
expect_msg = "PROXY exception: Connection lost while waiting for tail part"
assert reslt.error == expect_msg
expect = ("mail.debug", 30, expect_msg)
assert expect in caplog.record_tuples
def test_asyncio_no_recycle_stopping_worker(
asyncio_event_loop: asyncio.AbstractEventLoop,
) -> None:
"""Regression test for #323."""
async def taskfunc1() -> None:
await anyio.to_thread.run_sync(time.sleep, 0)
event1.set()
await event2.wait()
async def taskfunc2() -> None:
await event1.wait()
asyncio_event_loop.call_soon(event2.set)
await anyio.to_thread.run_sync(time.sleep, 0)
# At this point, the worker would be stopped but still in the idle workers pool, so the
# following would hang prior to the fix
await anyio.to_thread.run_sync(time.sleep, 0)
event1 = asyncio.Event()
event2 = asyncio.Event()
task1 = asyncio_event_loop.create_task(taskfunc1())
task2 = asyncio_event_loop.create_task(taskfunc2())
asyncio_event_loop.run_until_complete(asyncio.gather(task1, task2))
def __init__(
self,
dtype: typing.Type[MessageClass],
transport_session: pyuavcan.transport.InputSession,
finalizer: PortFinalizer,
loop: asyncio.AbstractEventLoop,
):
self.dtype = dtype
self.transport_session = transport_session
self.deserialization_failure_count = 0
self._maybe_finalizer: typing.Optional[PortFinalizer] = finalizer
self._loop = loop
self._task = loop.create_task(self._task_function())
self._listeners: typing.List[_Listener[MessageClass]] = []
async def test_async_set_color_lights(event_loop: asyncio.AbstractEventLoop,
MockConnectedGateway):
mode = 7
result = event_loop.create_future()
result.set_result(b"")
with patch(
"screenlogicpy.requests.lights.ScreenLogicProtocol.await_send_message",
return_value=result,
) as mockRequest:
gateway = MockConnectedGateway
assert await gateway.async_set_color_lights(mode)
await gateway.async_disconnect()
assert mockRequest.call_args.args[0] == 12556
assert mockRequest.call_args.args[1] == struct.pack("<II", 0, mode)
def __init__(self, loop: AbstractEventLoop, idx: int,
schedule_details: List[bytes]) -> None:
"""Initialize the schedule."""
self._loop = loop
self._enabled = False
self._recurring = False
self._schedule_id = str(int(schedule_details[idx][0:2], 16))
self._days = [] # type: List[str]
self._schedule_data = b""
self._start_time = WAITING_TEXT
self._end_time = WAITING_TEXT
self._duration = WAITING_TEXT
self._init_future = loop.create_future()
ensure_future(self.initialize(idx, schedule_details), loop=loop)
async def test_paginate_not_paginated(
self,
context: MockContext,
fetcher: ListPageSource[int],
event_loop: asyncio.AbstractEventLoop,
advance_time: ClockAdvancer,
) -> None:
p = InteractivePager.create(cast(Any, context), fetcher)
assert not p.paginating
event_loop.create_task(p.paginate())
await advance_time(125)
assert not p.paginating
page = await fetcher.get_page(1)
assert p.embed.description == page['entry_text']
assert isinstance(p.message, MockMessage)
assert context.message is p.message
context.channel.send.assert_awaited_once_with(embed=p.embed)
p.message.add_reaction.assert_not_awaited()
p.message.clear_reactions.assert_not_awaited()
p.message.clear_reactions.assert_not_awaited()
p.message.remove_reaction.assert_not_awaited()
assert p.match is None
def handle_exception(loop: asyncio.AbstractEventLoop, context: Dict[str, Any]) -> None:
"""Exception handler for event loop."""
# context["message"] will always be there;
# but context["exception"] and context["future"] may not
try:
exc: Exception = context["exception"]
future = context["future"]
future_name = "unknown"
try:
# If future is a Task, get its name
future_name = future.get_name()
except AttributeError:
pass
_logger.error(
"Caught exception `%s` in %s task: %s",
exc.__class__.__name__,
future_name,
exc,
)
except KeyError:
_logger.error("Caught exception: %s", context["message"])
_logger.info("Shutting down...")
loop.create_task(shutdown())
def get_left_time(task: asyncio.Task = None,
loop: asyncio.AbstractEventLoop = None):
if loop is None:
loop = get_running_loop()
if task is None:
task = current_task()
out_time = getattr(task, _MODULE_TIMEOUT, None)
if not out_time:
return sys.maxsize
now_time = loop.time()
left_time = out_time - now_time
if left_time < 0:
return 0
return left_time
def request_schedules(addr: Tuple[str, int],
print_results: bool = True,
loop: asyncio.AbstractEventLoop = None):
"""Send a command to the server to Update Commands.
Args:
addr (tuple): Server IP address
print_results (bool)[True]: If true print the schedules that were returned.
loop (asyncio.AbstractEventLoop)[None]: Event loop to run the async command with.
"""
if loop is None:
loop = get_loop()
return loop.run_until_complete(
request_schedules_async(addr, print_results=print_results))
def enable_async_loop_debugging(
event_loop: AbstractEventLoop,
slow_callback_duration: float = 0.1) -> AbstractEventLoop:
"""Enables debugging on an event loop.
Args:
event_loop: The event loop to enable debugging on
slow_callback_duration: The threshold at which a callback should be
alerted as slow.
"""
logging.info("Enabling coroutine debugging. Loop id {}.".format(
id(asyncio.get_event_loop())))
# Enable debugging
event_loop.set_debug(True)
# Make the threshold for "slow" tasks very very small for
# illustration. The default is 0.1 (= 100 milliseconds).
event_loop.slow_callback_duration = slow_callback_duration
# Report all mistakes managing asynchronous resources.
warnings.simplefilter("always", ResourceWarning)
return event_loop
async def test_race_timeout(event_loop: asyncio.AbstractEventLoop,
advance_time: ClockAdvancer) -> None:
async def one() -> int:
await asyncio.sleep(100, loop=event_loop)
return 1
async def two() -> int:
await asyncio.sleep(50, loop=event_loop)
return 2
task = event_loop.create_task(
util.race([one(), two()], timeout=25, loop=event_loop))
await advance_time(30)
assert isinstance(task.exception(), asyncio.TimeoutError)
def client(
event_loop: asyncio.AbstractEventLoop,
aiohttp_client: Callable,
app_config: Dict,
postgres_with_template_db: aiopg.sa.engine.Engine,
mock_orphaned_services: mock.Mock,
monkeypatch_setenv_from_app_config: Callable,
):
# test config & env vars ----------------------
cfg = deepcopy(app_config)
assert cfg["rest"]["version"] == API_VERSION
assert cfg["rest"]["enabled"]
cfg["projects"]["enabled"] = True
cfg["director"]["enabled"] = True
cfg["exporter"]["enabled"] = True
monkeypatch_setenv_from_app_config(cfg)
# app setup ----------------------------------
app = create_safe_application(cfg)
# activates only security+restAPI sub-modules
assert setup_settings(app)
assert get_exporter_settings(app) is not None, "Should capture defaults"
setup_db(app)
setup_session(app)
setup_security(app)
setup_rest(app)
setup_login(app)
setup_users(app)
setup_socketio(app)
setup_projects(app)
setup_director(app)
setup_director_v2(app)
setup_exporter(app) # <---- under test
setup_storage(app)
setup_products(app)
setup_catalog(app)
setup_scicrunch(app)
assert setup_resource_manager(app)
setup_garbage_collector(app)
yield event_loop.run_until_complete(
aiohttp_client(
app,
server_kwargs={"port": cfg["main"]["port"], "host": cfg["main"]["host"]},
)
)
def __init__(self, request: RequestType, deadline: Optional[float],
metadata: Metadata,
credentials: Optional[grpc.CallCredentials],
wait_for_ready: Optional[bool], channel: cygrpc.AioChannel,
method: bytes, request_serializer: SerializingFunction,
response_deserializer: DeserializingFunction,
loop: asyncio.AbstractEventLoop) -> None:
super().__init__(
channel.call(method, deadline, credentials, wait_for_ready),
metadata, request_serializer, response_deserializer, loop)
self._request = request
self._send_unary_request_task = loop.create_task(
self._send_unary_request())
self._init_stream_response_mixin(self._send_unary_request_task)
def get_client_session(
loop: asyncio.AbstractEventLoop) -> aiohttp.ClientSession:
"""Get a shared ClientSession object that can be reused. If none exists yet it will
create one using the passed-in loop.
:param loop: an active (i.e. not closed) asyncio event loop
:return: a ClientSession instance that can be used to do HTTP requests
"""
try:
return loop._bravado_asyncio_client_session # type: ignore
except AttributeError:
client_session = aiohttp.ClientSession(loop=loop)
loop._bravado_asyncio_client_session = client_session # type: ignore
return client_session
async def start(
self, loop: asyncio.AbstractEventLoop
) -> Tuple[HttpExchangeStream, asyncio.Task]:
self.worker = faust.Worker(self.app, loop=loop, loglevel="info")
async def start_worker(worker: faust.Worker) -> None:
await worker.start()
source = self.http_exchange_stream()
worker_coro = start_worker(self.worker)
self.worker_task = loop.create_task(worker_coro)
return source, self.worker_task
def create_client(client_type, request, event_loop: asyncio.AbstractEventLoop,
session, region, config, **kw):
client = session.create_client(client_type,
region_name=region,
config=config,
**kw)
def fin():
event_loop.run_until_complete(client.__aexit__(None, None, None))
request.addfinalizer(fin)
client = event_loop.run_until_complete(client.__aenter__())
return client
def test_create_user(client: TestClient, event_loop: asyncio.AbstractEventLoop): # nosec
response = client.post("/users", json={"username": "******"})
assert response.status_code == 200, response.text
data = response.json()
assert data["username"] == "admin"
assert "id" in data
user_id = data["id"]
async def get_user_by_db():
user = await Users.get(id=user_id)
return user
user_obj = event_loop.run_until_complete(get_user_by_db())
assert user_obj.id == user_id
def shutdown(loop: asyncio.AbstractEventLoop) -> None:
"""Cancel all pending tasks on `loop`, wait for them, and close the loop."""
try:
if sys.version_info[:2] >= (3, 7):
all_tasks = asyncio.all_tasks
else:
all_tasks = asyncio.Task.all_tasks
# This part is borrowed from asyncio/runners.py in Python 3.7b2.
to_cancel = [task for task in all_tasks(loop) if not task.done()]
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(
asyncio.gather(*to_cancel, loop=loop, return_exceptions=True))
finally:
# `concurrent.futures.Future` objects cannot be cancelled once they
# are already running. There might be some when the `shutdown()` happened.
# Silence their logger's spew about the event loop being closed.
cf_logger = logging.getLogger("concurrent.futures")
cf_logger.setLevel(logging.CRITICAL)
loop.close()
def __start_read_loop(self,
*,
loop: asyncio.AbstractEventLoop = None) -> None:
"""Start the web socket reader.
If the reader is already running, this is a no-op.
"""
if self.__read_loop and not self.__read_loop.done():
return
if loop is None:
loop = asyncio.get_event_loop()
self.__read_loop = loop.create_task(self.__web_socket_reader())
async def test_stream_buffer_push_and_pop(
event_loop: asyncio.AbstractEventLoop) -> None:
stream_buffer = StreamBuffer(EventWrapper)
async def _push_over_limit() -> None:
await stream_buffer.push(b"a" * (BUFFER_HIGH_WATER + 1))
return True
task = event_loop.create_task(_push_over_limit())
assert not task.done() # Blocked as over high water
await stream_buffer.pop(BUFFER_HIGH_WATER / 4)
assert not task.done() # Blocked as over low water
await stream_buffer.pop(BUFFER_HIGH_WATER / 4)
assert (await task) is True
def handler(loop: asyncio.AbstractEventLoop,
context: Dict[str, Any]) -> None: # pragma: no cover
message = context["message"]
exception = context.get("exception", Exception)
if sys.version_info < (3, 7, 4) and message in (
"SSL error in data received",
"Fatal error on transport",
):
# Ignore aiohttp #3535 / cpython #13548 issue with SSL data after close
# Adapted from https://github.com/aio-libs/aiohttp/issues/3535#issuecomment-483268542
import ssl
from asyncio.sslproto import SSLProtocol
protocol = context.get("protocol")
if (isinstance(exception, ssl.SSLError)
and exception.reason == "KRB5_S_INIT"
and isinstance(protocol, SSLProtocol)):
return
# Our main interest here is minimising the other various errors and tracebacks that
# drown out the politely formatted errors from cli.py when things go wrong
if "exception was never retrieved" in message:
from .request import MissingSession
# While closing down, we remove the global session, causing other requests to fail. This
# just causes noise, so ignore if that's the exception.
if not isinstance(exception, MissingSession):
print(
f"ERROR (while closing down): {type(exception).__name__}: {exception}",
file=sys.stderr,
)
else:
print(
f"ERROR (from event loop): {type(exception).__name__}: {message}",
file=sys.stderr)
if loop.get_debug():
loop.default_exception_handler(context)
def aiohttp_client(
loop: asyncio.AbstractEventLoop, aiohttp_client_cls: Type[TestClient]
) -> Generator[AiohttpClient, None, None]:
"""Factory to create a TestClient instance.
aiohttp_client(app, **kwargs)
aiohttp_client(server, **kwargs)
aiohttp_client(raw_server, **kwargs)
"""
clients = []
async def go(
__param: Union[Application, BaseTestServer],
*,
server_kwargs: Optional[Dict[str, Any]] = None,
**kwargs: Any
) -> TestClient:
if isinstance(__param, Application):
server_kwargs = server_kwargs or {}
server = TestServer(__param, **server_kwargs)
client = aiohttp_client_cls(server, **kwargs)
elif isinstance(__param, BaseTestServer):
client = aiohttp_client_cls(__param, **kwargs)
else:
raise ValueError("Unknown argument type: %r" % type(__param))
await client.start_server()
clients.append(client)
return client
yield go
async def finalize() -> None:
while clients:
await clients.pop().close()
loop.run_until_complete(finalize())
def _get_okta_group_users(
self, group: Group, event_loop: asyncio.AbstractEventLoop
) -> Iterable[User]:
logger.debug(f"Extracting users from Okta group named {group.profile.name}")
# Note that this is not taking full advantage of Python AsyncIO; we are blocking on calls.
query_parameters = {"limit": self.config.page_size}
users = resp = err = None
try:
users, resp, err = event_loop.run_until_complete(
self.okta_client.list_group_users(group.id, query_parameters)
)
except OktaAPIException as api_err:
self.report.report_failure(
"okta_group_users",
f"Failed to fetch Users of Group {group.profile.name} from Okta API: {api_err}",
)
while True:
if err:
self.report.report_failure(
"okta_group_users",
f"Failed to fetch Users of Group {group.profile.name} from Okta API: {err}",
)
if users:
for user in users:
yield user
if resp and resp.has_next():
sleep(self.config.delay_seconds)
try:
users, err = event_loop.run_until_complete(resp.next())
except OktaAPIException as api_err:
self.report.report_failure(
"okta_group_users",
f"Failed to fetch Users of Group {group.profile.name} from Okta API: {api_err}",
)
else:
break
def get_time_accelerator(loop: asyncio.AbstractEventLoop,
now: typing.Optional[float] = None) -> typing.Callable[[float], typing.Awaitable[None]]:
"""
Returns an async advance() function
This provides a way to advance() the BaseEventLoop.time for the scheduled TimerHandles
made by call_later, call_at, and call_soon.
"""
_time = now or loop.time()
loop.time = functools.wraps(loop.time)(lambda: _time)
async def accelerate_time(seconds: float) -> None:
nonlocal _time
if seconds < 0:
raise ValueError(f'Cannot go back in time ({seconds} seconds)')
_time += seconds
await past_events()
await asyncio.sleep(0)
async def past_events() -> None:
while loop._scheduled:
timer: asyncio.TimerHandle = loop._scheduled[0]
if timer not in loop._ready and timer._when <= _time:
loop._scheduled.remove(timer)
loop._ready.append(timer)
if timer._when > _time:
break
await asyncio.sleep(0)
async def accelerator(seconds: float):
steps = seconds * 10.0
for _ in range(max(int(steps), 1)):
await accelerate_time(0.1)
return accelerator
async def main(loop: asyncio.AbstractEventLoop, session: aiohttp.ClientSession,
language_to_build: str, args: dict) -> None:
"""
Main method that starts the entire build process
:param args:
:param loop:
:param session:
:param language_to_build:
:return:
"""
def get_next_batch_of_sets(queue: Iterator[List[str]]) -> List[List[str]]:
"""
To ensure better performance, we limit the number of sets built at a time
to limit our memory impact. This will return the next group of sets to
build.
"""
max_pops = int(args['max_sets_build'][0])
# User disabled this memory protection feature
if max_pops == 0:
return list(queue)
return list(itertools.islice(queue, max_pops))
# Main Applied
mtg_storage.ensure_set_dir_exists()
sets_queue = iter(SETS_TO_BUILD)
async with session:
# Start asyncio tasks for building each set
json_builder = mtg_builder.MTGJSON(SETS_TO_BUILD, session, loop)
# We will only be building a few sets at a time, to allow for partial outputs
sets_to_build_now = get_next_batch_of_sets(sets_queue)
while sets_to_build_now:
# Create our builders for the few sets
futures = [
loop.create_task(
json_builder.build_set(set_name, language_to_build))
for set_name in sets_to_build_now
]
# Then wait until all of them are completed
await asyncio.wait(futures)
# Then queue up our next sets to build
sets_to_build_now = get_next_batch_of_sets(sets_queue)
# And we're done! :)
return
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool = False) -> None:
"""Teardown and cleanup an event_loop created by setup_test_loop."""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def start_async_callback(
event_loop: asyncio.AbstractEventLoop,
coroutine_func,
shutdown_event: Event,
):
asyncio.set_event_loop(event_loop)
try:
event_loop.run_until_complete(coroutine_func())
event_loop.run_until_complete(event_loop.shutdown_asyncgens())
event_loop.close()
finally:
shutdown_event.set()
def client(loop: AbstractEventLoop, aiohttp_client: Callable,
aiohttp_unused_port: Callable) -> TestClient:
ports = [aiohttp_unused_port() for _ in range(2)]
async def redirect(request: web.Request) -> web.Response:
return web.HTTPFound(location="/return/200")
async def return_response(request: web.Request) -> web.Response:
code = int(request.match_info["code"])
return web.Response(status=code)
async def raise_response(request: web.Request):
status_code = int(request.match_info["code"])
status_to_http_exception = _collect_http_exceptions()
http_exception_cls = status_to_http_exception[status_code]
raise http_exception_cls(
reason=f"raised from raised_error with code {status_code}")
async def skip(request: web.Request):
return web.HTTPServiceUnavailable(reason="should not happen")
app = web.Application()
app.add_routes([
web.get("/redirect", redirect),
web.get("/return/{code}", return_response),
web.get("/raise/{code}", raise_response),
web.get("/skip", skip, name="skip"),
])
print("Resources:")
for resource in app.router.resources():
print(resource)
# UNDER TEST ---
# SEE RoutesView to understand how resources can be iterated to get routes
resource = app.router["skip"]
routes_in_a_resource = list(resource)
setup_tracing(
app,
service_name=f"{__name__}.client",
host="127.0.0.1",
port=ports[0],
jaeger_base_url=DEFAULT_JAEGER_BASE_URL,
skip_routes=routes_in_a_resource,
)
return loop.run_until_complete(
aiohttp_client(app, server_kwargs={"port": ports[0]}))
def __init__(self, interceptors: Sequence[UnaryUnaryClientInterceptor],
request: RequestType, timeout: Optional[float],
metadata: MetadataType,
credentials: Optional[grpc.CallCredentials],
wait_for_ready: Optional[bool], channel: cygrpc.AioChannel,
method: bytes, request_serializer: SerializingFunction,
response_deserializer: DeserializingFunction,
loop: asyncio.AbstractEventLoop) -> None:
self._loop = loop
self._channel = channel
interceptors_task = loop.create_task(
self._invoke(interceptors, method, timeout, metadata, credentials,
wait_for_ready, request, request_serializer,
response_deserializer))
super().__init__(interceptors_task)
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool=False) -> None:
"""Teardown and cleanup an event_loop created
by setup_test_loop.
"""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def redis_pool(event_loop: asyncio.AbstractEventLoop, redis_port: int):
redis_pool = event_loop.run_until_complete(
aioredis.create_pool(('localhost', redis_port)))
yield redis_pool
redis_pool.close()
event_loop.run_until_complete(redis_pool.wait_closed())
def run(self, event_loop: AbstractEventLoop = None):
# Configure the logging system
if isinstance(self.logging_config, dict):
logging.config.dictConfig(self.logging_config)
elif self.logging_config:
logging.basicConfig(level=logging.INFO)
# Assign a new default executor with the given max worker thread limit
event_loop = event_loop or asyncio.get_event_loop()
event_loop.set_default_executor(ThreadPoolExecutor(self.max_threads))
# Create the application context
context = self.create_context()
try:
# Start all the components and run the loop until they've finished
self.logger.info("Starting components")
coroutines = (component.start(context) for component in self.components)
coroutines = [coro for coro in coroutines if coro is not None]
event_loop.run_until_complete(asyncio.gather(*coroutines))
self.logger.info("All components started")
# Run the application's custom startup code
coro = self.start(context)
if coro is not None:
event_loop.run_until_complete(coro)
# Run all the application context's start callbacks
event_loop.run_until_complete(context.run_callbacks(ContextEventType.started))
self.logger.info("Application started")
except Exception as exc:
self.logger.exception("Error during application startup")
context.exception = exc
else:
# Finally, run the event loop until the process is terminated or Ctrl+C is pressed
try:
event_loop.run_forever()
except (KeyboardInterrupt, SystemExit):
pass
event_loop.run_until_complete(context.run_callbacks(ContextEventType.finished))
event_loop.close()
self.logger.info("Application stopped")
