async def run(*, loop: asyncio.BaseEventLoop = None): loop = loop or asyncio.get_event_loop() ev = asyncio.Event() loop.add_signal_handler(signal.SIGINT, ev.set) update_task = loop.create_task(update_lifetime(loop, ev, lifetime=3)) update_task.add_done_callback(lambda fut: fut.cancelled() or fut.result()) atask = loop.create_task(do_task()) done, pending = await asyncio.wait( [ev.wait(), atask], return_when=asyncio.FIRST_COMPLETED ) result = None if atask in done: try: result = await atask except Exception as e: result = f"ng ({e!r})" update_task.cancel() interrupted = ev.is_set() if interrupted: logger.info("task is interrupted (catch SIGINT)") else: logger.info("task completed, result=%r", result)
async def run(*, loop: asyncio.BaseEventLoop = None): loop = loop or asyncio.get_event_loop() ev = asyncio.Event() loop.add_signal_handler(signal.SIGINT, ev.set) loop.add_signal_handler(signal.SIGTERM, ev.set) update_task = loop.create_task(update_lifetime(ev, lifetime=3)) atask = loop.create_task(do_task(loop)) done, pending = await asyncio.wait( [ev.wait(), atask], return_when=asyncio.FIRST_COMPLETED ) result = None if atask in done: try: result = await atask except Exception as e: result = f"ng ({e!r})" update_task.cancel() interrupted = ev.is_set() if interrupted: logger.info("** task is interrupted (catch SIGINT) **") else: logger.info("** task completed, result=%r **", result)
def sig_handler(sig: enum.Enum, loop: asyncio.BaseEventLoop): nonlocal stopping if stopping: return logger.info(f"got {sig}, terminating") loop.create_task(server.stop(0)) stopping = True
async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): """ *required Subscribe to user stream via web socket, and keep the connection open for incoming messages :param ev_loop: ev_loop to execute this function in :param output: an async queue where the incoming messages are stored """ while True: try: async with websockets.connect(Constants.BAEE_WS_URL) as ws: ws: websockets.WebSocketClientProtocol = ws ev_loop.create_task(self.custom_ping(ws)) # Send a auth request first auth_request: Dict[str, Any] = { "event": Constants.WS_AUTH_REQUEST_EVENT, "data": self._liquid_auth.get_ws_auth_data() } await ws.send(ujson.dumps(auth_request)) quoted_currencies = [ trading_pair.split('-')[1] for trading_pair in self._trading_pairs ] for trading_pair, quoted_currency in zip( self._trading_pairs, quoted_currencies): subscribe_request: Dict[str, Any] = { "event": Constants.WS_PUSHER_SUBSCRIBE_EVENT, "data": { "channel": Constants.WS_USER_ACCOUNTS_SUBSCRIPTION.format( quoted_currency=quoted_currency.lower()) } } await ws.send(ujson.dumps(subscribe_request)) async for raw_msg in self._inner_messages(ws): diff_msg = ujson.loads(raw_msg) event_type = diff_msg.get('event', None) if event_type == 'updated': output.put_nowait(diff_msg) self._last_recv_time = time.time() elif event_type == "pusher:pong": self._last_recv_time = time.time() elif not event_type: raise ValueError( f"Liquid Websocket message does not contain an event type - {diff_msg}" ) except asyncio.CancelledError: raise except Exception: self.logger().error( "Unexpected error with Liquid WebSocket connection. " "Retrying after 30 seconds...", exc_info=True) await asyncio.sleep(30.0)
def __init__(self, loop: asyncio.BaseEventLoop): self.static = pathlib.Path(rel_path('../frontend/build', check=False)) self.loop = loop self.app = Starlette(routes=self.routes, on_shutdown=[self.exit]) self.config = uvicorn.config.Config(self.app, log_config=None, host='0.0.0.0', port=7999) self.server = uvicorn.Server(config=self.config) self.serve_task = loop.create_task(self.server.serve()) self.update_task = loop.create_task(self.update_loop()) self.ws_clients = []
def __init__(self, loop: asyncio.BaseEventLoop, pool: PubSubPool, node: int): self.loop = loop self._pool = pool self._node = node self._topics = [] self._websocket = None self._timeout = asyncio.Event() self._last_result = None loop.create_task(self.handle_ping()) self._listener = None
async def test_exectution_limit_once(coresys: CoreSys, loop: asyncio.BaseEventLoop): """Test the ignore conditions decorator.""" class TestClass: """Test class.""" def __init__(self, coresys: CoreSys): """Initialize the test class.""" self.coresys = coresys self.run = asyncio.Lock() @Job(limit=JobExecutionLimit.ONCE, on_condition=JobException) async def execute(self, sleep: float): """Execute the class method.""" assert not self.run.locked() async with self.run: await asyncio.sleep(sleep) test = TestClass(coresys) run_task = loop.create_task(test.execute(0.3)) await asyncio.sleep(0.1) with pytest.raises(JobException): await test.execute(0.1) await run_task
async def server_loop(self, loop: asyncio.BaseEventLoop): self.server = aiohttp.web.Server(self.handle_request) runner = aiohttp.web.ServerRunner(self.server) await runner.setup() site = aiohttp.web.TCPSite(runner, self.config.server.ip, self.config.server.port) await site.start() print("==== PyPubSub v/%s starting... ====" % PUBSUB_VERSION) print("==== Serving up PubSub goodness at %s:%s ====" % (self.config.server.ip, self.config.server.port)) if self.config.sqs: for key, config in self.config.sqs.items(): loop.create_task(plugins.sqs.get_payloads(self, config)) self.read_backlog_storage() loop.create_task(self.write_backlog_storage()) await self.poll()
async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue): trading_pairs: List[str] = await self.get_trading_pairs() tasks = [ ev_loop.create_task(self._listen_trades_for_pair(pair, output)) for pair in trading_pairs ] await asyncio.gather(*tasks)
def __init__(self, loop: asyncio.BaseEventLoop = asyncio.get_event_loop()): self._local_tz = pytz.timezone("Europe/Helsinki") self.__name = type(self).__name__ self.__client_session = aiohttp.ClientSession(loop=loop) self.update_in_progress = False # Data variables self.__hospitalised_data: dict = dict() self.__corona_data: dict = dict() self.__vaccination_data: dict = dict() self.__daily_cases: dict = dict(confirmed=0, deaths=0, totalHospitalised=0, inWard=0, inIcu=0, shots=0) self.last_update_dt: Union[None, datetime.datetime] = None # Cache update cooldown in minutes self.cooldown = 30 # Start the loop for caching loop.create_task(self.__cache_loop())
def start_child_thread_loop( loop: asyncio.BaseEventLoop, break_sign: BreakSign, main_fetch_callback: Callable[[Union[Future, SyncFuture]], None], child_fetch_callback: Callable[[Union[Future, SyncFuture]], None], request_package_list: RequestPackageList, headers: Union[dict, None], total_timeout: int, concurrency_number: int) -> None: downloader_logger.debug('Async downloader start.') asyncio.set_event_loop(loop) main_fetch_task = loop.create_task( main_fetch(request_package_list, break_sign, child_fetch_callback, headers, total_timeout, concurrency_number)) main_fetch_task.add_done_callback(main_fetch_callback) loop.run_until_complete(main_fetch_task) downloader_logger.debug('Async download thread over.')
def serve(self, host, port, loop: BaseEventLoop): """Creates an asyncio coroutine, that serves requests on the provided host and port. ```python loop = asyncio.get_event_loop() server.serve('localhost', 8080, loop=loop) loop.run_forever() ``` """ async def _serve(): self._server = await start_server(self.respond, host, port, loop=loop, start_serving=False) return await self._server.serve_forever() self._server_task = loop.create_task(_serve())
async def gpus_mon(self, loop: asyncio.BaseEventLoop = None, ignore=tuple()): subproc_exec = self.async_exec nv2cuda, pid2owner = await asyncio.gather( self.nv2cuda_coro(subproc_exec), self.pid2owner_coro(subproc_exec)) p = await self.async_exec('nvidia-smi', '-l', '1', stdout=asyncio.subprocess.PIPE, stderr=FNULL) loop = loop or asyncio.get_event_loop() gpus = dict() gpu_nvprocs = dict() do_GPUComb = GPUComb not in ignore do_GPUProcess = GPUProcess not in ignore nvdev = None tasks = list() seen_pids = list() while not self.terminated: line = await p.stdout.readline() line = line.decode() if do_GPUComb: nvdev = nv_line2nvdev(line, nvdev) nvgpu = nv_line2GPUNv(line, nvdev) # a gpu was found in stdout if nvgpu: prev_gpu = gpus.get(nvgpu.nvdev, None) # has anything changed? if prev_gpu != nvgpu[1:]: # translate to cuda dev and update gpus gpu = GPUComb(nv2cuda[nvgpu.nvdev], *nvgpu[1:]) gpus[nvgpu.nvdev] = nvgpu[1:] # put into change stream await self.change_stream.put(gpu) continue if do_GPUProcess: nvproc = nv_line2GPUNvProcess(line) if nvproc: seen_pids.append(nvproc.pid) tasks.append( loop.create_task( self._nvproc2proc(subproc_exec, nvproc, pid2owner, nv2cuda, gpu_nvprocs))) continue if tasks: await asyncio.wait(tasks) tasks.clear() dead_pids = set(gpu_nvprocs.keys()).difference(seen_pids) for dead_proc in (gpu_nvprocs[pid] for pid in dead_pids): await self.change_stream.put( GPUProcess(dead_proc.pid, pid2owner[dead_proc.pid], nv2cuda[dead_proc.nvdev], 0)) gpu_nvprocs.pop(dead_proc.pid) seen_pids.clear()
async def gpus_mon(self, loop: asyncio.BaseEventLoop = None, ignore=tuple()): subproc_exec = self.async_exec nv2cuda, pid2owner = await asyncio.gather( self.nv2cuda_coro(subproc_exec), self.pid2owner_coro(subproc_exec)) p = await self.async_exec('nvidia-smi', '-l', '1', stdout=asyncio.subprocess.PIPE, stderr=FNULL) loop = loop or asyncio.get_event_loop() gpus = dict() gpu_nvprocs = dict() do_GPUComb = GPUComb not in ignore do_GPUProcess = GPUProcess not in ignore nvdev = None tasks = list() seen_pids = list() while not self.terminated: line = await p.stdout.readline() line = line.decode() if do_GPUComb: nvdev = nv_line2nvdev(line, nvdev) nvgpu = nv_line2GPUNv(line, nvdev) # a gpu was found in stdout if nvgpu: prev_gpu = gpus.get(nvgpu.nvdev, None) # has anything changed? if prev_gpu != nvgpu[1:]: # translate to cuda dev and update gpus gpu = GPUComb(nv2cuda[nvgpu.nvdev], *nvgpu[1:]) gpus[nvgpu.nvdev] = nvgpu[1:] # put into change stream await self.change_stream.put(gpu) continue if do_GPUProcess: nvproc = nv_line2GPUNvProcess(line) if nvproc: seen_pids.append(nvproc.pid) tasks.append( loop.create_task(self._nvproc2proc(subproc_exec, nvproc, pid2owner, nv2cuda, gpu_nvprocs))) continue if tasks: await asyncio.wait(tasks) tasks.clear() dead_pids = set(gpu_nvprocs.keys()).difference(seen_pids) for dead_proc in (gpu_nvprocs[pid] for pid in dead_pids): await self.change_stream.put(GPUProcess(dead_proc.pid, pid2owner[ dead_proc.pid], nv2cuda[ dead_proc.nvdev], 0)) gpu_nvprocs.pop(dead_proc.pid) seen_pids.clear()