async def run(*, loop: asyncio.BaseEventLoop = None):
loop = loop or asyncio.get_event_loop()
ev = asyncio.Event()
loop.add_signal_handler(signal.SIGINT, ev.set)
update_task = loop.create_task(update_lifetime(loop, ev, lifetime=3))
update_task.add_done_callback(lambda fut: fut.cancelled() or fut.result())
atask = loop.create_task(do_task())
done, pending = await asyncio.wait(
[ev.wait(), atask], return_when=asyncio.FIRST_COMPLETED
)
result = None
if atask in done:
try:
result = await atask
except Exception as e:
result = f"ng ({e!r})"
update_task.cancel()
interrupted = ev.is_set()
if interrupted:
logger.info("task is interrupted (catch SIGINT)")
else:
logger.info("task completed, result=%r", result)
async def run(*, loop: asyncio.BaseEventLoop = None):
loop = loop or asyncio.get_event_loop()
ev = asyncio.Event()
loop.add_signal_handler(signal.SIGINT, ev.set)
loop.add_signal_handler(signal.SIGTERM, ev.set)
update_task = loop.create_task(update_lifetime(ev, lifetime=3))
atask = loop.create_task(do_task(loop))
done, pending = await asyncio.wait(
[ev.wait(), atask], return_when=asyncio.FIRST_COMPLETED
)
result = None
if atask in done:
try:
result = await atask
except Exception as e:
result = f"ng ({e!r})"
update_task.cancel()
interrupted = ev.is_set()
if interrupted:
logger.info("** task is interrupted (catch SIGINT) **")
else:
logger.info("** task completed, result=%r **", result)
def sig_handler(sig: enum.Enum, loop: asyncio.BaseEventLoop):
nonlocal stopping
if stopping:
return
logger.info(f"got {sig}, terminating")
loop.create_task(server.stop(0))
stopping = True
async def listen_for_user_stream(self, ev_loop: asyncio.BaseEventLoop,
output: asyncio.Queue):
"""
*required
Subscribe to user stream via web socket, and keep the connection open for incoming messages
:param ev_loop: ev_loop to execute this function in
:param output: an async queue where the incoming messages are stored
"""
while True:
try:
async with websockets.connect(Constants.BAEE_WS_URL) as ws:
ws: websockets.WebSocketClientProtocol = ws
ev_loop.create_task(self.custom_ping(ws))
# Send a auth request first
auth_request: Dict[str, Any] = {
"event": Constants.WS_AUTH_REQUEST_EVENT,
"data": self._liquid_auth.get_ws_auth_data()
}
await ws.send(ujson.dumps(auth_request))
quoted_currencies = [
trading_pair.split('-')[1]
for trading_pair in self._trading_pairs
]
for trading_pair, quoted_currency in zip(
self._trading_pairs, quoted_currencies):
subscribe_request: Dict[str, Any] = {
"event": Constants.WS_PUSHER_SUBSCRIBE_EVENT,
"data": {
"channel":
Constants.WS_USER_ACCOUNTS_SUBSCRIPTION.format(
quoted_currency=quoted_currency.lower())
}
}
await ws.send(ujson.dumps(subscribe_request))
async for raw_msg in self._inner_messages(ws):
diff_msg = ujson.loads(raw_msg)
event_type = diff_msg.get('event', None)
if event_type == 'updated':
output.put_nowait(diff_msg)
self._last_recv_time = time.time()
elif event_type == "pusher:pong":
self._last_recv_time = time.time()
elif not event_type:
raise ValueError(
f"Liquid Websocket message does not contain an event type - {diff_msg}"
)
except asyncio.CancelledError:
raise
except Exception:
self.logger().error(
"Unexpected error with Liquid WebSocket connection. "
"Retrying after 30 seconds...",
exc_info=True)
await asyncio.sleep(30.0)
def __init__(self, loop: asyncio.BaseEventLoop):
self.static = pathlib.Path(rel_path('../frontend/build', check=False))
self.loop = loop
self.app = Starlette(routes=self.routes, on_shutdown=[self.exit])
self.config = uvicorn.config.Config(self.app, log_config=None, host='0.0.0.0', port=7999)
self.server = uvicorn.Server(config=self.config)
self.serve_task = loop.create_task(self.server.serve())
self.update_task = loop.create_task(self.update_loop())
self.ws_clients = []
def __init__(self, loop: asyncio.BaseEventLoop, pool: PubSubPool, node: int):
self.loop = loop
self._pool = pool
self._node = node
self._topics = []
self._websocket = None
self._timeout = asyncio.Event()
self._last_result = None
loop.create_task(self.handle_ping())
self._listener = None
async def test_exectution_limit_once(coresys: CoreSys,
loop: asyncio.BaseEventLoop):
"""Test the ignore conditions decorator."""
class TestClass:
"""Test class."""
def __init__(self, coresys: CoreSys):
"""Initialize the test class."""
self.coresys = coresys
self.run = asyncio.Lock()
@Job(limit=JobExecutionLimit.ONCE, on_condition=JobException)
async def execute(self, sleep: float):
"""Execute the class method."""
assert not self.run.locked()
async with self.run:
await asyncio.sleep(sleep)
test = TestClass(coresys)
run_task = loop.create_task(test.execute(0.3))
await asyncio.sleep(0.1)
with pytest.raises(JobException):
await test.execute(0.1)
await run_task
async def server_loop(self, loop: asyncio.BaseEventLoop):
self.server = aiohttp.web.Server(self.handle_request)
runner = aiohttp.web.ServerRunner(self.server)
await runner.setup()
site = aiohttp.web.TCPSite(runner, self.config.server.ip,
self.config.server.port)
await site.start()
print("==== PyPubSub v/%s starting... ====" % PUBSUB_VERSION)
print("==== Serving up PubSub goodness at %s:%s ====" %
(self.config.server.ip, self.config.server.port))
if self.config.sqs:
for key, config in self.config.sqs.items():
loop.create_task(plugins.sqs.get_payloads(self, config))
self.read_backlog_storage()
loop.create_task(self.write_backlog_storage())
await self.poll()
async def listen_for_trades(self, ev_loop: asyncio.BaseEventLoop, output: asyncio.Queue):
trading_pairs: List[str] = await self.get_trading_pairs()
tasks = [
ev_loop.create_task(self._listen_trades_for_pair(pair, output))
for pair in trading_pairs
]
await asyncio.gather(*tasks)
def __init__(self, loop: asyncio.BaseEventLoop = asyncio.get_event_loop()):
self._local_tz = pytz.timezone("Europe/Helsinki")
self.__name = type(self).__name__
self.__client_session = aiohttp.ClientSession(loop=loop)
self.update_in_progress = False
# Data variables
self.__hospitalised_data: dict = dict()
self.__corona_data: dict = dict()
self.__vaccination_data: dict = dict()
self.__daily_cases: dict = dict(confirmed=0,
deaths=0,
totalHospitalised=0,
inWard=0,
inIcu=0,
shots=0)
self.last_update_dt: Union[None, datetime.datetime] = None
# Cache update cooldown in minutes
self.cooldown = 30
# Start the loop for caching
loop.create_task(self.__cache_loop())
def start_child_thread_loop(
loop: asyncio.BaseEventLoop, break_sign: BreakSign,
main_fetch_callback: Callable[[Union[Future, SyncFuture]], None],
child_fetch_callback: Callable[[Union[Future, SyncFuture]], None],
request_package_list: RequestPackageList, headers: Union[dict, None],
total_timeout: int, concurrency_number: int) -> None:
downloader_logger.debug('Async downloader start.')
asyncio.set_event_loop(loop)
main_fetch_task = loop.create_task(
main_fetch(request_package_list, break_sign, child_fetch_callback,
headers, total_timeout, concurrency_number))
main_fetch_task.add_done_callback(main_fetch_callback)
loop.run_until_complete(main_fetch_task)
downloader_logger.debug('Async download thread over.')
def serve(self, host, port, loop: BaseEventLoop):
"""Creates an asyncio coroutine, that serves requests on the provided host and port.
```python
loop = asyncio.get_event_loop()
server.serve('localhost', 8080, loop=loop)
loop.run_forever()
```
"""
async def _serve():
self._server = await start_server(self.respond,
host,
port,
loop=loop,
start_serving=False)
return await self._server.serve_forever()
self._server_task = loop.create_task(_serve())
async def gpus_mon(self,
loop: asyncio.BaseEventLoop = None,
ignore=tuple()):
subproc_exec = self.async_exec
nv2cuda, pid2owner = await asyncio.gather(
self.nv2cuda_coro(subproc_exec), self.pid2owner_coro(subproc_exec))
p = await self.async_exec('nvidia-smi',
'-l',
'1',
stdout=asyncio.subprocess.PIPE,
stderr=FNULL)
loop = loop or asyncio.get_event_loop()
gpus = dict()
gpu_nvprocs = dict()
do_GPUComb = GPUComb not in ignore
do_GPUProcess = GPUProcess not in ignore
nvdev = None
tasks = list()
seen_pids = list()
while not self.terminated:
line = await p.stdout.readline()
line = line.decode()
if do_GPUComb:
nvdev = nv_line2nvdev(line, nvdev)
nvgpu = nv_line2GPUNv(line, nvdev)
# a gpu was found in stdout
if nvgpu:
prev_gpu = gpus.get(nvgpu.nvdev, None)
# has anything changed?
if prev_gpu != nvgpu[1:]:
# translate to cuda dev and update gpus
gpu = GPUComb(nv2cuda[nvgpu.nvdev], *nvgpu[1:])
gpus[nvgpu.nvdev] = nvgpu[1:]
# put into change stream
await self.change_stream.put(gpu)
continue
if do_GPUProcess:
nvproc = nv_line2GPUNvProcess(line)
if nvproc:
seen_pids.append(nvproc.pid)
tasks.append(
loop.create_task(
self._nvproc2proc(subproc_exec, nvproc, pid2owner,
nv2cuda, gpu_nvprocs)))
continue
if tasks:
await asyncio.wait(tasks)
tasks.clear()
dead_pids = set(gpu_nvprocs.keys()).difference(seen_pids)
for dead_proc in (gpu_nvprocs[pid] for pid in dead_pids):
await self.change_stream.put(
GPUProcess(dead_proc.pid, pid2owner[dead_proc.pid],
nv2cuda[dead_proc.nvdev], 0))
gpu_nvprocs.pop(dead_proc.pid)
seen_pids.clear()
async def gpus_mon(self, loop: asyncio.BaseEventLoop = None,
ignore=tuple()):
subproc_exec = self.async_exec
nv2cuda, pid2owner = await asyncio.gather(
self.nv2cuda_coro(subproc_exec),
self.pid2owner_coro(subproc_exec))
p = await self.async_exec('nvidia-smi', '-l', '1',
stdout=asyncio.subprocess.PIPE,
stderr=FNULL)
loop = loop or asyncio.get_event_loop()
gpus = dict()
gpu_nvprocs = dict()
do_GPUComb = GPUComb not in ignore
do_GPUProcess = GPUProcess not in ignore
nvdev = None
tasks = list()
seen_pids = list()
while not self.terminated:
line = await p.stdout.readline()
line = line.decode()
if do_GPUComb:
nvdev = nv_line2nvdev(line, nvdev)
nvgpu = nv_line2GPUNv(line, nvdev)
# a gpu was found in stdout
if nvgpu:
prev_gpu = gpus.get(nvgpu.nvdev, None)
# has anything changed?
if prev_gpu != nvgpu[1:]:
# translate to cuda dev and update gpus
gpu = GPUComb(nv2cuda[nvgpu.nvdev], *nvgpu[1:])
gpus[nvgpu.nvdev] = nvgpu[1:]
# put into change stream
await self.change_stream.put(gpu)
continue
if do_GPUProcess:
nvproc = nv_line2GPUNvProcess(line)
if nvproc:
seen_pids.append(nvproc.pid)
tasks.append(
loop.create_task(self._nvproc2proc(subproc_exec,
nvproc,
pid2owner,
nv2cuda,
gpu_nvprocs)))
continue
if tasks:
await asyncio.wait(tasks)
tasks.clear()
dead_pids = set(gpu_nvprocs.keys()).difference(seen_pids)
for dead_proc in (gpu_nvprocs[pid] for pid in dead_pids):
await self.change_stream.put(GPUProcess(dead_proc.pid,
pid2owner[
dead_proc.pid],
nv2cuda[
dead_proc.nvdev],
0))
gpu_nvprocs.pop(dead_proc.pid)
seen_pids.clear()
