def run_server(self, loop: asyncio.BaseEventLoop):
asyncio.set_event_loop(loop)
self.loop = loop
handler = self.app.make_handler()
server = loop.create_server(handler, host=self.local, port=self.port)
loop.run_until_complete(server)
loop.run_forever()
def start(self, loop: asyncio.BaseEventLoop):
self.ws = websockets.serve(lambda ws, p: self.handler(ws, p),
self.host,
self.port,
loop=loop)
loop.run_until_complete(self.ws)
log.debug("Started websocket server.")
log.info(f"Started websocket bridge on {self.host}:{self.port}")
def asyncio_shutdown(loop: asyncio.BaseEventLoop = asyncio.get_event_loop()) -> None:
try:
asyncio.runners._cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
if sys.version_info.minor > 8:
loop.run_until_complete(loop.shutdown_default_executor())
finally:
asyncio.set_event_loop(None)
loop.close()
def start(self, loop: asyncio.BaseEventLoop):
app = web.Application()
app.add_routes([web.get("/{command}", lambda r: self.handler(r))])
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
self.site = web.TCPSite(runner, self.host, self.port)
loop.run_until_complete(self.site.start())
log.info(f"Serving HTTP bridge on port {self.port}")
def start_background_lock_extender(lock_manager: Aioredlock, lock: Lock,
loop: asyncio.BaseEventLoop) -> None:
"""Will periodically extend the duration of the lock"""
async def extender_worker(lock_manager: Aioredlock):
sleep_interval = 0.9 * config.REDLOCK_REFRESH_INTERVAL_SECONDS
while True:
await lock_manager.extend(lock,
config.REDLOCK_REFRESH_INTERVAL_SECONDS)
await asyncio.sleep(sleep_interval)
loop.run_until_complete(extender_worker(lock_manager))
def distribute_global_resources(
loop: asyncio.BaseEventLoop,
blob_client: azure.storage.blob.BlockBlobService,
queue_client: azure.storage.queue.QueueService,
table_client: azure.storage.table.TableService,
ipaddress: str) -> None:
"""Distribute global services/resources
:param asyncio.BaseEventLoop loop: event loop
:param azure.storage.blob.BlockBlobService blob_client: blob client
:param azure.storage.queue.QueueService queue_client: queue client
:param azure.storage.table.TableService table_client: table client
:param str ipaddress: ip address
"""
# set torrent session port listen
if _ENABLE_P2P:
global _TORRENT_SESSION
# create torrent session
logger.info('creating torrent session on {}:{}'.format(
ipaddress, _DEFAULT_PORT_BEGIN))
_TORRENT_SESSION = libtorrent.session()
_TORRENT_SESSION.listen_on(_DEFAULT_PORT_BEGIN, _DEFAULT_PORT_END)
_TORRENT_SESSION.stop_lsd()
_TORRENT_SESSION.stop_upnp()
_TORRENT_SESSION.stop_natpmp()
# bootstrap dht nodes
bootstrap_dht_nodes(loop, table_client, ipaddress, 0)
_TORRENT_SESSION.start_dht()
# get globalresources from table
try:
entities = table_client.query_entities(
_STORAGE_CONTAINERS['table_globalresources'],
filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY))
except azure.common.AzureMissingResourceHttpError:
entities = None
nentities = 0
# check torrent info table for resource
if entities is not None:
for ent in entities:
nentities += 1
if _ENABLE_P2P:
_check_resource_has_torrent(blob_client, table_client,
ent['Resource'])
else:
with _DIRECTDL_LOCK:
_DIRECTDL.append(ent['Resource'])
if nentities == 0:
logger.info('no global resources specified')
return
# run async func in loop
loop.run_until_complete(
download_monitor_async(loop, blob_client, queue_client, table_client,
ipaddress, nentities))
def start(self, loop: asyncio.BaseEventLoop):
ws_future = websockets.serve(
lambda ws, p: self.handler(ws, p),
self.host,
self.port,
subprotocols=["com.microsoft.minecraft.wsencrypt"],
ping_interval=None,
loop=loop,
)
self.ws = loop.run_until_complete(ws_future)
loop.run_until_complete(
self.set_uninitiated_handler_status(ready=False))
log.info(f"Started Minecraft connector on {self.host}:{self.port}")
def start_child_thread_loop(
loop: asyncio.BaseEventLoop, break_sign: BreakSign,
main_fetch_callback: Callable[[Union[Future, SyncFuture]], None],
child_fetch_callback: Callable[[Union[Future, SyncFuture]], None],
request_package_list: RequestPackageList, headers: Union[dict, None],
total_timeout: int, concurrency_number: int) -> None:
downloader_logger.debug('Async downloader start.')
asyncio.set_event_loop(loop)
main_fetch_task = loop.create_task(
main_fetch(request_package_list, break_sign, child_fetch_callback,
headers, total_timeout, concurrency_number))
main_fetch_task.add_done_callback(main_fetch_callback)
loop.run_until_complete(main_fetch_task)
downloader_logger.debug('Async download thread over.')
def feeds_check(*feed_names,
data_dir: str = None,
loop: asyncio.BaseEventLoop = None):
"""Check feeds for name validity.
:raises: ValueError if any feed name is invalid
"""
loop = loop or asyncio.get_event_loop()
# remove local feeds
distinct_feeds = list(
filter(
lambda f: not FeedManager.feeds_exist(
f, data_dir=data_dir, loop=loop), feed_names))
futures = [
JSONFeedMetadata.url_exists(feed, loop=loop)
for feed in distinct_feeds
]
tasks = asyncio.gather(*futures)
results = loop.run_until_complete(tasks)
invalid = [
feed for valid, feed in zip(results, distinct_feeds) if not valid
]
if any(invalid):
raise ValueError(f"Invalid feeds found: {invalid}")
def aiohttp_raw_server(event_loop: asyncio.BaseEventLoop, unused_tcp_port):
from aiohttp.test_utils import RawTestServer
servers = []
async def go(handler):
server = RawTestServer(handler, port=unused_tcp_port)
await server.start_server()
servers.append(server)
return server
yield go
async def finalize() -> None:
while servers:
await servers.pop().close()
event_loop.run_until_complete(finalize())
def get_ip(service_list: Iterable[Service] = None,
loop: asyncio.BaseEventLoop = None) -> str:
if service_list is None:
service_list = services()
if loop is None:
loop = asyncio.get_event_loop()
ip_task = _get_ip(service_list)
ip = loop.run_until_complete(ip_task)
return ip
def monitor(self, loop: asyncio.BaseEventLoop):
"""Monitor for new messages published to the subscribed Redis channel.
When a message is received, it will be published to all
available websockets associated with a specific user ID.
"""
# `listen()` blocks forever so we monitor for messages
# in a separate event loop in a new thread running as a daemon.
asyncio.set_event_loop(loop)
return loop.run_until_complete(self._handle_new_messages())
def distribute_global_resources(loop: asyncio.BaseEventLoop,
blob_client: azureblob.BlockBlobService,
table_client: azuretable.TableService) -> None:
"""Distribute global services/resources
:param asyncio.BaseEventLoop loop: event loop
:param azureblob.BlockBlobService blob_client: blob client
:param azuretable.TableService table_client: table client
"""
# remove node from the image table because cascade relies on it to know
# when its work is done
_unmerge_resources(table_client)
# get globalresources from table
try:
entities = table_client.query_entities(
_STORAGE_CONTAINERS['table_globalresources'],
filter='PartitionKey eq \'{}\''.format(_PARTITION_KEY))
except azure.common.AzureMissingResourceHttpError:
entities = []
nentities = 0
for ent in entities:
resource = ent['Resource']
grtype, image = get_container_image_name_from_resource(resource)
if grtype == _CONTAINER_MODE.name.lower():
nentities += 1
_DIRECTDL_QUEUE.put(resource)
key_fingerprint = ent.get('KeyFingerprint', None)
if key_fingerprint is not None:
_DIRECTDL_KEY_FINGERPRINT_DICT[image] = key_fingerprint
else:
logger.info('skipping resource {}:'.format(resource) +
'not matching container mode "{}"'.format(
_CONTAINER_MODE.name.lower()))
if nentities == 0:
logger.info('no global resources specified')
return
logger.info('{} global resources matching container mode "{}"'.format(
nentities, _CONTAINER_MODE.name.lower()))
# run async func in loop
loop.run_until_complete(
download_monitor_async(loop, blob_client, table_client, nentities))
def check_proxy_async(proxies: list, loop: asyncio.BaseEventLoop) -> list:
"""异步检查代理是否可用
`proxies`: 代理列表
`loop`: 事件循环
"""
tasks = []
url = "http://httpbin.org/ip"
for proxy in proxies:
task = asyncio.ensure_future(
send_request(url=url, method="get", proxy=proxy))
tasks.append(task)
result = loop.run_until_complete(asyncio.gather(*tasks))
check_result = [
True if item["origin"] != "false" else False for item in result
]
return check_result
def fake_aioredis(request: FixtureRequest, loop: asyncio.BaseEventLoop):
r: aioredis.Redis = loop.run_until_complete(
aioredis.create_redis('redis://localhost:6379', db=2,
encoding='utf-8'))
loop.run_until_complete(r.flushdb())
redis_objects = filter(
lambda i: i != db.RedisObject and issubclass(i, db.RedisObject),
filter(lambda i: type(i) == type, map(lambda i: getattr(db, i),
dir(db))))
for redis_object in redis_objects:
redis_object.register_db(r)
yield r
loop.run_until_complete(r.flushdb())
r._pool_or_conn.close()
loop.run_until_complete(r._pool_or_conn.wait_closed())
def client(loop: asyncio.BaseEventLoop, test_server, test_client,
credentials_checker, token_provider):
app = aiohttp.web.Application(loop=loop)
db = None # TODO
app_server = Server(app, credentials_checker, token_provider, db, loop=loop)
loop.run_until_complete(app_server.start())
server = loop.run_until_complete(test_server(app))
client = loop.run_until_complete(test_client(server))
yield client
loop.run_until_complete(app_server.stop())
def start_async_request(urls: list,
loop: asyncio.BaseEventLoop,
method: Optional[str] = "get",
text_type: Optional[str] = "json") -> list:
"""调用异步请求同时发送多条请求并返回请求的响应
`urls`: url列表
`loop`: 事件循环
`method`: http请求的方法,默认为get,支持post
`text_type`: 返回的响应的格式,默认的为json,支持text,binary
"""
tasks = []
for url in urls:
task = asyncio.ensure_future(
send_request(url, method, text_type=text_type))
tasks.append(task)
result = loop.run_until_complete(asyncio.gather(*tasks))
return result
def feeds_exist(*feed_names,
data_dir: str = None,
loop: asyncio.BaseEventLoop = None) -> bool:
"""Check feeds whether exist locally.
:raises: ValueError if feed does not exist.
"""
loop = loop or asyncio.get_event_loop()
futures = [
JSONFeedMetadata.metadata_exist(feed, data_dir=data_dir, loop=loop)
for feed in feed_names
]
tasks = asyncio.gather(*futures)
results = loop.run_until_complete(tasks)
for valid, feed in zip(results, feed_names):
if not valid:
return False
return True
def start_manager(self, loop: asyncio.BaseEventLoop):
loop.run_until_complete(self.run_manager())
def _after_stop(self, loop: BaseEventLoop):
for (hook_name, hook_handler) in self._hooks:
if hook_name == HookTypes.after_server_stop:
loop.run_until_complete(hook_handler(context))
def _thread_run(_thread_loop: asyncio.BaseEventLoop):
# Running on thread that will host proc_loop
async def __run():
# Running within proc_loop
# asyncio.get_event_loop().set_debug(True)
try:
# print("Server is launching", cmd, *args, flush=True)
proc = await asyncio.create_subprocess_exec(
cmd, *args,
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
async def __process_output(_out: asyncio.StreamReader, _output_callback: Callable):
# Runs within proc_loop
try:
while True:
buf = b''
line = None
while line is None:
try:
# Handle an incomplete line output such as when
# a command prompt leaves the input cursor at the end.
c = await asyncio.wait_for(_out.read(1), 0.1)
except asyncio.futures.TimeoutError:
if buf:
line = buf
# except Exception as ex:
# print("Exception", type(ex), ex, file=sys.stderr, flush=True)
# pass
else:
buf += c
if c == b'\n':
line = buf
# Handle EOF
elif c == b'':
line = buf
if line:
# First send whatever line we have left
part = partial(_output_callback, line)
asyncio.run_coroutine_threadsafe(output_callback_queue.put(part),
parent_loop)
# Then send a marker saying we're done
part = partial(_output_callback, None)
asyncio.run_coroutine_threadsafe(output_callback_queue.put(part), parent_loop)
return
if line:
part = partial(_output_callback, line)
asyncio.run_coroutine_threadsafe(output_callback_queue.put(part), parent_loop)
else:
break
except Exception as ex:
print("Error in __process_output:", ex.__class__.__name__, ex, file=sys.stderr, flush=True)
traceback.print_tb(sys.exc_info()[2])
async def __receive_input(_input_provider: AsyncIterator[str]):
# Runs in parent_loop
# asyncio.get_event_loop().set_debug(True)
async for __line in _input_provider:
proc.stdin.write(f"{__line}\n".encode())
proc.stdin.write_eof()
# input_done_evt.set()
tasks = []
if provide_stdin:
asyncio.run_coroutine_threadsafe(__receive_input(provide_stdin), parent_loop)
# parent_loop_tasks.add(parent_loop.create_task(input_done_evt.wait()))
if handle_stdout:
tasks.append(_thread_loop.create_task(__process_output(proc.stdout, handle_stdout)))
if handle_stderr:
tasks.append(_thread_loop.create_task(__process_output(proc.stderr, handle_stderr)))
# print("GATHERING...", flush=True)
await asyncio.gather(*tasks)
# print(f"GATHERED {pprint.pformat(tasks)}", flush=True)
except Exception as ex:
print(ex, file=sys.stderr, flush=True)
traceback.print_tb(sys.exc_info()[2])
asyncio.set_event_loop(_thread_loop)
_thread_loop.run_until_complete(__run())
parent_loop.call_soon_threadsafe(thread_done_evt.set)
# parent_loop.call_soon_threadsafe(input_done_evt.set)
print("Thread-proc run closed.", flush=True)
def client(application: web.Application, loop: BaseEventLoop,
aiohttp_client: Any, mocker: MockerFixture) -> TestClient:
mocker.patch("pathlib.Path.iterdir", return_value=[])
return loop.run_until_complete(aiohttp_client(application))
def loop_in_thread(event_loop: asyncio.BaseEventLoop, function: Callable):
asyncio.set_event_loop(event_loop)
event_loop.run_until_complete(function())
def qaueue_client(loop: asyncio.BaseEventLoop, aiohttp_client,
qaueue_app: web.Application):
yield loop.run_until_complete(aiohttp_client(qaueue_app))
def aloop(loop: asyncio.BaseEventLoop, f, a):
asyncio.set_event_loop(loop)
loop.run_until_complete(f(a))
def __call__(self, loop: asyncio.BaseEventLoop = None) -> Any:
if loop is None:
loop = asyncio.get_event_loop()
return loop.run_until_complete(self.wait())
