class ConnectionPool:
def __init__(self):
self._config_dict = None
self._queue = Queue()
self._outstanding_connections = WeakSet()
async def get_conn(self):
self._check_config()
try:
while True:
conn = self._queue.get_nowait()
if conn.is_open():
break
try:
await conn.close()
except Exception:
l.debug('Exception in close rethink connection', exc_info=True)
except QueueEmpty:
conn = await r.connect(**self._config_dict)
self._outstanding_connections.add(conn)
return conn
async def put_conn(self, conn):
self._queue.put_nowait(conn)
self._outstanding_connections.remove(conn)
def set_config(self, config):
self._config_dict = config
def get_config(self):
self._check_config()
return self._config_dict
async def teardown(self):
while True:
try:
conn = self._queue.get_nowait()
except QueueEmpty:
break
self._outstanding_connections.add(conn)
for conn in self._outstanding_connections:
try:
await conn.close()
except Exception:
l.debug('Exception in close rethink connection', exc_info=True)
def _check_config(self):
assert self._config_dict is not None, "Did you remember to run resync.setup()?"
class MockConnection:
def __init__(self):
self.results = Queue()
self.completed_queries = []
async def general_query(self, query, *args, **kwargs):
self.completed_queries.append((query, *args, kwargs))
return self.results.get_nowait()
def __getattr__(self, item):
if item in ('execute', 'fetch', 'fetchval', 'fetchrow'):
return self.general_query
raise Exception('Sorry, {} doesnt exist yet. '
'Consider making a PR.'.format(item))
async def prepare(self, query, *, timeout=None):
return MockPreparedStatement(self, query, None)
async def close(self):
pass
class PyCyrlMultiHander(object):
def __init__(self, max_query=10, loop=get_event_loop()):
self.loop = loop
self.multi = pycurl.CurlMulti()
self.share = pycurl.CurlShare()
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_COOKIE)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_DNS)
self.share.setopt(pycurl.SH_SHARE, pycurl.LOCK_DATA_SSL_SESSION)
self.handles = set()
self.waiting_handles = Queue(loop=self.loop)
self.running = True
self.max_active = max_query
def query(self, handle, future):
def manage_callback(status, headers, data):
handle.close()
future.set_result((status, headers, data))
def failed_callback(ex):
handle.close()
future.set_exception(ex)
handle.cb = manage_callback
handle.f_cb = failed_callback
# put the query in the waiting queue, that launch it if possible
# and wait for the processing to be finished
yield from wait((future, self.waiting_handles.put(handle)),
loop=self.loop)
return future
def close(self):
self.multi.close()
self.share.close()
def _perform_loop(self):
ret, num_handles = self.multi.perform()
# libcurl < 6.20 needed this loop (see https://curl.haxx.se/libcurl/c/libcurl-errors.html#CURLMCALLMULTIPERFORM)
while ret == pycurl.E_CALL_MULTI_PERFORM:
ret, num_handles = self.multi.perform()
return ret, num_handles
def _try_load_queries(self, wait=True, timeout=1.0):
added = 0
while len(self.handles) < self.max_active:
try:
if wait:
handler = yield from wait_for(self.waiting_handles.get(),
timeout,
loop=self.loop)
else:
handler = self.waiting_handles.get_nowait()
# only wait once
wait = False
# needed to keep reference count
self.handles.add(handler)
self.multi.add_handle(handler)
added += 1
except QueueEmpty:
break
except TimeoutError:
break
if added > 0:
ret, num_handles = self._perform_loop()
if ret > 0:
raise ConnectionError("pycurl failed", ret)
def perform(self, timeout=0.1):
"""
Loop on waiting handles to process them until they are no more waiting one and all send are finished.
It's never finished until closed for end of all processing, don't wait for it on loop
:param timeout: the timeout for the loop
:return: Nothing
"""
while self.running:
if len(self.handles) == 0:
# no activity, just sleep, for new queries
yield from self._try_load_queries(True, timeout)
else:
yield from self._try_load_queries(False)
if len(self.handles) == 0:
continue
# wait for something to happen
selected = self.multi.select(timeout)
if selected < 0:
continue
# it was not a select time out, something to do
ret, num_handles = self._perform_loop()
if ret > 0:
raise ConnectionError("pycurl failed", ret)
if len(self.handles) == 0:
continue
else:
# some handles to process
(waiting, succeded, failed) = self.multi.info_read()
for handle in succeded:
self.handles.remove(handle)
self.multi.remove_handle(handle)
status = handle.getinfo(pycurl.RESPONSE_CODE)
content_type, decoded = decode_body(handle)
if not self.running:
# is stopped, just swallow content
continue
elif status >= 200 and status < 300:
handle.cb(status, handle.headers, decoded)
elif status >= 300:
handle.f_cb(
NexusException(
status,
decoded,
content_type,
http_message=handle.headers.pop('__STATUS__'),
url=handle.getinfo(pycurl.EFFECTIVE_URL)))
for handle, code, message in failed:
self.handles.remove(handle)
self.multi.remove_handle(handle)
ex = ConnectionError(code, message)
ex.url = pycurl.EFFECTIVE_URL
ex.code = code
handle.f_cb(ex)
class ClientConnectionPool:
def __init__(self, loop, scheme, host, port, ssl=None, max_size=0):
self.loop = loop
self.scheme = scheme
self.host = host if isinstance(host, str) else host.decode()
self.port = int(port)
self.ssl = self._ssl_option(ssl)
self.max_size = max_size
self._idle_connections = Queue(maxsize=max_size)
self.disposed = False
def _ssl_option(self, ssl):
if self.scheme == b'https':
if ssl is None:
return SECURE_SSLCONTEXT
if ssl is False:
return INSECURE_SSLCONTEXT
if isinstance(ssl, SSLContext):
return ssl
raise InvalidArgument('Invalid ssl argument, expected one of: '
'{None, False, True, instance of ssl.SSLContext}')
if ssl:
raise InvalidArgument('SSL argument specified for non-https scheme.')
return None
def _get_connection(self):
# if there are no connections, let QueueEmpty exception happen
# if all connections are closed, remove all of them and let QueueEmpty exception happen
while True:
connection = self._idle_connections.get_nowait() # type: ClientConnection
if connection.open:
logger.debug(f'Reusing connection {id(connection)} to: {self.host}:{self.port}')
return connection
def try_return_connection(self, connection):
if self.disposed:
return
try:
self._idle_connections.put_nowait(connection)
except QueueFull:
pass
async def get_connection(self):
try:
return self._get_connection()
except QueueEmpty:
return await self.create_connection()
async def create_connection(self):
logger.debug(f'Creating connection to: {self.host}:{self.port}')
transport, connection = await self.loop.create_connection(
lambda: ClientConnection(self.loop, self),
self.host,
self.port,
ssl=self.ssl)
await connection.ready.wait()
# NB: a newly created connection is going to be used by a request-response cycle;
# so we don't put it inside the pool (since it's not immediately reusable for other requests)
return connection
def dispose(self):
self.disposed = True
while True:
try:
connection = self._idle_connections.get_nowait()
except QueueEmpty:
break
else:
logger.debug(f'Closing connection {id(connection)} to: {self.host}:{self.port}')
connection.close()
from sys import stdin, stdout
from asyncio import Queue
suma = 0
harrys = Queue()
monks = []
n = int(stdin.readline().strip())
for x in stdin.readline().split(' '):
num = int(x)
harrys.put_nowait(num)
Q, target = map(int, stdin.readline().split(' '))
res = -1
if suma == target:
res = 0
else:
for q in range(Q):
op = stdin.readline().strip()
if op == "Harry":
num = harrys.get_nowait()
monks.append(num)
suma += num
else:
num = monks.pop()
suma -= num
if suma == target:
res = monks.__len__()
break
stdout.write(str(res) + '\n')
class LiveChatAsync:
'''LiveChatAsync object fetches chat data and stores them
in a buffer with asyncio.
Parameter
---------
video_id : str
seektime : int
start position of fetching chat (seconds).
This option is valid for archived chat only.
If negative value, chat data posted before the start of the broadcast
will be retrieved as well.
processor : ChatProcessor
buffer : Buffer
buffer of chat data fetched background.
interruptable : bool
Allows keyboard interrupts.
Set this parameter to False if your own threading program causes
the problem.
callback : func
function called periodically from _listen().
done_callback : func
function called when listener ends.
exception_handler : func
direct_mode : bool
If True, invoke specified callback function without using buffer.
callback is required. If not, IllegalFunctionCall will be raised.
force_replay : bool
force to fetch archived chat data, even if specified video is live.
topchat_only : bool
If True, get only top chat.
replay_continuation : str
If this parameter is not None, the processor will attempt to get chat data from continuation.
This parameter is only allowed in archived mode.
Attributes
---------
_is_alive : bool
Flag to stop getting chat.
'''
_setup_finished = False
def __init__(self,
video_id,
seektime=-1,
processor=DefaultProcessor(),
buffer=None,
client=httpx.AsyncClient(http2=True),
interruptable=True,
callback=None,
done_callback=None,
exception_handler=None,
direct_mode=False,
force_replay=False,
topchat_only=False,
logger=config.logger(__name__),
replay_continuation=None):
self._client: httpx.AsyncClient = client
self._video_id = util.extract_video_id(video_id)
self.seektime = seektime
if isinstance(processor, tuple):
self.processor = Combinator(processor)
else:
self.processor = processor
self._buffer = buffer
self._callback = callback
self._done_callback = done_callback
self._exception_handler = exception_handler
self._direct_mode = direct_mode
self._is_alive = True
self._is_replay = force_replay or (replay_continuation is not None)
self._parser = Parser(is_replay=self._is_replay)
self._pauser = Queue()
self._pauser.put_nowait(None)
self._first_fetch = replay_continuation is None
self._fetch_url = config._sml if replay_continuation is None else config._smr
self._topchat_only = topchat_only
self._dat = ''
self._last_offset_ms = 0
self._logger = logger
self.exception = None
self.continuation = replay_continuation
LiveChatAsync._logger = logger
if exception_handler:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
(lambda a, b: self._keyboard_interrupt()))
self._setup()
def _setup(self):
# An exception is raised when direct mode is true and no callback is set.
if self._direct_mode:
if self._callback is None:
raise exceptions.IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.")
else:
# Create a default buffer if `direct_mode` is False and buffer is not set.
if self._buffer is None:
self._buffer = Buffer(maxsize=20)
# Create a loop task to call callback if the `callback` param is specified.
if self._callback is None:
pass
else:
# Create a loop task to call callback if the `callback` param is specified.
loop = asyncio.get_event_loop()
loop.create_task(self._callback_loop(self._callback))
# Start a loop task for _listen()
loop = asyncio.get_event_loop()
self.listen_task = loop.create_task(self._startlisten())
# Register add_done_callback
if self._done_callback is None:
self.listen_task.add_done_callback(self._finish)
else:
self.listen_task.add_done_callback(self._done_callback)
async def _startlisten(self):
"""Fetch first continuation parameter,
create and start _listen loop.
"""
if not self.continuation:
channel_id = await util.get_channelid_async(
self._client, self._video_id)
self.continuation = liveparam.getparam(self._video_id,
channel_id,
past_sec=3)
await self._listen(self.continuation)
async def _listen(self, continuation):
''' Fetch chat data and store them into buffer,
get next continuaiton parameter and loop.
Parameter
---------
continuation : str
parameter for next chat data
'''
try:
async with self._client as client:
while (continuation and self._is_alive):
continuation = await self._check_pause(continuation)
contents = await self._get_contents(
continuation, client, headers) #Q#
metadata, chatdata = self._parser.parse(contents)
continuation = metadata.get('continuation')
if continuation:
self.continuation = continuation
timeout = metadata['timeoutMs'] / 1000
chat_component = {
"video_id": self._video_id,
"timeout": timeout,
"chatdata": chatdata
}
time_mark = time.time()
if self._direct_mode:
processed_chat = self.processor.process(
[chat_component])
if isinstance(processed_chat, tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
else:
await self._buffer.put(chat_component)
diff_time = timeout - (time.time() - time_mark)
await asyncio.sleep(diff_time)
self._last_offset_ms = metadata.get('last_offset_ms', 0)
except exceptions.ChatParseException as e:
self._logger.debug(f"[{self._video_id}]{str(e)}")
raise
except Exception:
self._logger.error(f"{traceback.format_exc(limit=-1)}")
raise
self._logger.debug(f"[{self._video_id}] finished fetching chat.")
async def _check_pause(self, continuation):
if self._pauser.empty():
'''pause'''
await self._pauser.get()
'''resume:
prohibit from blocking by putting None into _pauser.
'''
self._pauser.put_nowait(None)
if not self._is_replay:
async with self._client as client:
channel_id = await util.get_channelid_async(
client, self.video_id)
continuation = liveparam.getparam(self._video_id,
channel_id,
past_sec=3)
return continuation
async def _get_contents(self, continuation, client, headers):
'''Get 'continuationContents' from livechat json.
If contents is None at first fetching,
try to fetch archive chat data.
Return:
-------
'continuationContents' which includes metadata & chatdata.
'''
livechat_json = await self._get_livechat_json(
continuation,
client,
replay=self._is_replay,
offset_ms=self._last_offset_ms)
contents, dat = self._parser.get_contents(livechat_json)
if self._dat == '' and dat:
self._dat = dat
if self._first_fetch:
if contents is None or self._is_replay:
'''Try to fetch archive chat data.'''
self._parser.is_replay = True
self._fetch_url = config._smr
channelid = await util.get_channelid_async(
client, self._video_id)
continuation = arcparam.getparam(self._video_id, self.seektime,
self._topchat_only, channelid)
livechat_json = (await self._get_livechat_json(
continuation,
client,
replay=True,
offset_ms=self.seektime * 1000))
reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json)[0])
if reload_continuation:
livechat_json = (await self._get_livechat_json(
reload_continuation, client, headers))
contents, _ = self._parser.get_contents(livechat_json)
self._is_replay = True
self._first_fetch = False
return contents
async def _get_livechat_json(self,
continuation,
client,
replay: bool,
offset_ms: int = 0):
'''
Get json which includes chat data.
'''
livechat_json = None
if offset_ms < 0:
offset_ms = 0
param = util.get_param(continuation,
dat=self._dat,
replay=replay,
offsetms=offset_ms)
for _ in range(MAX_RETRY + 1):
try:
resp = await client.post(self._fetch_url, json=param)
livechat_json = resp.json()
break
except (json.JSONDecodeError, httpx.HTTPError):
await asyncio.sleep(2)
continue
else:
self._logger.error(f"[{self._video_id}]" f"Exceeded retry count.")
raise exceptions.RetryExceedMaxCount()
return livechat_json
async def _callback_loop(self, callback):
""" If a callback is specified in the constructor,
it throws chat data at regular intervals to the
function specified in the callback in the backgroun
Parameter
---------
callback : func
function to which the processed chat data is passed.
"""
while self.is_alive():
items = await self._buffer.get()
processed_chat = self.processor.process(items)
if isinstance(processed_chat, tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
async def get(self):
"""
Retrieves data from the buffer,
throws it to the processor,
and returns the processed chat data.
Returns
: Chat data processed by the Processor
"""
if self._callback is None:
if self.is_alive():
items = await self._buffer.get()
return self.processor.process(items)
else:
return []
raise exceptions.IllegalFunctionCall(
"Callback parameter is already set, so get() cannot be performed.")
def is_replay(self):
return self._is_replay
def pause(self):
if self._callback is None:
return
if not self._pauser.empty():
self._pauser.get_nowait()
def resume(self):
if self._callback is None:
return
if self._pauser.empty():
self._pauser.put_nowait(None)
def is_alive(self):
return self._is_alive
def _finish(self, sender):
'''Called when the _listen() task finished.'''
try:
self._task_finished()
except CancelledError:
self._logger.debug(f'[{self._video_id}] cancelled:{sender}')
def terminate(self):
if not self.is_alive():
return
if self._pauser.empty():
self._pauser.put_nowait(None)
self._is_alive = False
self._buffer.put_nowait({})
self.processor.finalize()
def _keyboard_interrupt(self):
self.exception = exceptions.ChatDataFinished()
self.terminate()
def _task_finished(self):
if self.is_alive():
self.terminate()
try:
self.listen_task.result()
except Exception as e:
self.exception = e
if not isinstance(e, exceptions.ChatParseException):
self._logger.error(f'Internal exception - {type(e)}{str(e)}')
self._logger.info(f'[{self._video_id}] finished.')
def raise_for_status(self):
if self.exception is not None:
raise self.exception
@classmethod
def _set_exception_handler(cls, handler):
loop = asyncio.get_event_loop()
loop.set_exception_handler(handler)
class Executor:
'''
Executor — класс, предназначенный для выполнения
множества задач, которые берутся из Лиры и кладутся
после успешного выполнения туда же. Для корректной
работы необходимо установить следующие члены:
lira - объект lira, откуда берутся задачи и куда
кладутся
fun - функция, которая выполняет задачу и возвращает
True в случае успеха и False — иначе; если
используются асинхронные методы, то и эта
функция должна быть асинхронной
Дополнительно можно установить следующие члены:
name - имя исполнителя (используются в сообщениях
о работе, если silent = False)
silent - флаг, который указывает, нужно ли выводить
сообщения о работе
workerc - количество работников (по умолчанию 5),
которые будут выполнять задания из очереди
(используется только в методе extasks)
Заданием может выступать любой объект, который
имеет булевый член done
'''
def __init__(self, lira: Lira, fun, name='Ex', silent=True, workerc=5):
self.lira = lira
self.fun = fun
self.name = name
self.silent = silent
self.workerc = workerc
self.stopflag = False
self.stoplock = Lock()
self.taskc = None # task count
self.compc = None # complete task count
self.btime = None
self.que = Queue()
self.workers = []
pass
def stop(self):
'Останавливает выполнение задач'
with self.stoplock:
self.stopflag = True
for w in self.workers:
w.cancel()
return
def stopped(self):
'Проверяет, остановлено ли выполнение задач'
with self.stoplock:
res = self.stopflag
return res
def ex(self, id, outcat) -> bool:
'Выполняет задачу id и сообщает об успехе'
task = self.lira.get(id)
if task is None:
raise NoneTaskException(id)
if task.done:
return True
result = self.fun(task)
if not result:
self._print_task_message(task, False)
return False
task.done = True
self.lira.put(task, id, cat=outcat)
if self.compc is not None:
self.compc += 1
self._print_task_message(task, True)
return True
async def ex_async(self, id, outcat) -> bool:
'''
Выполняет задачу id и сообщает об успехе
(асинхронная версия)
'''
task = self.lira.get(id)
if task is None:
raise NoneTaskException(id)
if task.done:
return True
try:
result = await self.fun(task)
except CancelledError as e:
raise e
except Exception as e:
print(e)
raise e
if not result:
self._print_task_message(task, False)
return False
task.done = True
self.lira.put(task, id, cat=outcat)
if self.compc is not None:
self.compc += 1
self._print_task_message(task, True)
return True
async def exque(self, outcat):
try:
while not self.stopped():
try:
id = await self.que.get()
await self.ex_async(id, outcat)
self.que.task_done()
except CancelledError:
if not self.stopped():
return
self.que.task_done()
while not self.que.empty():
self.que.get_nowait()
self.que.task_done()
except Exception as e:
print(e)
return
async def extasks(self, cat, outcat):
'''
Формирует очередь из задач указанной
категории (берутся все объекты из lira)
и исполняет в асинхронном режиме
'''
if not self.silent:
print('\n(%s) BEGIN\n' % self.name, flush=True)
# Формирование очереди
self.que = Queue()
objs = self.lira[cat]
if len(objs) != 0:
for id in objs:
self.que.put_nowait(id)
self.taskc = self.que.qsize()
self.compc = 0
self.btime = dt.datetime.now()
# Создание работников
with self.stoplock:
self.stopflag = False
self.workers = [
create_task(self.exque(outcat)) for _ in range(self.workerc)
]
# Окончание работы
await self.que.join()
for w in self.workers:
w.cancel()
await gather(*self.workers)
self.taskc = None
self.compc = None
self.btime = None
if not self.silent:
print('\n(%s) END\n' % self.name, flush=True)
return
def run_que(self, outcat):
self.workers = [
create_task(self.exque(self.que, outcat))
for _ in range(self.workerc)
]
return
def put(self, task):
id = self.lira.put(task)
self.que.put_nowait(id)
return
def putid(self, id):
self.que.put_nowait(id)
return
async def stop_que(self):
for w in self.workers:
w.cancel()
await gather(*self.workers)
return
async def join_que(self):
await self.que.join()
await self.stop_que()
return
def _print_task_message(self, task, done):
if self.silent:
return
done = 'Done:' if done else 'Fail:'
try:
if self.taskc is not None and self.btime is not None:
delta = (dt.datetime.now() - self.btime).total_seconds()
speed = self.compc / delta
still = (self.taskc - self.compc) / speed
print('(%s | %6.2f%% : %5.1fs) %s %s' %
(self.name, 100 * self.compc / self.taskc, still, done,
str(task)))
else:
print('(%s) %s %s' % (self.name, done, str(task)))
except Exception as e:
print(e)
return
class Platform:
service_running = False
match_updates = None
match_updates_faulty = None
worker_count = 20
task_queue = None
proxy_endpoint = None
def __init__(self, region, platforms, handler):
self.name = region
self.platforms = platforms
self.handler = handler
self.logging = logging.getLogger("%s" % region)
self._worker = [] # Worker promises
self.updater = None # Updater promise
self.retry_after = datetime.now()
self.proxy = handler.proxy
self.endpoint_url = (
f"https://{self.name}.api.riotgames.com/lol/match/v5/matches/%s_%s/timeline"
)
async def init(self):
"""Init background runner."""
self.task_queue = Queue()
self.match_updates = Queue()
self.match_updates_faulty = Queue()
self.logging.info("Ready.")
async def shutdown(self):
self.logging.info("Shutdown")
self.service_running = False
await asyncio.gather(*self._worker, self.updater)
await self.flush_tasks()
async def start(self):
"""Start the service calls."""
if not self.service_running:
self.service_running = True
self.logging.info("Started service calls.")
self.proxy_endpoint = await self.handler.proxy.get_endpoint(
server=self.name, zone="match-timeline-v5"
)
self.updater = asyncio.create_task(self.task_updater())
self._worker = [
asyncio.create_task(self.worker()) for _ in range(self.worker_count)
]
async def stop(self):
"""Halt the service calls."""
if self.service_running:
self.service_running = False
self.logging.info("Stopped service calls.")
await self.updater
for worker in self._worker:
worker.cancel()
try:
await asyncio.gather(*self._worker)
except asyncio.CancelledError:
pass
await self.flush_tasks()
async def task_updater(self):
"""Pull new tasks when the list is empty."""
self.logging.debug("Task Updater initiated.")
while self.service_running:
if self.match_updates.qsize() >= 200:
await self.flush_tasks()
continue
if self.task_queue.qsize() > 200:
await asyncio.sleep(5)
continue
async with self.handler.postgres.acquire() as connection:
entries = await connection.fetch(
"""UPDATE %s.match
SET reserved_timeline = current_date + INTERVAL '10 minute'
FROM (
SELECT match_id,
platform
FROM %s.match
WHERE timeline IS NULL
AND details IS NOT NULL
AND find_fails <= 10
AND (reserved_timeline IS NULL OR reserved_timeline < current_timestamp)
ORDER BY find_fails, match_id DESC
LIMIT $1
) selection
WHERE match.match_id = selection.match_id
AND match.platform = selection.platform
RETURNING match.platform, match.match_id
"""
% tuple([self.name for _ in range(2)]),
1000,
)
self.logging.debug(
"Refilling tasks [%s -> %s].",
self.task_queue.qsize(),
self.task_queue.qsize() + len(entries),
)
if len(entries) == 0:
await asyncio.sleep(30)
await self.flush_tasks()
pass
for entry in entries:
await self.task_queue.put([entry["platform"], entry["match_id"]])
async def worker(self):
"""Execute requests."""
while self.service_running:
for i in range(10):
async with aiohttp.ClientSession(
headers={"X-Riot-Token": self.handler.api_key}
) as session:
task = await self.task_queue.get()
try:
url = self.endpoint_url % (task[0], task[1])
response = await self.proxy_endpoint.request(url, session)
folder = str(task[1])[:5]
path = os.path.join("data", "timeline", task[0], folder)
if not os.path.exists(path):
os.makedirs(path)
filename = os.path.join(path, "%s_%s.json" % (task[0], task[1]))
if not os.path.isfile(filename):
with open(
filename,
"w+",
) as file:
file.write(json.dumps(response))
del response
await self.match_updates.put([task[0], task[1]])
self.task_queue.task_done()
except LimitBlocked as err:
self.retry_after = datetime.now() + timedelta(
seconds=err.retry_after
)
await self.task_queue.put(task)
self.task_queue.task_done()
except aiohttp.ServerDisconnectedError:
self.logging.error("Server Disconnected")
await self.task_queue.put(task)
self.task_queue.task_done()
except RatelimitException:
self.logging.error("Ratelimit")
await self.task_queue.put(task)
self.task_queue.task_done()
except Non200Exception:
self.logging.error("Others")
await self.task_queue.put(task)
self.task_queue.task_done()
except NotFoundException:
await self.match_updates_faulty.put([task[0], task[1]])
self.task_queue.task_done()
except Exception:
self.logging.exception("General Exception")
await self.task_queue.put(task)
self.task_queue.task_done()
async def flush_tasks(self):
"""Insert results from requests into the db."""
match_updates = []
while True:
try:
match_updates.append(self.match_updates.get_nowait())
self.match_updates.task_done()
except asyncio.QueueEmpty:
break
match_not_found = []
while True:
try:
match_not_found.append(self.match_updates_faulty.get_nowait())
self.match_updates_faulty.task_done()
except asyncio.QueueEmpty:
break
async with self.handler.postgres.acquire() as connection:
async with connection.transaction():
if match_updates:
# Insert match updates
query = await connection.prepare(
"""UPDATE %s.match
SET timeline = TRUE,
reserved_timeline = NULL
WHERE platform = $1
AND match_id = $2
"""
% self.name,
)
await query.executemany(match_updates)
if match_not_found:
await connection.execute(
"""UPDATE %s.match
SET find_fails = find_fails + 1,
reserved_timeline = current_date + INTERVAL '10 minute'
WHERE platform::varchar || '_' || match_id::varchar = any($1::varchar[])
"""
% self.name,
["%s_%s" % match for match in match_not_found],
)
if match_updates or match_not_found:
self.logging.info(
"Flushing %s match_updates (%s not found).",
len(match_updates) + len(match_not_found),
len(match_not_found),
)
async def _verify(self, actual: asyncio.Queue, expected: list, items_left: bool):
await asyncio.sleep(0.001)
for expected_msg in expected:
actual_msg = actual.get_nowait()
assert actual_msg == expected_msg
assert items_left or actual.empty()
class TorrentClient:
"""
The torrent client is the local peer that holds peer-to-peer
connections to download and upload pieces for a given torrent.
Once started, the client makes periodic announce calls to the tracker
registered in the torrent meta-data. These calls results in a list of
peers that should be tried in order to exchange pieces.
Each received peer is kept in a queue that a pool of PeerConnection
objects consume. There is a fix number of PeerConnections that can have
a connection open to a peer. Since we are not creating expensive threads
(or worse yet processes) we can create them all at once and they will
be waiting until there is a peer to consume in the queue.
"""
def __init__(self, torrent):
self.tracker = Tracker(torrent)
# The list of potential peers is the work queue, consumed by the
# PeerConnections
self.available_peers = Queue()
# The list of peers is the list of workers that *might* be connected
# to a peer. Else they are waiting to consume new remote peers from
# the `available_peers` queue. These are our workers!
self.peers = []
# The piece manager implements the strategy on which pieces to
# request, as well as the logic to persist received pieces to disk.
self.piece_manager = PieceManager(torrent)
self.abort = False
async def start(self):
"""
Start downloading the torrent held by this client.
This results in connecting to the tracker to retrieve the list of
peers to communicate with. Once the torrent is fully downloaded or
if the download is aborted this method will complete.
"""
self.peers = [
PeerConnection(self.available_peers,
self.tracker.torrent.info_hash,
self.tracker.peer_id, self.piece_manager,
self._on_block_retrieved)
for _ in range(MAX_PEER_CONNECTIONS)
]
# The time we last made an announce call (timestamp)
previous = None
# Default interval between announce calls (in seconds)
interval = 30 * 60
while True:
if self.piece_manager.complete:
logging.info('Torrent fully downloaded!')
break
if self.abort:
logging.info('Aborting download...')
break
current = time.time()
if (not previous) or (previous + interval < current):
response = await self.tracker.connect(
first=previous if previous else False,
uploaded=self.piece_manager.bytes_uploaded,
downloaded=self.piece_manager.bytes_downloaded)
if response:
previous = current
interval = response.interval
self._empty_queue()
for peer in response.peers:
self.available_peers.put_nowait(peer)
else:
await asyncio.sleep(5)
self.stop()
def _empty_queue(self):
while not self.available_peers.empty():
self.available_peers.get_nowait()
def stop(self):
"""
Stop the download or seeding process.
"""
self.abort = True
for peer in self.peers:
peer.stop()
self.piece_manager.close()
self.tracker.close()
def _on_block_retrieved(self, peer_id, piece_index, block_offset, data):
"""
Callback function called by the `PeerConnection` when a block is
retrieved from a peer.
:param peer_id: The id of the peer the block was retrieved from
:param piece_index: The piece index this block is a part of
:param block_offset: The block offset within its piece
:param data: The binary data retrieved
"""
self.piece_manager.block_received(peer_id=peer_id,
piece_index=piece_index,
block_offset=block_offset,
data=data)
class LiveChatAsync:
'''asyncio(aiohttp)を利用してYouTubeのライブ配信のチャットデータを取得する。
Parameter
---------
video_id : str
動画ID
seektime : int
（ライブチャット取得時は無視）
取得開始するアーカイブ済みチャットの経過時間(秒）
マイナス値を指定した場合は、配信開始前のチャットも取得する。
processor : ChatProcessor
チャットデータを加工するオブジェクト
buffer : Buffer(maxsize:20[default])
チャットデータchat_componentを格納するバッファ。
maxsize : 格納できるchat_componentの個数
default値20個。1個で約5~10秒分。
interruptable : bool
Ctrl+Cによる処理中断を行うかどうか。
callback : func
_listen()関数から一定間隔で自動的に呼びだす関数。
done_callback : func
listener終了時に呼び出すコールバック。
exception_handler : func
例外を処理する関数
direct_mode : bool
Trueの場合、bufferを使わずにcallbackを呼ぶ。
Trueの場合、callbackの設定が必須
(設定していない場合IllegalFunctionCall例外を発生させる）
force_replay : bool
Trueの場合、ライブチャットが取得できる場合であっても
強制的にアーカイブ済みチャットを取得する。
topchat_only : bool
Trueの場合、上位チャットのみ取得する。
Attributes
---------
_is_alive : bool
チャット取得を停止するためのフラグ
'''
_setup_finished = False
def __init__(self, video_id,
seektime = 0,
processor = DefaultProcessor(),
buffer = None,
interruptable = True,
callback = None,
done_callback = None,
exception_handler = None,
direct_mode = False,
force_replay = False,
topchat_only = False,
logger = config.logger(__name__),
):
self.video_id = video_id
self.seektime = seektime
if isinstance(processor, tuple):
self.processor = Combinator(processor)
else:
self.processor = processor
self._buffer = buffer
self._callback = callback
self._done_callback = done_callback
self._exception_handler = exception_handler
self._direct_mode = direct_mode
self._is_alive = True
self._is_replay = force_replay
self._parser = Parser(is_replay = self._is_replay)
self._pauser = Queue()
self._pauser.put_nowait(None)
self._setup()
self._first_fetch = True
self._fetch_url = "live_chat/get_live_chat?continuation="
self._topchat_only = topchat_only
self._logger = logger
LiveChatAsync._logger = logger
if not LiveChatAsync._setup_finished:
LiveChatAsync._setup_finished = True
if exception_handler:
self._set_exception_handler(exception_handler)
if interruptable:
signal.signal(signal.SIGINT,
(lambda a, b:asyncio.create_task(
LiveChatAsync.shutdown(None,signal.SIGINT,b))
))
def _setup(self):
#direct modeがTrueでcallback未設定の場合例外発生。
if self._direct_mode:
if self._callback is None:
raise IllegalFunctionCall(
"When direct_mode=True, callback parameter is required.")
else:
#direct modeがFalseでbufferが未設定ならばデフォルトのbufferを作成
if self._buffer is None:
self._buffer = Buffer(maxsize = 20)
#callbackが指定されている場合はcallbackを呼ぶループタスクを作成
if self._callback is None:
pass
else:
#callbackを呼ぶループタスクの開始
loop = asyncio.get_event_loop()
loop.create_task(self._callback_loop(self._callback))
#_listenループタスクの開始
loop = asyncio.get_event_loop()
listen_task = loop.create_task(self._startlisten())
#add_done_callbackの登録
if self._done_callback is None:
listen_task.add_done_callback(self.finish)
else:
listen_task.add_done_callback(self._done_callback)
async def _startlisten(self):
"""Fetch first continuation parameter,
create and start _listen loop.
"""
initial_continuation = liveparam.getparam(self.video_id,3)
await self._listen(initial_continuation)
async def _listen(self, continuation):
''' Fetch chat data and store them into buffer,
get next continuaiton parameter and loop.
Parameter
---------
continuation : str
parameter for next chat data
'''
try:
async with aiohttp.ClientSession() as session:
while(continuation and self._is_alive):
continuation = await self._check_pause(continuation)
contents = await self._get_contents(
continuation, session, headers)
metadata, chatdata = self._parser.parse(contents)
timeout = metadata['timeoutMs']/1000
chat_component = {
"video_id" : self.video_id,
"timeout" : timeout,
"chatdata" : chatdata
}
time_mark =time.time()
if self._direct_mode:
processed_chat = self.processor.process([chat_component])
if isinstance(processed_chat,tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
else:
await self._buffer.put(chat_component)
diff_time = timeout - (time.time()-time_mark)
await asyncio.sleep(diff_time)
continuation = metadata.get('continuation')
except ChatParseException as e:
self._logger.debug(f"[{self.video_id}]{str(e)}")
return
except (TypeError , json.JSONDecodeError) :
self._logger.error(f"{traceback.format_exc(limit = -1)}")
return
self._logger.debug(f"[{self.video_id}]finished fetching chat.")
async def _check_pause(self, continuation):
if self._pauser.empty():
'''pause'''
await self._pauser.get()
'''resume:
prohibit from blocking by putting None into _pauser.
'''
self._pauser.put_nowait(None)
if not self._is_replay:
continuation = liveparam.getparam(
self.video_id, 3, self._topchat_only)
return continuation
async def _get_contents(self, continuation, session, headers):
'''Get 'continuationContents' from livechat json.
If contents is None at first fetching,
try to fetch archive chat data.
Return:
-------
'continuationContents' which includes metadata & chatdata.
'''
livechat_json = (await
self._get_livechat_json(continuation, session, headers)
)
contents = self._parser.get_contents(livechat_json)
if self._first_fetch:
if contents is None or self._is_replay:
'''Try to fetch archive chat data.'''
self._parser.is_replay = True
self._fetch_url = "live_chat_replay/get_live_chat_replay?continuation="
continuation = arcparam.getparam(
self.video_id, self.seektime, self._topchat_only)
livechat_json = (await self._get_livechat_json(
continuation, session, headers))
reload_continuation = self._parser.reload_continuation(
self._parser.get_contents(livechat_json))
if reload_continuation:
livechat_json = (await self._get_livechat_json(
reload_continuation, session, headers))
contents = self._parser.get_contents(livechat_json)
self._is_replay = True
self._first_fetch = False
return contents
async def _get_livechat_json(self, continuation, session, headers):
'''
Get json which includes chat data.
'''
continuation = urllib.parse.quote(continuation)
livechat_json = None
status_code = 0
url =f"https://www.youtube.com/{self._fetch_url}{continuation}&pbj=1"
for _ in range(MAX_RETRY + 1):
async with session.get(url ,headers = headers) as resp:
try:
text = await resp.text()
livechat_json = json.loads(text)
break
except (ClientConnectorError,json.JSONDecodeError) :
await asyncio.sleep(1)
continue
else:
self._logger.error(f"[{self.video_id}]"
f"Exceeded retry count. status_code={status_code}")
return None
return livechat_json
async def _callback_loop(self,callback):
""" コンストラクタでcallbackを指定している場合、バックグラウンドで
callbackに指定された関数に一定間隔でチャットデータを投げる。
Parameter
---------
callback : func
加工済みのチャットデータを渡す先の関数。
"""
while self.is_alive():
items = await self._buffer.get()
processed_chat = self.processor.process(items)
if isinstance(processed_chat, tuple):
await self._callback(*processed_chat)
else:
await self._callback(processed_chat)
async def get(self):
""" bufferからデータを取り出し、processorに投げ、
加工済みのチャットデータを返す。
Returns
: Processorによって加工されたチャットデータ
"""
if self._callback is None:
items = await self._buffer.get()
return self.processor.process(items)
raise IllegalFunctionCall(
"既にcallbackを登録済みのため、get()は実行できません。")
def is_replay(self):
return self._is_replay
def pause(self):
if self._callback is None:
return
if not self._pauser.empty():
self._pauser.get_nowait()
def resume(self):
if self._callback is None:
return
if self._pauser.empty():
self._pauser.put_nowait(None)
def is_alive(self):
return self._is_alive
def finish(self,sender):
'''Listener終了時のコールバック'''
try:
self.terminate()
except CancelledError:
self._logger.debug(f'[{self.video_id}]cancelled:{sender}')
def terminate(self):
'''
Listenerを終了する。
'''
self._is_alive = False
if self._direct_mode == False:
#bufferにダミーオブジェクトを入れてis_alive()を判定させる
self._buffer.put_nowait({'chatdata':'','timeout':0})
self._logger.info(f'[{self.video_id}]finished.')
@classmethod
def _set_exception_handler(cls, handler):
loop = asyncio.get_event_loop()
loop.set_exception_handler(handler)
@classmethod
async def shutdown(cls, event, sig = None, handler=None):
cls._logger.debug("shutdown...")
tasks = [t for t in asyncio.all_tasks() if t is not
asyncio.current_task()]
[task.cancel() for task in tasks]
cls._logger.debug(f"complete remaining tasks...")
await asyncio.gather(*tasks,return_exceptions=True)
loop = asyncio.get_event_loop()
loop.stop()
class WebsocketBase(Client):
"""
A client for interacting with the rippled WebSocket API.
:meta private:
"""
def __init__(self: WebsocketBase, url: str) -> None:
"""
Constructs a WebsocketBase.
Arguments:
url: The URL of the rippled node to submit requests to.
"""
self.url = url
self._open_requests: Dict[str, Future[Dict[str, Any]]] = {}
self._websocket: Optional[WebSocketClientProtocol] = None
self._handler_task: Optional[Task[None]] = None
self._messages: Optional[Queue[Dict[str, Any]]] = None
def is_open(self: WebsocketBase) -> bool:
"""
Returns whether the client is currently open.
Returns:
Whether the client is currently open.
"""
return (self._handler_task is not None and self._messages is not None
and self._websocket is not None and self._websocket.open)
async def _do_open(self: WebsocketBase) -> None:
"""Connects the client to the Web Socket API at the given URL."""
if self.is_open():
return
# open the connection
self._websocket = await connect(self.url)
# make a message queue
self._messages = Queue()
# start the handler
self._handler_task = create_task(self._handler())
async def _do_close(self: WebsocketBase) -> None:
"""Closes the connection."""
if not self.is_open():
return
assert self._handler_task is not None # mypy
assert self._websocket is not None # mypy
assert self._messages is not None # mypy
# cancel the handler
self._handler_task.cancel()
self._handler_task = None
# cancel any pending request Futures
for future in self._open_requests.values():
future.cancel()
self._open_requests = {}
# clear the message queue
for _ in range(self._messages.qsize()):
self._messages.get_nowait()
self._messages.task_done()
self._messages = None
# close the connection
await self._websocket.close()
async def _handler(self: WebsocketBase) -> None:
"""
This is basically a middleware for the websocket library. For all received
messages we check whether there is an outstanding future we need to resolve,
and if so do so.
Then we store the already-parsed JSON in our own queue for generic iteration.
As long as a given client remains open, this handler will be running as a Task.
"""
assert self._websocket is not None # mypy
assert self._messages is not None # mypy
async for response in self._websocket:
response_dict = json.loads(response)
# if this response corresponds to request, fulfill the Future
if "id" in response_dict and response_dict[
"id"] in self._open_requests:
self._open_requests[response_dict["id"]].set_result(
response_dict)
# enqueue the response for the message queue
self._messages.put_nowait(response_dict)
def _set_up_future(self: WebsocketBase, request: Request) -> None:
"""
Only to be called from the public send and request_impl functions.
Given a request with an ID, ensure that that ID is backed by an open
Future in self._open_requests.
"""
if request.id is None:
return
request_str = str(request.id)
if (request_str in self._open_requests
and not self._open_requests[request_str].done()):
raise XRPLWebsocketException(
f"Request {request_str} is already in progress.")
self._open_requests[request_str] = get_running_loop().create_future()
async def _do_send_no_future(self: WebsocketBase,
request: Request) -> None:
assert self._websocket is not None # mypy
await self._websocket.send(json.dumps(
request_to_websocket(request), ), )
async def _do_send(self: WebsocketBase, request: Request) -> None:
# we need to set up a future here, even if no one cares about it, so
# that if a user submits a few requests with the same ID they fail.
self._set_up_future(request)
await self._do_send_no_future(request)
async def _do_pop_message(self: WebsocketBase) -> Dict[str, Any]:
assert self._messages is not None # mypy
msg = await self._messages.get()
self._messages.task_done()
return msg
async def request_impl(self: WebsocketBase, request: Request) -> Response:
"""
Asynchronously submits the request represented by the request to the
rippled node specified by this client's URL and waits for a response.
Note: if this is used for an API method that returns many responses, such as
`subscribe`, this method only returns the first response; all subsequent
responses will be available if you use the async iterator pattern on this
client, IE `async for message in client`. You can create an async task to
read messages from subscriptions.
Arguments:
request: An object representing information about a rippled request.
Returns:
The response from the server, as a Response object.
Raises:
XRPLWebsocketException: If there is already an open request by the
request's ID, or if this WebsocketBase is not open.
:meta private:
"""
if not self.is_open():
raise XRPLWebsocketException("Websocket is not open")
# if no ID on this request, generate and inject one, and ensure it
# is backed by a future
request_with_id = _inject_request_id(request)
request_str = str(request_with_id.id)
self._set_up_future(request_with_id)
# fire-and-forget the send, and await the Future
create_task(self._do_send_no_future(request_with_id))
raw_response = await self._open_requests[request_str]
# remove the resolved Future, hopefully getting it garbage colleted
del self._open_requests[request_str]
return websocket_to_response(raw_response)
class M3u8Downloader(object):
def __init__(self, url, path, worker_num, ts_timeout=120, loop=None):
self.path = path
self.ts_timeout = ts_timeout
self.cache_dir = path + ".cache"
if not os.path.exists(self.cache_dir):
os.mkdir(self.cache_dir)
self.loop = loop or asyncio.get_event_loop()
self.ts_list = M3u8Parser(url).ts
self.worker_num = worker_num
self.q = Queue()
self._ts_name_list = []
self._fill_q()
def _fill_q(self):
for index, ts in enumerate(self.ts_list):
name = str(index) + ".ts"
self.q.put_nowait((name, ts))
self._ts_name_list.append(name)
async def worker(self):
async with aiohttp.ClientSession() as session:
while not self.q.empty():
name, ts = self.q.get_nowait()
path = os.path.join(self.cache_dir, name)
if os.path.exists(path):
print("{name} already exists".format(name=name))
else:
try:
await self.download(session, path, ts)
except:
print("retry download {name}".format(name=name))
self.q.put_nowait((name, ts))
self.q.task_done()
async def download(self, session, path, ts):
print("start download {p} / {total}".format(p=path,
total=len(self.ts_list)))
async with async_timeout.timeout(self.ts_timeout):
async with session.get(ts, headers=headers) as response:
with open(path + ".downloading", "wb") as fd:
while True:
chunk = await response.content.read(1024)
if not chunk:
break
fd.write(chunk)
os.rename(path + ".downloading", path)
async def run(self):
# download all .ts
print("start download...")
workers = [
asyncio.Task(self.worker(), loop=self.loop)
for _ in range(self.worker_num)
]
await self.q.join()
for w in workers:
w.cancel()
print("download complete , start merge...")
self.merge_all_ts()
print("remove .ts ...")
shutil.rmtree(self.cache_dir)
def merge_all_ts(self):
command = '{ffmpeg_path} -y -i concat:{path_list} -acodec copy -vcodec copy -absf aac_adtstoasc {output}'.format(
path_list="|".join([
os.path.join(self.cache_dir, name)
for name in self._ts_name_list
]),
ffmpeg_path=ffmpeg_path,
output=self.path).split(" ")
p = subprocess.Popen(command)
p.wait()
class Writer:
"""
"""
terminator = "\n"
delay = 1.0
def __init__(self, levelno=logging.DEBUG):
self.queue = Queue()
self.canWrite = Event()
self.levelno = levelno
def filterMessage(self, message) -> bool:
if not isinstance(message, Message) or message.type != "log":
return False # ignore invalid
if message.levelno < self.levelno:
return False # filter level
return True
async def start(self):
loop = get_running_loop()
while True:
try:
await self.canWrite.wait()
if not self.check():
self.canWrite.clear()
continue
message = await self.queue.get()
if not self.filterMessage(message):
self.queue.task_done()
continue # avoid acquiring the lock
await loop.run_in_executor(None, self.acquire)
while True:
if self.filterMessage(message):
await loop.run_in_executor(
None, self.emit, message.msg, message.levelno
)
self.queue.task_done()
try: # handle any other records while we have the lock
message = self.queue.get_nowait()
except QueueEmpty:
break
await loop.run_in_executor(None, self.release)
await sleep(self.delay) # rate limit
except CancelledError:
break # exit the writer
except Exception: # catch all
logging.warning(f"Caught exception in {self.__class__.__name__}. Stopping", exc_info=True)
self.canWrite.clear()
def check(self) -> bool:
return True
def acquire(self):
pass
@abstractmethod
def emit(self, msg: str, levelno: int):
raise NotImplementedError()
def release(self):
pass
def main():
url = "http://isoredirect.centos.org/centos/7/isos/x86_64"
f = urllib.request.urlopen(url)
html = f.read()
source = BeautifulSoup(html, 'html.parser')
threads = []
queue = Queue()
# Testing each url for speed
for link in source.find_all('a'):
href = str(link.string)
if href.startswith('http') and 'x86_64' in href:
thread = Thread(target=TestSpeed, args=(href, queue))
thread.start()
threads.append(thread)
for th in threads:
th.join()
bestspeed = 0.2
while not queue.empty():
speedtest = queue.get_nowait()
besturl = speedtest['url']
speed = speedtest['speed']
if args.verbose:
print(str(speed) + " - Mirror: " + url)
if speed < bestspeed:
bestspeed = speed
besturl = url
print("Best speed: " + besturl + " (speed: " + str(bestspeed) + ")")
# If verbose enable interactive
isoAsk = "n"
if args.verbose and not args.list:
isoAsk = raw_input('Download latest CentOS7? (Y/n): ')
while isoAsk != "Y" and isoAsk != "n":
isoAsk = raw_input('Download latest CentOS7? (Y/n): ')
# As before, fetch page
f = urllib.request.urlopen(besturl)
html_linksPage = f.read()
linksPage = BeautifulSoup(html_linksPage, 'html.parser')
# For each link
for link in linksPage.find_all('a'):
href = str(link.string)
if ('torrent' not in href and 'Minimal' in href):
file_name = href.rstrip()
if args.list and not args.tree:
print("Best download option: " + besturl + "/" + file_name)
exit()
if args.tree:
dirtree = besturl.replace("isos", "os")
print("Use directory tree: " + dirtree)
exit()
# Downloading the ISO
if isoAsk == "Y" or args.verbose == False:
with open(file_name, "wb") as f:
print("Downloading %s" % file_name)
response = requests.get(besturl + "/" + file_name, stream=True)
total_length = response.headers.get('content-length')
if total_length is None:
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
done = int(50 * dl / total_length)
sys.stdout.write("\r[%s%s]" % ('=' * done, ' ' *
(50 - done)))
sys.stdout.flush()
print()
else:
print("Quitting")
class Stream:
"""
API for working with streams, used by clients and request handlers
"""
id = None
__buffer__ = None
__wrapper__ = None
def __init__(self,
connection: Connection,
h2_connection: H2Connection,
transport: Transport,
*,
loop: AbstractEventLoop,
stream_id: Optional[int] = None,
wrapper: Optional[Wrapper] = None) -> None:
self._connection = connection
self._h2_connection = h2_connection
self._transport = transport
self._loop = loop
self.__wrapper__ = wrapper
if stream_id is not None:
self.id = stream_id
self.__buffer__ = Buffer(self.id,
self._connection,
self._h2_connection,
loop=self._loop)
self.__headers__ = Queue(loop=loop) \
# type: Queue[List[Tuple[str, str]]]
self.__window_updated__ = Event(loop=loop)
async def recv_headers(self):
return await self.__headers__.get()
def recv_headers_nowait(self):
try:
return self.__headers__.get_nowait()
except QueueEmpty:
return None
async def recv_data(self, size):
return await self.__buffer__.read(size)
async def send_request(self, headers, end_stream=False, *, _processor):
assert self.id is None, self.id
while True:
# this is the first thing we should check before even trying to
# create new stream, because this wait() can be cancelled by timeout
# and we wouldn't need to create new stream at all
if not self._connection.write_ready.is_set():
await self._connection.write_ready.wait()
# `get_next_available_stream_id()` should be as close to
# `connection.send_headers()` as possible, without any async
# interruptions in between, see the docs on the
# `get_next_available_stream_id()` method
stream_id = self._h2_connection.get_next_available_stream_id()
try:
self._h2_connection.send_headers(stream_id,
headers,
end_stream=end_stream)
except TooManyStreamsError:
# we're going to wait until any of currently opened streams will
# be closed, and we will be able to open a new one
# TODO: maybe implement FIFO for waiters, but this limit
# shouldn't be reached in a normal case, so why bother
# TODO: maybe we should raise an exception here instead of
# waiting, if timeout wasn't set for the current request
self._connection.stream_close_waiter.clear()
await self._connection.stream_close_waiter.wait()
# while we were trying to create a new stream, write buffer
# can became full, so we need to repeat checks from checking
# if we can write() data
continue
else:
self.id = stream_id
self.__buffer__ = Buffer(self.id,
self._connection,
self._h2_connection,
loop=self._loop)
release_stream = _processor.register(self)
self._transport.write(self._h2_connection.data_to_send())
return release_stream
async def send_headers(self, headers, end_stream=False):
assert self.id is not None
if not self._connection.write_ready.is_set():
await self._connection.write_ready.wait()
# Workaround for the H2Connection.send_headers method, which will try
# to create a new stream if it was removed earlier from the
# H2Connection.streams, and therefore will raise StreamIDTooLowError
if self.id not in self._h2_connection.streams:
raise StreamClosedError(self.id)
self._h2_connection.send_headers(self.id,
headers,
end_stream=end_stream)
self._transport.write(self._h2_connection.data_to_send())
async def send_data(self, data, end_stream=False):
f = BytesIO(data)
f_pos, f_last = 0, len(data)
while True:
if not self._connection.write_ready.is_set():
await self._connection.write_ready.wait()
window = self._h2_connection.local_flow_control_window(self.id)
if not window:
self.__window_updated__.clear()
await self.__window_updated__.wait()
window = self._h2_connection.local_flow_control_window(self.id)
max_frame_size = self._h2_connection.max_outbound_frame_size
f_chunk = f.read(min(window, max_frame_size, f_last - f_pos))
f_pos = f.tell()
if f_pos == f_last:
self._h2_connection.send_data(self.id,
f_chunk,
end_stream=end_stream)
self._transport.write(self._h2_connection.data_to_send())
break
else:
self._h2_connection.send_data(self.id, f_chunk)
self._transport.write(self._h2_connection.data_to_send())
async def end(self):
if not self._connection.write_ready.is_set():
await self._connection.write_ready.wait()
self._h2_connection.end_stream(self.id)
self._transport.write(self._h2_connection.data_to_send())
async def reset(self, error_code=ErrorCodes.NO_ERROR):
if not self._connection.write_ready.is_set():
await self._connection.write_ready.wait()
self._h2_connection.reset_stream(self.id, error_code=error_code)
self._transport.write(self._h2_connection.data_to_send())
def reset_nowait(self, error_code=ErrorCodes.NO_ERROR):
self._h2_connection.reset_stream(self.id, error_code=error_code)
if self._connection.write_ready.is_set():
self._transport.write(self._h2_connection.data_to_send())
def __ended__(self):
self.__buffer__.eof()
def __terminated__(self, reason):
if self.__wrapper__ is not None:
self.__wrapper__.cancel(StreamTerminatedError(reason))
@property
def closable(self):
if self._h2_connection.state_machine.state is ConnectionState.CLOSED:
return False
stream = self._h2_connection.streams.get(self.id)
if stream is None:
return False
return not stream.closed
async def project_runner(
idx: int,
config: Dict[str, Any],
queue: asyncio.Queue,
work_path: Path,
results: Results,
long_checkouts: bool = False,
rebase: bool = False,
keep: bool = False,
no_diff: bool = False,
) -> None:
"""Check out project and run Black on it + record result"""
loop = asyncio.get_event_loop()
py_version = f"{version_info[0]}.{version_info[1]}"
while True:
try:
project_name = queue.get_nowait()
except asyncio.QueueEmpty:
LOG.debug(f"project_runner {idx} exiting")
return
LOG.debug(f"worker {idx} working on {project_name}")
project_config = config["projects"][project_name]
# Check if disabled by config
if "disabled" in project_config and project_config["disabled"]:
results.stats["disabled"] += 1
LOG.info(f"Skipping {project_name} as it's disabled via config")
continue
# Check if we should run on this version of Python
if (
"all" not in project_config["py_versions"]
and py_version not in project_config["py_versions"]
):
results.stats["wrong_py_ver"] += 1
LOG.debug(f"Skipping {project_name} as it's not enabled for {py_version}")
continue
# Check if we're doing big projects / long checkouts
if not long_checkouts and project_config["long_checkout"]:
results.stats["skipped_long_checkout"] += 1
LOG.debug(f"Skipping {project_name} as it's configured as a long checkout")
continue
repo_path: Optional[Path] = Path(__file__)
stdin_project = project_name.upper() == "STDIN"
if not stdin_project:
repo_path = await git_checkout_or_rebase(work_path, project_config, rebase)
if not repo_path:
continue
await black_run(project_name, repo_path, project_config, results, no_diff)
if not keep and not stdin_project:
LOG.debug(f"Removing {repo_path}")
rmtree_partial = partial(
rmtree, path=repo_path, onerror=handle_PermissionError
)
await loop.run_in_executor(None, rmtree_partial)
LOG.info(f"Finished {project_name}")
class TorrentClient:
def __init__(self, torrent):
self.tracker = Tracker(torrent)
self.available_peers = Queue()
self.peers = []
self.piece_manager = PieceManager(torrent)
self.abort = False
async def start(self):
self.peers = [
PeerConnection(self.available_peers,
self.tracker.torrent.info_hash,
self.tracker.peer_id, self.piece_manager,
self._on_block_retrieved)
for _ in range(MAX_PEER_CONNECTIONS)
]
previous = None
interval = 30 * 60
while True:
if self.piece_manager.complete:
logging.info('Torrent fully downloaded!')
break
if self.abort:
logging.info('Aborting download...')
break
current = time.time()
if (not previous) or (previous + interval < current):
response = await self.tracker.connect(
first=previous if previous else False,
uploaded=self.piece_manager.bytes_uploaded,
downloaded=self.piece_manager.bytes_downloaded)
if response:
previous = current
interval = response.interval
self._empty_queue()
for peer in response.peers:
self.available_peers.put_nowait(peer)
else:
await asyncio.sleep(5)
self.stop()
def _empty_queue(self):
while not self.available_peers.empty():
self.available_peers.get_nowait()
def stop(self):
self.abort = True
for peer in self.peers:
peer.stop()
self.piece_manager.close()
self.tracker.close()
def _on_block_retrieved(self, peer_id, piece_index, block_offset, data):
self.piece_manager.block_received(peer_id=peer_id,
piece_index=piece_index,
block_offset=block_offset,
data=data)
class ProxyResponse(object):
'''Asynchronous wsgi response.
'''
_started = False
_headers = None
_done = False
def __init__(self, environ, start_response):
self._loop = environ['pulsar.connection']._loop
self.environ = environ
self.start_response = start_response
self.queue = Queue()
def __iter__(self):
while True:
if self._done:
try:
yield self.queue.get_nowait()
except QueueEmpty:
break
else:
yield async(self.queue.get(), loop=self._loop)
def pre_request(self, response, exc=None):
self._started = True
response.bind_event('data_processed', self.data_processed)
return response
def error(self, exc):
if not self._started:
request = wsgi.WsgiRequest(self.environ)
content_type = request.content_types.best_match(
('text/html', 'text/plain'))
uri = self.environ['RAW_URI']
msg = 'Could not find %s' % uri
logger.info(msg=msg)
if content_type == 'text/html':
html = wsgi.HtmlDocument(title=msg)
html.body.append('<h1>%s</h1>' % msg)
data = html.render()
resp = wsgi.WsgiResponse(504, data, content_type='text/html')
elif content_type == 'text/plain':
resp = wsgi.WsgiResponse(504, msg, content_type='text/html')
else:
resp = wsgi.WsgiResponse(504, '')
self.start_response(resp.status, resp.get_headers())
self._done = True
self.queue.put_nowait(resp.content[0])
def data_processed(self, response, exc=None, **kw):
'''Receive data from the requesting HTTP client.'''
status = response.get_status()
if status == '100 Continue':
stream = self.environ.get('wsgi.input') or io.BytesIO()
body = yield stream.read()
response.transport.write(body)
if response.parser.is_headers_complete():
if self._headers is None:
headers = self.remove_hop_headers(response.headers)
self._headers = Headers(headers, kind='server')
# start the response
self.start_response(status, list(self._headers))
body = response.recv_body()
if response.parser.is_message_complete():
self._done = True
self.queue.put_nowait(body)
def remove_hop_headers(self, headers):
for header, value in headers:
if header.lower() not in wsgi.HOP_HEADERS:
yield header, value
class Mailbox(Generic[T]):
"""Similar to `asyncio.Queue`_, but able to perform "selective receive".
See `~.select`.
"""
# ---- Queue API ----
def __init__(
self,
maxsize=0,
*,
_queue: Optional[Queue[T]] = None,
_pending: Optional[List[T]] = None,
):
"""PRIVATE ARGS: _queue and _pending are not part of the public API)."""
self._queue = Queue(maxsize) if _queue is None else _queue
self._pending = [] if _pending is None else _pending
@property
def maxsize(self):
return self._queue.maxsize
def empty(self):
return self._queue.empty() and len(self._pending) < 1
async def get(self):
"""See `asyncio.Queue.get`_."""
if len(self._pending) > 0:
return self._pending.pop()
return await self._queue.get()
def full(self):
if self.maxsize <= 0 or self.qsize() < self.maxsize:
return False
return True
def get_nowait(self, default=NO_VALUE):
"""Similar to `asyncio.Queue.get_nowait`_, but can return a default value.
When the argument ``default`` is passed, instead of raising an exception if the
queue is empty, the default value is passed.
"""
if len(self._pending) > 0:
return self._pending.pop()
try:
return self._queue.get_nowait()
except asyncio.QueueEmpty:
if default is not NO_VALUE:
return default
raise
async def join(self):
return await self._queue.join()
async def put(self, item: T):
await self._queue.put(item)
def put_nowait(self, item: T):
if self.full():
raise asyncio.QueueFull
self._queue.put_nowait(item)
def qsize(self) -> int:
return len(self._pending) + self._queue.qsize()
def task_done(self):
self._queue.task_done()
# ---- Select API ----
@overload # noqa
async def select(self, *args: Topic) -> Tuple[T, ...]:
...
@overload # noqa
async def select(self, selection: Dict[Topic, Callable[[T],
S]]) -> S: # noqa
...
async def select(self, *args): # noqa
selectors, callbacks, value, callback = self._select_common(args)
if value is NO_VALUE:
value, callback = await self._select_on_queue(selectors, callbacks)
if callback:
return callback(value)
return value
def select_nowait(self, *args, default=NO_VALUE): # noqa
selectors, callbacks, value, callback = self._select_common(args)
if value is NO_VALUE:
value, callback = self._select_nowait(selectors, callbacks,
default)
if callback:
return callback(value)
return value
def _select_common(self, args):
"""This internal method avoid code duplication between get and get_nowait,
encapsulating the common parts.
"""
n = len(args)
if n == 1 and isinstance(args[0], dict):
selectors = list(args[0].keys())
callbacks = list(args[0].values())
else:
selectors = args
callbacks = []
selectors = [pattern(s) for s in selectors]
value, callback = self._select_on_pending(selectors, callbacks)
return selectors, callbacks, value, callback
def _select_on_pending(
self, selectors: List[Pattern], callbacks: Optional[List[Callable[[T],
S]]]
) -> Tuple[OptVal[T], Optional[Callable[[T], S]]]:
for i in range(len(self._pending)):
value, callback = _select_value(self._pending[i], selectors,
callbacks)
if value is not NO_VALUE:
del self._pending[i]
return value, callback
return NO_VALUE, None
async def _select_on_queue(
self, selectors: List[Pattern], callbacks: Optional[List[Callable[[T],
S]]]
) -> Tuple[OptVal[T], Optional[Callable[[T], S]]]:
while True:
item = await self._queue.get()
value, callback = _select_value(item, selectors, callbacks)
if value is not NO_VALUE:
return value, callback
else:
self._pending.append(item)
def _select_nowait(
self,
selectors: List[Pattern],
callbacks: Optional[List[Callable[[T], S]]],
default: OptVal[T] = NO_VALUE,
) -> Tuple[OptVal[T], Optional[Callable[[T], S]]]:
try:
while True:
item = self._queue.get_nowait()
value, callback = _select_value(item, selectors, callbacks)
if value is not NO_VALUE:
return value, callback
else:
self._pending.append(item)
except asyncio.QueueEmpty:
if default is not NO_VALUE:
return default, None
raise
def _size(self):
return f"[queued: {self._queue.qsize()} + pending: {len(self._pending)}]"
def __str__(self):
return f"<{type(self).__name__}{self._size()}>"
def __repr__(self):
return (f"<{type(self).__name__} at {id(self):#x} "
f"queue={self._queue}, pending={self._pending}>")
from sys import stdin,stdout
from asyncio import Queue
suma = 0
harrys = Queue()
monks = []
n = int(stdin.readline().strip())
for x in stdin.readline().split(' '):
num = int(x)
harrys.put_nowait(num)
Q,target = map(int,stdin.readline().split(' '))
res = -1
if suma == target:
res = 0
else:
for q in range(Q):
op = stdin.readline().strip()
if op == "Harry":
num = harrys.get_nowait()
monks.append(num)
suma += num
else:
num = monks.pop()
suma -= num
if suma == target:
res = monks.__len__()
break
stdout.write(str(res)+'\n')
def fit(self, dataset):
nodes_at_start = Node.totalcount
self.Xnames = dataset.Xnames
self.Ynames = dataset.Ynames
self.root = Node(dataset, parent=None)
self.lc_nodes = 0
que = Queue()
que.put_nowait(self.root)
while not que.empty():
c = que.get_nowait()
if len(c.data.Ynames) == 1:
# One class remaining, create an answer node
for answername, answer in c.data.Ynames.items():
assert(c.answer is None)
c.answer = node_types.Answer(answer, answername, c.data.Y.size)
continue
if len(c.data.Ynames) == 2 and self.use_lc:
# Possible to crate a linear classifier here
# Conjunction (penalty='l1' loss='squared_hinge' dual=True) not supported
lc = LinearSVC(penalty='l1', tol=0.000001, C=10000.0,
dual=False, fit_intercept=True, random_state=42)
# Create a mask to only include features that
# do not contain the same value in all samples
feature_mask = []
for i, dom in enumerate(c.data.Xdomains):
assert(len(dom) >= 1)
feature_mask.append(len(dom) > 1)
# Create a scaler to transform data before fitting
scaler = StandardScaler()
X_transformed = scaler.fit_transform(c.data.X[:,feature_mask])
lc.fit(X_transformed, c.data.Y)
sc = accuracy_score(normalize=False, y_true=c.data.Y,
y_pred=lc.predict(X_transformed))
if sc == c.data.Y.size:
# Linear classifier is correct, make a leaf node with it
c.line = node_types.Line(lc, feature_mask, scaler,
list(c.data.Ynames.keys()), c.data.Y.size)
self.lc_nodes += 1
continue
# We need to split here
c.predicate = self.split_criterion.best(c.data, c)
mask, s_Xdomains, u_Xdomains = c.predicate.evaluate_domains(c.data.X)
Ynames_back = {}
for name, idx in c.data.Ynames.items():
assert(idx not in Ynames_back)
Ynames_back[idx] = name
# SAT
s_X = c.data.X[mask]
s_Y = c.data.Y[mask]
s_Xnames = c.data.Xnames
s_Ynames = {}
s_Yids = set()
for y in s_Y:
if y not in s_Yids:
s_Yids.add(y)
assert(y in Ynames_back and Ynames_back[y] not in s_Ynames)
s_Ynames[Ynames_back[y]] = y
c.childSAT = Node(Dataset(s_X, s_Y, s_Xnames, s_Xdomains, s_Ynames,
c.data.Xineqforbidden.copy(),
c.data.ActionIDtoName.copy(),
c.data.ModuleIDtoName.copy()),
c)
que.put_nowait(c.childSAT)
# UNSAT
u_X = c.data.X[~mask]
u_Y = c.data.Y[~mask]
u_Xnames = c.data.Xnames
u_Ynames = {}
u_Yids = set()
for y in u_Y:
if y not in u_Yids:
u_Yids.add(y)
assert(y in Ynames_back and Ynames_back[y] not in u_Ynames)
u_Ynames[Ynames_back[y]] = y
c.childUNSAT = Node(Dataset(u_X, u_Y, u_Xnames, u_Xdomains, u_Ynames,
c.data.Xineqforbidden.copy(),
c.data.ActionIDtoName.copy(),
c.data.ModuleIDtoName.copy()),
c)
que.put_nowait(c.childUNSAT)
# Clean up previous dataset
c.data = None
assert(dataset is not None) # c.data was pointer copy, not reference
# Finished
self.nodes = Node.totalcount - nodes_at_start
