def filter_stderr(popen:subprocess.Popen, future:asyncio.Future):
last_ex = None
while True:
data = popen.stderr.readline()
if data:
# print("FFmpeg says:", data, flush=True)
try:
if check_stderr(data):
sys.stderr.buffer.write(data)
sys.stderr.buffer.flush()
except FFmpegError as e:
print("Error from FFmpeg:", e)
last_ex = e
except FFmpegWarning:
pass # useless message
else:
break
if last_ex:
future.set_exception(last_ex)
else:
future.set_result(True)
def filter_stderr(popen:subprocess.Popen, future:asyncio.Future):
last_ex = None
while True:
data = popen.stderr.readline()
if data:
log.ffmpeg("Data from ffmpeg: {}".format(data))
try:
if check_stderr(data):
sys.stderr.buffer.write(data)
sys.stderr.buffer.flush()
except FFmpegError as e:
log.ffmpeg("Error from ffmpeg: %s", str(e).strip())
last_ex = e
except FFmpegWarning:
pass # useless message
else:
break
if last_ex:
future.set_exception(last_ex)
else:
future.set_result(True)
def start_tls(self, server_side, ssl_options=None, server_hostname=None, connect_timeout=None):
if not self._transport or self._read_future:
raise ValueError("IOStream is not idle; cannot convert to SSL")
self._connect_ssl_future = connect_ssl_future = Future(loop=self._loop)
waiter = Future(loop=self._loop)
def on_connected(future):
if self._loop_connect_timeout:
self._loop_connect_timeout.cancel()
self._loop_connect_timeout = None
if connect_ssl_future._exception is not None:
self.on_closed(future.exception())
self._connect_ssl_future = None
else:
self._connect_ssl_future = None
connect_ssl_future.set_result(self)
waiter.add_done_callback(on_connected)
if connect_timeout:
def on_timeout():
self._loop_connect_timeout = None
if not waiter.done():
self.close((None, IOError("Connect timeout"), None))
self._loop_connect_timeout = self._loop.call_later(connect_timeout, on_timeout)
self._transport.pause_reading()
sock, self._transport._sock = self._transport._sock, None
self._transport = self._loop._make_ssl_transport(
sock, self, ssl_options, waiter,
server_side=False, server_hostname=server_hostname)
return connect_ssl_future
class Tester:
def __init__(self):
self.end = Future()
def __call__(self, *args, **kwargs):
if not self.end.done():
self.end.set_result((args, kwargs))
def test_json_with_async_string2(self):
d = Future()
astr = wsgi.AsyncString(d)
response = wsgi.Json({'bla': astr})
self.assertEqual(len(response.children), 1)
result = response.render()
self.assertIsInstance(result, Future)
d.set_result('ciao')
result = yield result
self.assertEqual(result, json.dumps({'bla': 'ciao'}))
def side_effect(*args, **kwargs):
def _side_effect(*args, **kwargs):
fut = Future()
fut.set_result({'origin': '127.0.0.1'})
return fut
resp = Mock()
resp.json.side_effect = resp.release.side_effect = _side_effect
fut = Future()
fut.set_result(resp)
return fut
def js_query(self, query) -> Future:
if self.connected:
self.js_exec(query, self._reqid)
fut = Future()
self._tasks[self._reqid] = fut
self._reqid += 1
return fut
else:
fut = Future()
fut.set_result(None)
return fut
def wait_for_fd(fd, *, loop=None): """Given a file descriptor, block on it until we have input to read""" if hasattr(fd, 'fileno'): fd = fd.fileno() if not loop: loop = get_event_loop() waiter = Future(loop=loop) loop.add_reader(fd, lambda : waiter.set_result(None)) yield from waiter loop.remove_reader(fd)
class CustomWSClientProtocol(WebSocketClientProtocol):
"""Add auto-ping switch (dirty way) and let us start handshaking manually."""
# this framework mix camel and underline naming style, nice!
def __init__(self):
WebSocketClientProtocol.__init__(self)
self.customUriPath = '/'
self.customWsKey = None
self._delayedHandshake = Future()
def setAutoPing(self, interval, timeout):
"""Set auto-ping interval. Start it if it's not running."""
self.disableAutoPing()
self.autoPingInterval = interval
self.autoPingTimeout = timeout
self.autoPingPendingCall = loop.call_later(interval, self._sendAutoPing)
def disableAutoPing(self):
if self.autoPingPendingCall:
self.autoPingPendingCall.cancel()
self.autoPingPendingCall = None
def startHandshake(self):
"""Delay handshake because some states must be set right before handshake (so
they can't be set in factory)."""
self._delayedHandshake.set_result(None)
@coroutine
def restartHandshake(self):
"""Resume delayed handshake. It enable us to customize handshake HTTP header."""
yield from wait_for(self._delayedHandshake, None)
if config.compatible:
self.websocket_key = base64.b64encode(os.urandom(16))
else:
self.websocket_key = self.customWsKey
request = [
'GET %s HTTP/1.1' % self.customUriPath,
'Host: %s:%d' % (self.factory.host, self.factory.port),
'Sec-WebSocket-Key: %s' % self.websocket_key.decode(),
'Sec-WebSocket-Version: %d' % self.SPEC_TO_PROTOCOL_VERSION[self.version],
'Pragma: no-cache',
'Cache-Control: no-cache',
'Connection: Upgrade',
'Upgrade: WebSocket',
]
if config.compatible:
# store custom ws key in cookie to prevent it from being changed by ws proxy
request.append('Cookie: %s=%s' % (config.cookie_key, self.customWsKey.decode()))
if self.factory.useragent:
request.append('User-Agent: %s' % self.factory.useragent)
self.http_request_data = '\r\n'.join(request).encode('utf8') + b'\r\n\r\n'
self.sendData(self.http_request_data)
if self.debug:
self.log.debug(request)
async def test_run_coroutine_job(asyncio_scheduler, asyncio_executor, exception):
from asyncio import Future, sleep
future = Future()
job = asyncio_scheduler.add_job(waiter, 'interval', seconds=1, args=[sleep, exception])
asyncio_executor._run_job_success = lambda job_id, events: future.set_result(events)
asyncio_executor._run_job_error = lambda job_id, exc, tb: future.set_exception(exc)
asyncio_executor.submit_job(job, [datetime.now(utc)])
events = await future
assert len(events) == 1
if exception:
assert str(events[0].exception) == 'dummy error'
else:
assert events[0].retval is True
def __init__(self, title='', label_text='', completer=None):
self.future = Future()
def accept_text(buf):
get_app().layout.focus(ok_button)
buf.complete_state = None
return True
def accept():
self.future.set_result(self.text_area.text)
def cancel():
self.future.set_result(None)
self.text_area = TextArea(
completer=completer,
multiline=False,
width=D(preferred=40),
accept_handler=accept_text)
ok_button = Button(text='OK', handler=accept)
cancel_button = Button(text='Cancel', handler=cancel)
self.dialog = Dialog(
title=title,
body=HSplit([
Label(text=label_text),
self.text_area
]),
buttons=[ok_button, cancel_button],
width=D(preferred=80),
modal=True)
def go():
future = Future()
future.set_result(42)
source = rx.from_future(future)
def on_next(x):
success[0] = x == 42
def on_error(err):
success[1] = False
def on_completed():
success[2] = True
source.subscribe(on_next, on_error, on_completed)
def go():
future = Future()
future.set_result(42)
source = Observable.from_future(future)
def on_next(x):
success[0] = 42 == x
def on_error(err):
success[1] = False
def on_completed():
success[2] = True
subscription = source.subscribe(on_next, on_error, on_completed)
def _send_request(self, app_name, endpoint, entity, params):
packet = MessagePacket.request(self.name, self.version, app_name, _Service._REQ_PKT_STR, endpoint, params,
entity)
future = Future()
request_id = params['request_id']
self._pending_requests[request_id] = future
try:
self.tcp_bus.send(packet)
except ClientException as e:
if not future.done() and not future.cancelled():
ERROR = '101_Client not found'
exception = RequestException(ERROR)
exception.error = ERROR
future.set_exception(exception)
_Service.time_future(future, TCPServiceClient.REQUEST_TIMEOUT_SECS)
return future
def _send_request(self, app_name, endpoint, entity, params, timeout):
packet = MessagePacket.request(self.name, self.version, app_name, _Service._REQ_PKT_STR, endpoint, params,
entity)
future = Future()
request_id = params['request_id']
self._pending_requests[request_id] = future
try:
self.tcp_bus.send(packet)
except ClientException:
if not future.done() and not future.cancelled():
error = 'Client not found'
exception = ClientException(error)
exception.error = error
future.set_exception(exception)
_Service.time_future(future, timeout)
return future
def _on_reply(self, action, name, on_reply_ok, on_reply_error):
def callback(future):
reply = future.result()
if query['id'] == reply['id'] and reply['status'] == 'ok':
on_reply_ok(reply)
elif query['id'] == reply['id']:
on_reply_error(reply['reason'].capitalize() + '.')
else:
raise Exception()
query = {'id': uuid4().hex, 'command': action}
query['properties'] = {'name': name}
if action == 'incr' or action == 'decr':
query['properties'].update({'waiting': False, 'nb': 1})
elif action == 'start' or action == 'stop':
query['properties'].update({'waiting': False, 'match': 'glob'})
# Block to prevent writing multiple requests before reading a reply.
if self._rep2:
yield from self._rep2
self._rep2 = Future()
self._rep2.add_done_callback(callback)
self._req2.write([dumps(query).encode()])
async def _uwsgi_entry_point(self, current_greenlet: greenlet.greenlet, future: asyncio.Future,
*args, **kwargs):
"""
uWSGI wrapper entry point.
"""
try:
# Call the underlying wrapped function.
result = await func(self, *args, **kwargs)
except Exception as e:
# Set the exception on the Future, allowing the web server to retrieve it.
future.set_exception(e)
else:
# Set the result of the function on the Future.
future.set_result(result)
finally:
# Switch back context.
current_greenlet.switch()
def go():
future = Future()
future.set_result(42)
source = rx.from_future(future)
def on_next(x):
success[0] = False
def on_error(err):
success[1] = False
def on_completed():
success[2] = False
subscription = source.subscribe(on_next, on_error, on_completed)
subscription.dispose()
def go():
error = Exception('woops')
future = Future()
future.set_exception(error)
source = Observable.from_future(future)
def on_next(x):
success[0] = False
def on_error(err):
success[1] = str(err) == str(error)
def on_completed():
success[2] = False
subscription = source.subscribe(on_next, on_error, on_completed)
def multi_future(children, quiet_exceptions=()):
"""
Wraps multiple futures in a single future.
:param quiet_exceptions:
:param children:
"""
if isinstance(children, dict):
keys = list(children.keys())
children = list(children.values())
else:
keys = None
unfinished_children = set(children)
future = Future()
if not children:
future.set_result({} if keys is not None else [])
def callback(ft):
unfinished_children.remove(ft)
if not unfinished_children:
result_list = []
for ft in children:
try:
result_list.append(ft.result())
except Exception as e:
if future.done():
if not isinstance(e, quiet_exceptions):
print("Multiple exceptions in yield list",
file=sys.stderr)
else:
future.set_exception(sys.exc_info())
if not future.done():
if keys is not None:
future.set_result(dict(list(zip(keys, result_list))))
else:
future.set_result(result_list)
listening = set()
for f in children:
if f not in listening:
listening.add(f)
f.add_done_callback(callback)
return future
def doreqcb(self, jobh: asyncio.Future):
fures = jobh.result()
res = fures[2]
req = fures[1]
reqid = fures[0]
# qDebug(str(res.status_code) + ', ' + str(res.headers))
self._res_map[reqid] = [req, res]
self._req_map.pop(reqid)
self.reqFinished.emit(reqid)
return
def _consume(self):
self.waiter = Future(loop=self.factory.loop or txaio.config.loop)
def process(_):
while len(self.receive_queue):
data = self.receive_queue.popleft()
if self.transport:
self._dataReceived(data)
self._consume()
self.waiter.add_done_callback(process)
def get(self):
"""Remove and return an item from the channel.
If channel is empty, wait until an item is available.
This method is a coroutine.
"""
while self.empty() and not self._close.is_set():
getter = Future(loop=self._loop)
self._getters.append(getter)
try:
yield from getter
except ChannelClosed:
raise
except:
getter.cancel() # Just in case getter is not done yet.
if not self.empty() and not getter.cancelled():
# We were woken up by put_nowait(), but can't take
# the call. Wake up the next in line.
self._wakeup_next(self._getters)
raise
return self.get_nowait()
class MessageDialog:
def __init__(self, title, text):
self.future = Future()
def set_done():
self.future.set_result(None)
ok_button = Button(text='OK', handler=(lambda: set_done()))
self.dialog = Dialog(
title=title,
body=HSplit([
Label(text=text),
]),
buttons=[ok_button],
width=D(preferred=80),
modal=True)
def __pt_container__(self):
return self.dialog
async def _inner_wrapper_ordered(arg, future: asyncio.Future, loop):
try:
error = None
ret = None
try:
ret = await self._coro_fun(*arg, loop=loop)
except asyncio.CancelledError as e:
raise e
except Exception as e:
error = e
if self._sema is not None:
await self._sema.acquire()
async with self._wakeup_iterator:
if error is not None:
future.set_error(ret)
else:
future.set_result(ret)
self._wakeup_iterator.notify_all()
except asyncio.CancelledError:
pass
async def read(self, amt):
if not self.size:
self.next_size = Future(loop=self.sock.loop)
if self.chunk_read:
self.chunk_read.set_result(None)
try:
[self.size, self.chunk_read] = await self.next_size
except EOFError:
return b""
data = await self.sock.recv(min(self.size, amt))
self.size -= len(data)
return data
def as_future(fun, *args, **kwargs):
"""
Executes a function with the given arguments
and wraps the result into a future.
:param fun: The function to be called.
:type fun: func
:param args: The argument list for the supplied function.
:type args: list
:param kwargs: The keyword argument dict for the supplied function.
:type kwargs: dict
:return: The functions result wrapped in a Future
:rtype: asyncio.Future
"""
try:
res = fun(*args, **kwargs)
except Exception as e:
f = Future()
f.set_exception(e)
return f
else:
if isinstance(res, Future):
return res
elif iscoroutine(res):
return asyncio.Task(res)
else:
f = Future()
f.set_result(res)
return f
def put(self, item):
"""Put an item into the channel.
If the channel is full, wait until a free
slot is available before adding item.
If the channel is closed or closing, raise ChannelClosed.
This method is a coroutine.
"""
while self.full() and not self._close.is_set():
putter = Future(loop=self._loop)
self._putters.append(putter)
try:
yield from putter
except ChannelClosed:
raise
except:
putter.cancel() # Just in case putter is not done yet.
if not self.full() and not putter.cancelled():
# We were woken up by get_nowait(), but can't take
# the call. Wake up the next in line.
self._wakeup_next(self._putters)
raise
return self.put_nowait(item)
def _consume(self):
self.waiter = Future()
def process(_):
while len(self.receive_queue):
data = self.receive_queue.popleft()
if self.transport:
self._dataReceived(data)
else:
print("WebSocketAdapterProtocol._consume: no transport")
self._consume()
self.waiter.add_done_callback(process)
async def _launcher(self, coro, callback_future: asyncio.Future = None):
exception = None
try:
ret = await coro
except Exception as e:
exception = e
pass
finally:
self._current_workers -= 1
while self._current_workers < self._max_workers:
waiting_future = None
while len(self._pending_queue) > 0:
generator = self._pending_queue[0]
try:
waiting_future = next(generator)
break
except StopIteration:
self._pending_queue.popleft()
continue
if waiting_future is None:
break
self._current_workers += 1
asyncio.ensure_future(waiting_future, loop=self._loop)
if self._idle():
async with self._waiter:
self._waiter.notify_all()
if exception is not None:
if callback_future is None:
raise exception
else:
callback_future.set_exception(exception)
return
if callback_future is None:
return ret
else:
callback_future.set_result(ret)
return
class MySubHandler:
"""
More advanced subscription client using Future, so we can await events in tests.
"""
def __init__(self):
self.future = Future()
def reset(self):
self.future = Future()
async def result(self):
return await wait_for(self.future, 2)
def datachange_notification(self, node, val, data):
if not self.future.done():
self.future.set_result((node, val, data))
def event_notification(self, event):
if not self.future.done():
self.future.set_result(event)
async def _async_poll_for_reply(
self, msg_id: str, cell: NotebookNode, timeout: t.Optional[int],
task_poll_output_msg: asyncio.Future,
task_poll_kernel_alive: asyncio.Future) -> t.Dict:
assert self.kc is not None
new_timeout: t.Optional[float] = None
if timeout is not None:
deadline = monotonic() + timeout
new_timeout = float(timeout)
while True:
try:
msg = await ensure_async(
self.kc.shell_channel.get_msg(timeout=new_timeout))
if msg['parent_header'].get('msg_id') == msg_id:
if self.record_timing:
cell['metadata']['execution'][
'shell.execute_reply'] = timestamp()
try:
await asyncio.wait_for(task_poll_output_msg,
self.iopub_timeout)
except (asyncio.TimeoutError, Empty):
if self.raise_on_iopub_timeout:
task_poll_kernel_alive.cancel()
raise CellTimeoutError.error_from_timeout_and_cell(
"Timeout waiting for IOPub output",
self.iopub_timeout, cell)
else:
self.log.warning(
"Timeout waiting for IOPub output")
task_poll_kernel_alive.cancel()
return msg
else:
if new_timeout is not None:
new_timeout = max(0, deadline - monotonic())
except Empty:
# received no message, check if kernel is still alive
assert timeout is not None
task_poll_kernel_alive.cancel()
await self._async_check_alive()
await self._async_handle_timeout(timeout, cell)
class AIterator:
def __init__(self):
self.notifications = []
self.future = Future()
self.disposable = source.materialize().subscribe(self.on_next)
def __aiter__(self):
return self
def dispose(self):
self.disposable.dispose()
def feeder(self):
if not self.notifications or self.future.done():
return
notification = self.notifications.pop(0)
kind = notification.kind
if kind == "N":
self.future.set_result(notification.value)
if kind == "E":
self.future.set_exception(notification.exception)
if kind == "C":
self.future.set_exception(StopAsyncIteration)
def on_next(self, notification):
self.notifications.append(notification)
self.feeder()
async def __anext__(self):
self.feeder()
value = await self.future
self.future = Future()
return value
def _on_connection_lost(self, future: asyncio.Future, connection: AsyncioConnection, code, reason):
for callback in self._on_connection_lost_callbacks:
callback(self)
if self._closed:
return super()._on_connection_lost(future, connection, code, reason)
if isinstance(reason, ProbableAuthenticationError):
if not future.done():
future.set_exception(reason)
self.loop.create_task(self.close())
return
if not future.done():
future.set_result(None)
self.loop.call_later(
self.reconnect_interval,
lambda: self.loop.create_task(self.connect())
)
async def _poll_for_reply(
self,
msg_id: str,
timeout: t.Optional[int],
task_poll_output_msg: asyncio.Future,
task_poll_kernel_alive: asyncio.Future,
) -> t.Dict:
assert self.kc is not None
new_timeout: t.Optional[float] = None
if timeout is not None:
deadline = monotonic() + timeout
new_timeout = float(timeout)
while True:
try:
msg = await ensure_async(self.kc.shell_channel.get_msg(timeout=new_timeout))
if msg["parent_header"].get("msg_id") == msg_id:
try:
await asyncio.wait_for(task_poll_output_msg, self.iopub_timeout)
except (asyncio.TimeoutError, Empty):
if self.raise_on_iopub_timeout:
task_poll_kernel_alive.cancel()
raise ExecTimeoutError("Timeout waiting for IOPub output")
else:
self.log.warning("Timeout waiting for IOPub output")
task_poll_kernel_alive.cancel()
return msg
else:
if new_timeout is not None:
new_timeout = max(0, deadline - monotonic())
except Empty:
# received no message, check if kernel is still alive
assert timeout is not None
task_poll_kernel_alive.cancel()
await self._check_alive()
await self._handle_timeout(timeout)
class CatchupSubscription(ReadStreamEventsBehaviour,
PageStreamEventsBehaviour):
def __init__(
self,
stream,
start_from=0,
batch_size=100,
credential=None,
conversation_id=None,
):
self.stream = stream
self.iterator = StreamingIterator()
self.conversation_id = conversation_id or uuid4()
self._logger = logging.get_named_logger(CatchupSubscription,
self.conversation_id)
self.from_event = start_from
self.direction = StreamDirection.Forward
self.batch_size = batch_size
self.has_first_page = False
self.require_master = False
self.resolve_link_tos = True
self.credential = credential
self.result = Future()
self.phase = CatchupSubscriptionPhase.READ_HISTORICAL
self.buffer = []
self.subscribe_from = -1
self.next_event_number = self.from_event
self.last_event_number = -1
Conversation.__init__(self, conversation_id, credential)
ReadStreamEventsBehaviour.__init__(self, ReadStreamResult,
proto.ReadStreamEventsCompleted)
@property
def is_live(self):
return self.phase == CatchupSubscriptionPhase.LIVE
async def error(self, exn) -> None:
if self.result.done():
await self.subscription.raise_error(exn)
else:
self.result.set_exception(exn)
async def reconnect(self, output: Queue) -> None:
self.phase = CatchupSubscriptionPhase.RECONNECT
self.buffer = []
await self.subscription.unsubscribe()
async def start(self, output):
if self.phase > CatchupSubscriptionPhase.READ_HISTORICAL:
self._logger.info("Tear down previous subscription")
await self.reconnect(output)
return
self._logger.info("Starting catchup subscription at %s",
self.from_event)
self.from_event = max(self.from_event, self.next_event_number,
self.last_event_number)
await PageStreamEventsBehaviour.start(self, output)
async def drop_subscription(self, response: InboundMessage) -> None:
body = proto.SubscriptionDropped()
body.ParseFromString(response.payload)
if body.reason == messages.SubscriptionDropReason.Unsubscribed:
await self.subscription.events.enqueue(StopAsyncIteration())
return
if self.result.done():
await self.error(
exceptions.SubscriptionFailed(self.conversation_id,
body.reason))
return
await self.error(
exceptions.SubscriptionCreationFailed(self.conversation_id,
body.reason))
async def _yield_events(self, events):
for event in events:
if event.event_number <= self.last_event_number:
continue
await self.iterator.asend(event)
self.last_event_number = event.event_number
async def _move_to_next_phase(self, output):
if self.phase == CatchupSubscriptionPhase.READ_HISTORICAL:
self.phase = CatchupSubscriptionPhase.CATCH_UP
self._logger.info(
"Caught up with historical events, creating volatile subscription"
)
await self._subscribe(output)
elif self.phase == CatchupSubscriptionPhase.CATCH_UP:
self.phase = CatchupSubscriptionPhase.LIVE
await self._yield_events(self.buffer)
async def reply_from_live(self, message, output):
if message.command == TcpCommand.SubscriptionDropped:
await self.drop_subscription(message)
return
self.expect_only(TcpCommand.StreamEventAppeared, message)
result = proto.StreamEventAppeared()
result.ParseFromString(message.payload)
await self._yield_events([_make_event(result.event)])
async def reply_from_catch_up(self, message, output):
if message.command == TcpCommand.SubscriptionDropped:
await self.drop_subscription(message)
elif message.command == TcpCommand.SubscriptionConfirmation:
confirmation = proto.SubscriptionConfirmation()
confirmation.ParseFromString(message.payload)
self.subscribe_from = confirmation.last_event_number
self._logger.info(
"Subscribed successfully, catching up with missed events from %s",
self.next_event_number,
)
await output.put(self._fetch_page_message(self.next_event_number))
elif message.command == TcpCommand.StreamEventAppeared:
result = proto.StreamEventAppeared()
result.ParseFromString(message.payload)
self.buffer.append(_make_event(result.event))
else:
await ReadStreamEventsBehaviour.reply(self, message, output)
async def reply_from_reconnect(self, message: InboundMessage,
output: Queue):
if message.command != TcpCommand.SubscriptionDropped:
return
self.phase = CatchupSubscriptionPhase.READ_HISTORICAL
await self.start(output)
async def reply(self, message: InboundMessage, output: Queue):
if self.phase == CatchupSubscriptionPhase.READ_HISTORICAL:
self.expect_only(TcpCommand.ReadStreamEventsForwardCompleted,
message)
await ReadStreamEventsBehaviour.reply(self, message, output)
elif self.phase == CatchupSubscriptionPhase.CATCH_UP:
await self.reply_from_catch_up(message, output)
elif self.phase == CatchupSubscriptionPhase.RECONNECT:
await self.reply_from_reconnect(message, output)
else:
await self.reply_from_live(message, output)
async def success(self, result: proto.ReadStreamEventsCompleted,
output: Queue):
finished = False
events = []
for e in result.events:
event = _make_event(e)
events.append(event)
await self._yield_events(events)
self.next_event_number = result.next_event_number
# Todo: we should finish if the next event > subscription_start_pos
if result.is_end_of_stream:
finished = True
if not self.has_first_page:
self.subscription = VolatileSubscription(self.conversation_id,
self.stream, output, 0, 0,
self.iterator)
self.result.set_result(self.subscription)
self.has_first_page = True
if finished:
await self._move_to_next_phase(output)
else:
await output.put(self._fetch_page_message(result.next_event_number)
)
async def _subscribe(self, output: Queue) -> None:
msg = proto.SubscribeToStream()
msg.event_stream_id = self.stream
msg.resolve_link_tos = self.resolve_link_tos
await output.put(
OutboundMessage(
self.conversation_id,
TcpCommand.SubscribeToStream,
msg.SerializeToString(),
self.credential,
))
def mock_client():
client = Mock(TelegramClient, return_value=Future())
return client
class WebSocketAdapterProtocol(asyncio.Protocol):
"""
Adapter class for asyncio-based WebSocket client and server protocols.
"""
def connection_made(self, transport):
self.transport = transport
self.receive_queue = deque()
self._consume()
try:
peer = transport.get_extra_info('peername')
try:
# FIXME: tcp4 vs tcp6
self.peer = "tcp:%s:%d" % (peer[0], peer[1])
except:
# e.g. Unix Domain sockets don't have host/port
self.peer = "unix:{0}".format(peer)
except:
self.peer = "?"
self._connectionMade()
def connection_lost(self, exc):
self._connectionLost(exc)
self.transport = None
def _consume(self):
self.waiter = Future()
def process(_):
while len(self.receive_queue):
data = self.receive_queue.popleft()
if self.transport:
self._dataReceived(data)
else:
print("WebSocketAdapterProtocol._consume: no transport")
self._consume()
self.waiter.add_done_callback(process)
def data_received(self, data):
self.receive_queue.append(data)
if not self.waiter.done():
self.waiter.set_result(None)
# noinspection PyUnusedLocal
def _closeConnection(self, abort=False):
self.transport.close()
def _onOpen(self):
res = self.onOpen()
if yields(res):
asyncio. async (res)
def _onMessageBegin(self, isBinary):
res = self.onMessageBegin(isBinary)
if yields(res):
asyncio. async (res)
def _onMessageFrameBegin(self, length):
res = self.onMessageFrameBegin(length)
if yields(res):
asyncio. async (res)
def _onMessageFrameData(self, payload):
res = self.onMessageFrameData(payload)
if yields(res):
asyncio. async (res)
def _onMessageFrameEnd(self):
res = self.onMessageFrameEnd()
if yields(res):
asyncio. async (res)
def _onMessageFrame(self, payload):
res = self.onMessageFrame(payload)
if yields(res):
asyncio. async (res)
def _onMessageEnd(self):
res = self.onMessageEnd()
if yields(res):
asyncio. async (res)
def _onMessage(self, payload, isBinary):
res = self.onMessage(payload, isBinary)
if yields(res):
asyncio. async (res)
def _onPing(self, payload):
res = self.onPing(payload)
if yields(res):
asyncio. async (res)
def _onPong(self, payload):
res = self.onPong(payload)
if yields(res):
asyncio. async (res)
def _onClose(self, wasClean, code, reason):
res = self.onClose(wasClean, code, reason)
if yields(res):
asyncio. async (res)
def registerProducer(self, producer, streaming):
raise Exception("not implemented")
def _resolved_future(result) -> Future:
future = Future()
future.set_result(result)
return future
def _typing_done_callback(fut: asyncio.Future) -> None:
# just retrieve any exception and call it a day
try:
fut.exception()
except (asyncio.CancelledError, Exception):
pass
async def test_on_reply_to_activity(self):
resource_response_id = "resourceId"
self._conversation_id = await self._test_id_factory.create_skill_conversation_id(
self._conversation_reference)
types_to_test = [
ActivityTypes.end_of_conversation,
ActivityTypes.event,
ActivityTypes.message,
]
for activity_type in types_to_test:
with self.subTest(act_type=activity_type):
mock_adapter = Mock()
mock_adapter.continue_conversation = MagicMock(
return_value=Future())
mock_adapter.continue_conversation.return_value.set_result(
Mock())
mock_adapter.send_activities = MagicMock(return_value=Future())
mock_adapter.send_activities.return_value.set_result(
[ResourceResponse(id=resource_response_id)])
sut = self.create_skill_handler_for_testing(mock_adapter)
activity = Activity(type=activity_type,
attachments=[],
entities=[])
activity_id = str(uuid4())
TurnContext.apply_conversation_reference(
activity, self._conversation_reference)
resource_response = await sut.test_on_reply_to_activity(
self._claims_identity, self._conversation_id, activity_id,
activity)
# continue_conversation validation
(
args_continue,
kwargs_continue,
) = mock_adapter.continue_conversation.call_args_list[0]
mock_adapter.continue_conversation.assert_called_once()
assert isinstance(args_continue[0], ConversationReference)
assert callable(args_continue[1])
assert isinstance(kwargs_continue["claims_identity"],
ClaimsIdentity)
turn_context = TurnContext(
mock_adapter,
conversation_reference_extension.get_continuation_activity(
self._conversation_reference),
)
await args_continue[1](turn_context)
# assert the callback set the right properties.
assert (f"{CallerIdConstants.bot_to_bot_prefix}{self.skill_id}"
), turn_context.activity.caller_id
if activity_type == ActivityTypes.message:
# send_activities validation
(
args_send,
_,
) = mock_adapter.send_activities.call_args_list[0]
activity_from_send = args_send[1][0]
assert activity_from_send.caller_id is None
assert activity_from_send.reply_to_id, activity_id
assert resource_response.id, resource_response_id
else:
# Assert mock SendActivitiesAsync wasn't called
mock_adapter.send_activities.assert_not_called()
def _done(future: asyncio.Future) -> None:
try:
future.result()
except asyncio.CancelledError:
pass
async def set_after(fut: asyncio.Future):
await asyncio.sleep(2)
fut.set_result("end")
def _create_future(self):
return Future()
def __init__(self, *, loop=None):
'''Create an empty outgoing batch request. Members can be
added with add_request and add_notification.'''
# We don't call RPCBatch.__init__()
self.items = []
Future.__init__(self, loop=loop)
async def __anext__(self):
self.feeder()
value = await self.future
self.future = Future()
return value
def __init__(self):
self.notifications = []
self.future = Future()
self.disposable = source.materialize().subscribe(self.on_next)
def _callback(fut: Future, sess: BFDSession, state: int):
logging.warning("Callback with %s, %s" % (sess, SessionState(state)))
if state == SessionState.Up:
fut.set_result((sess, state))
def create_future(result=None):
f = Future()
f.set_result(result)
return f
def reschedule_banner_action(fut: Future) -> None:
banner = fut.result()
assert isinstance(banner, TextBanner), (banner, fut)
bucket.add((banner.interval, banner))
def wrapper(ft: asyncio.Future):
print('call wrapper')
res = fn(*args, **kwargs)
ft.set_result(res)
loop.stop()
class _ChunkedResponse(HTTPResponse):
def __init__(self, status, reason, msg, sock):
HTTPResponse.__init__(self, status, reason, msg)
self.sock = sock
self.chunk_read = None
sock.loop.create_task(self.Chunks())
self.size = None
async def read(self, amt):
if not self.size:
self.next_size = Future(loop=self.sock.loop)
if self.chunk_read:
self.chunk_read.set_result(None)
try:
[self.size, self.chunk_read] = await self.next_size
except EOFError:
return b""
data = await self.sock.recv(min(self.size, amt))
self.size -= len(data)
return data
async def Chunks(self):
"""
Generator that returns chunk length
"""
for _ in range(30000):
parser = Parser(self.sock)
await parser.next_char()
if parser.c == b"\r":
await parser.next_char()
if parser.c == b"\n":
await parser.next_char()
await parser.space()
size = 0
for _ in range(30):
try:
digit = int(parser.c, 16)
except ValueError:
break
size = size * 16 + digit
await parser.next_char()
else:
raise ExcessError("Chunk size of 30 or more digits")
i = 0
while parser.c not in b"\n": # Including EOF
i += 1
if i >= 3000:
raise ExcessError("Line of 3000 or more characters")
await parser.next_char()
if not size:
break
chunk_read = Future(loop=self.sock.loop)
self.next_size.set_result((size, chunk_read))
await chunk_read
else:
raise ExcessError("30000 or more chunks")
self.next_size.set_exception(EOFError())
await parser.headers()
def _update_preview(self, fut: asyncio.Future) -> None:
if fut.cancelled():
return
features = fut.result()
self._show_features(features)
class Bot:
__slots__ = ("logger", "loop", "values", "server", "main_task",
"longpoll_request", "settings", "api", "handler",
"logger_file")
def __init__(self,
settings,
logger=None,
handler=None,
loop=asyncio.get_event_loop()):
self.logger = None
self.init_logger(logger)
self.logger.info("Initializing bot")
self.loop = loop
self.values = {}
self.server = ""
self.longpoll_request = None
self.main_task = None
self.settings = settings
self.logger.info("Initializing vk clients")
self.api = VkController(settings, logger=self.logger)
self.logger.info("Loading plugins")
if handler:
self.handler = handler
else:
self.handler = MessageHandler(self,
self.api,
initiate_plugins=False)
self.handler.initiate_plugins()
signal.signal(signal.SIGINT, lambda x, y: self.stop_bot(True))
self.logger.info("Bot succesfully initialized")
def init_logger(self, logger):
if not logger:
logger = logging.Logger("sketal", level=logging.INFO)
formatter = logging.Formatter(
fmt=u'%(filename)-10s [%(asctime)s] %(levelname)-8s: %(message)s',
datefmt='%y.%m.%d %H:%M:%S')
file_handler = logging.FileHandler('logs.txt')
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(formatter)
self.logger_file = file_handler
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)
self.logger = logger
async def init_long_polling(self, update=0):
result = None
retries = 10
for x in range(retries):
result = await self.api(sender=self.api.target_client
).messages.getLongPollServer(use_ssl=1,
lp_version=2)
if result:
break
time.sleep(0.5)
if not result:
self.logger.error("Unable to connect to VK's long polling server")
exit()
last_ts = 0
longpoll_key = ""
if 'ts' in self.values:
last_ts = self.values['ts']
if 'key' in self.values:
longpoll_key = self.values['key']
if update == 0:
self.server = "https://" + result['server']
longpoll_key = result['key']
last_ts = result['ts']
elif update == 3:
longpoll_key = result['key']
last_ts = result['ts']
elif update == 2:
longpoll_key = result['key']
self.values = {
'act': 'a_check',
'key': longpoll_key,
'ts': last_ts,
'wait': 20,
'mode': 10,
'version': 2
}
async def process_longpoll_event(self, new_event):
if not new_event:
return
event_id = new_event[0]
if event_id != 4:
evnt = LongpollEvent(self.api, event_id, new_event)
return await self.process_event(evnt)
data = MessageEventData()
data.msg_id = new_event[1]
data.attaches = new_event[6]
data.time = int(new_event[4])
try:
data.user_id = int(data.attaches['from'])
data.chat_id = int(new_event[3]) - 2000000000
data.is_multichat = True
del data.attaches['from']
except KeyError:
data.user_id = int(new_event[3])
data.is_multichat = False
# https://vk.com/dev/using_longpoll_2
flags = parse_msg_flags(new_event[2])
if flags['outbox']:
if not self.settings.READ_OUT:
return
data.is_out = True
data.full_text = new_event[5].replace('<br>', '\n')
if "fwd" in data.attaches:
data.forwarded = MessageEventData.parse_brief_forwarded_messages_from_lp(
data.attaches["fwd"])
del data.attaches["fwd"]
else:
data.forwarded = []
msg = Message(self.api, data)
if await self.check_event(data.user_id, data.chat_id, data.attaches):
msg.is_event = True
await self.process_message(msg)
async def longpoll_processor(self):
await self.init_long_polling()
session = aiohttp.ClientSession(loop=self.loop)
while True:
try:
self.longpoll_request = session.get(self.server,
params=self.values)
resp = await self.longpoll_request
except aiohttp.ClientOSError:
session = aiohttp.ClientSession(loop=self.loop)
except (asyncio.TimeoutError, aiohttp.ServerDisconnectedError):
self.logger.warning(
"Long polling server doesn't respond. Changing server")
await self.init_long_polling()
continue
try:
events = json.loads(await resp.text())
except ValueError:
continue
failed = events.get('failed')
if failed:
err_num = int(failed)
if err_num == 1: # 1 - update timestamp
self.values['ts'] = events['ts']
elif err_num in (2, 3): # 2, 3 - new data for long polling
await self.init_long_polling(err_num)
continue
self.values['ts'] = events['ts']
for event in events['updates']:
asyncio.ensure_future(self.process_longpoll_event(event))
async def callback_processor(self, request):
try:
data = await request.json()
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
return web.Response(text="ok")
data_type = data["type"]
if data_type == "confirmation":
return web.Response(text=self.settings.CONF_CODE)
obj = data["object"]
if "user_id" in obj:
obj['user_id'] = int(obj['user_id'])
if data_type == 'message_new':
data = MessageEventData.from_message_body(obj)
msg = Message(self.api, data)
await self.process_message(msg)
else:
evnt = CallbackEvent(self.api, data_type, obj)
await self.process_event(evnt)
return web.Response(text="ok")
def longpoll_run(self, custom_process=False):
self.main_task = Task(self.longpoll_processor())
if custom_process:
return self.main_task
self.logger.info("Started to process messages")
try:
self.loop.run_until_complete(self.main_task)
except (KeyboardInterrupt, SystemExit):
self.stop()
self.logger.info("Stopped to process messages")
except asyncio.CancelledError:
pass
def callback_run(self, custom_process=False):
host = getenv('IP', '0.0.0.0')
port = int(getenv('PORT', 8000))
self.logger.info("Started to process messages")
try:
server_generator, handler, app = self.loop.run_until_complete(
self.init_app(host, port, self.loop))
server = self.loop.run_until_complete(server_generator)
except OSError:
self.logger.error("Address already in use: " + str(host) + ":" +
str(port))
return
self.main_task = Future()
if custom_process:
return self.main_task
print("======== Running on http://{}:{} ========\n"
" (Press CTRL+C to quit)".format(
*server.sockets[0].getsockname()))
def stop_server():
server.close()
if not self.loop.is_running():
return
self.loop.run_until_complete(server.wait_closed())
self.loop.run_until_complete(app.shutdown())
self.loop.run_until_complete(handler.shutdown(10))
self.loop.run_until_complete(app.cleanup())
try:
self.loop.run_until_complete(self.main_task)
except KeyboardInterrupt:
self.stop()
stop_server()
self.loop.close()
except asyncio.CancelledError:
pass
finally:
stop_server()
self.logger.info("Stopped to process messages")
async def init_app(self, host, port, loop):
app = web.Application()
app.router.add_post('/', self.callback_processor)
handler = app.make_handler()
server_generator = loop.create_server(handler, host, port)
return server_generator, handler, app
def stop_bot(self, full=False):
try:
self.main_task.cancel()
except:
pass
if full:
self.stop()
self.loop.stop()
self.logger.info("Attempting to turn bot off")
async def process_message(self, msg):
asyncio.ensure_future(self.handler.process(msg), loop=self.loop)
async def check_event(self, user_id, chat_id, attaches):
if chat_id != 0 and "source_act" in attaches:
photo = attaches.get("attach1_type") + attaches.get(
"attach1") if "attach1" in attaches else None
evnt = ChatChangeEvent(self.api, user_id, chat_id,
attaches.get("source_act"),
int(attaches.get("source_mid", 0)),
attaches.get("source_text"),
attaches.get("source_old_text"), photo,
int(attaches.get("from", 0)))
await self.process_event(evnt)
return True
return False
async def process_event(self, evnt):
asyncio.ensure_future(self.handler.process_event(evnt), loop=self.loop)
def do(self, coroutine):
if asyncio.iscoroutine(coroutine):
return self.loop.run_until_complete(coroutine)
return False
@staticmethod
def silent(func):
try:
func()
except:
pass
def stop(self):
self.handler.stop()
self.api.stop()
self.silent(self.main_task.cancel)
self.logger.removeHandler(self.logger_file)
self.logger_file.close()
def _create_future():
return Future()
def __init__(self):
self.future = Future()
def mock_remove(infohash, remove_content=False):
nonlocal remove_list
d = Future()
d.set_result(None)
remove_list.append((infohash, remove_content))
return d
def reset(self):
self.future = Future()
def inner(writer_future: asyncio.Future):
results[num] = writer_future.result()
def _on_published(self, fut: asyncio.Future,
message: FutureMessage) -> None:
res: RecordMetadata = fut.result()
message.set_result(res)
if message.message.callback:
message.message.callback(message)