# TODO message consumer
class Message:
def __init__(self, msgid):
self.msgid = msgid
def extend_deadline(self, seconds):
print(f'[{self.msgid}] message deadline extended by {seconds} seconds')
def ack(self):
print(f'[{self.msgid}] messaged acked')
subscriber = asyncio.Queue()
async def watch_update(message):
iters = 0
while True:
if iters == 3:
print('updated!')
return
iters += 1
print('not finished updating')
await asyncio.sleep(1)
async def start_update(message):
await asyncio.sleep(1) # an http call
async def get(self, request):
"""Provide a streaming interface for the event bus."""
if not request['hass_user'].is_admin:
raise Unauthorized()
hass = request.app['hass']
stop_obj = object()
to_write = asyncio.Queue(loop=hass.loop)
restrict = request.query.get('restrict')
if restrict:
restrict = restrict.split(',') + [EVENT_HOMEASSISTANT_STOP]
async def forward_events(event):
"""Forward events to the open request."""
if event.event_type == EVENT_TIME_CHANGED:
return
if restrict and event.event_type not in restrict:
return
_LOGGER.debug("STREAM %s FORWARDING %s", id(stop_obj), event)
if event.event_type == EVENT_HOMEASSISTANT_STOP:
data = stop_obj
else:
data = json.dumps(event, cls=JSONEncoder)
await to_write.put(data)
response = web.StreamResponse()
response.content_type = 'text/event-stream'
await response.prepare(request)
unsub_stream = hass.bus.async_listen(MATCH_ALL, forward_events)
try:
_LOGGER.debug("STREAM %s ATTACHED", id(stop_obj))
# Fire off one message so browsers fire open event right away
await to_write.put(STREAM_PING_PAYLOAD)
while True:
try:
with async_timeout.timeout(STREAM_PING_INTERVAL,
loop=hass.loop):
payload = await to_write.get()
if payload is stop_obj:
break
msg = "data: {}\n\n".format(payload)
_LOGGER.debug("STREAM %s WRITING %s", id(stop_obj),
msg.strip())
await response.write(msg.encode('UTF-8'))
except asyncio.TimeoutError:
await to_write.put(STREAM_PING_PAYLOAD)
except asyncio.CancelledError:
_LOGGER.debug("STREAM %s ABORT", id(stop_obj))
finally:
_LOGGER.debug("STREAM %s RESPONSE CLOSED", id(stop_obj))
unsub_stream()
return response
async def main(uid,chatauth):
#debug: ws://direct.smilebasicsource.com:45697/chatserver
#main: ws://direct.smilebasicsource.com:45695/chatserver
uri = "ws://direct.smilebasicsource.com:45697/chatserver"
async with websockets.connect(uri, ping_interval=None) as server:
# bind
message = json.dumps({"type":"bind","lessData":True,"uid":uid,"key":chatauth})
await server.send(message)
print(f"us:{message}")
response = await server.recv()
print(f"server:{response}")
# communicate with server
queuelist = [] # hold queues for sending back to client
async def listenserver(): # send messages to clients
print(f"listening on {uri}")
while True:
try:
received = await server.recv()
print(received)
for queue in queuelist:
await queue.put(received)
except:
break
messages = asyncio.Queue() # to get messages from clients
async def messageserver(): # forward messages to server
while True:
try:
message = await messages.get()
await server.send(message)
except:
break
async def handler(websocket, path): # BE the server
response = asyncio.Queue()
queuelist.append(response)
print("connected to client")
message = await websocket.recv()
bind = json.loads(message)
if bind["key"] == chatauth:
async def listenclient(): # send requests to server
await messages.put('{"type":"request","request":"userList"}')
await messages.put('{"type":"request","request":"messageList"}')
while True:
try:
received = await websocket.recv()
print(received)
await messages.put(received)
except:
break
async def sendclient(): # forward messages to client
while True:
try:
message = await response.get()
await websocket.send(message)
except:
break
await asyncio.gather(listenclient(),sendclient(),)
queuelist.remove(response) # GOSH i hope this works, otherwise IM F****D>EDIT: it worked
print("disconnected from client")
await asyncio.gather(listenserver(), messageserver(), websockets.serve(handler,"localhost",8765))
#!/usr/bin/env python
import asyncio
import time
from aiohttp import ClientSession
url = 'https://note.generals.space/aio'
task_queue = asyncio.Queue()
## 为了能明显的看出semaphore的效果, 这里设置的小一些.
sem = asyncio.Semaphore(5)
async def fetch_url(session, order, url):
async with sem:
print('fetching...', order)
res = await session.get(url)
resText = await res.json()
print('order %d: %s' % (order, resText))
return resText
async def customer(loop):
print('customer start...')
aioSession = ClientSession(loop=loop)
## 在这里加semaphore锁是无效的...为啥?
## sem = asyncio.Semaphore(5)
## while True:
## async with sem:
## _order, _url = await task_queue.get()
## asyncio.run_coroutine_threadsafe(fetch_url(aioSession, _order, _url), loop)
async def run(n):
queue = asyncio.Queue()
consumer = asyncio.ensure_future(consume(queue))
await produce(queue, n)
await queue.join()
consumer.cancel()
def __init__(self, queue_size=None):
if queue_size is None:
queue_size = 0
self._queue = asyncio.Queue(queue_size)
self._closed = False
self._transport = None
from urllib.parse import urlparse
import discord
import asyncio
from discord.ext import commands, tasks
import scraper
from utils import settings, log
client = commands.Bot(command_prefix='>')
messages_queue = asyncio.Queue()
scrap = scraper.Scraper()
@client.event
async def on_ready():
guild = client.get_guild(settings.GUILD_ID)
if guild:
log.info(f'{client.user} is connected to {guild.name}, {guild.id}')
else:
raise Exception("Desired guild not connected, turning off.")
await client.change_presence(activity=discord.Game(name=settings.STATUS))
get_messages.start()
post_messages.start()
log.info("Bot started")
@client.command(name='off', help='Turns the bot off')
@commands.has_role('Admin')
async def turn_off(context):
log.info("Turning off.")
await context.send("Turning off.")
def __init__(self):
self.event_received = False
self.queue = asyncio.Queue()
def __init__(self):
self.queue = asyncio.Queue()
self.count = 0
def __init__(self, connection, channels, get=False):
self.connection = connection
self.channels = channels
self.get = get
self.queue = asyncio.Queue()
import asyncio
import random
async def add(store, name):
""" input queue """
for i in range(5):
num = '{0} - {1}'.format(name, i)
await asyncio.sleep(random.randint(1, 3))
await store.put(i)
print('{0} add --- {1}, seize: {2}'.format(name, num, store.qsize()))
async def reduces(store):
""" dequeue """
for i in range(10):
rest = await store.get()
print('reduce --- {0}, size: {1}'.format(rest, store.qsize()))
if __name__ == "__main__":
store = asyncio.Queue(maxsize=5)
a1 = add(store, 'a1')
a2 = add(store, 'a2')
r1 = reduces(store)
# add into event queue
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(a1, a2, r1))
loop.close()
def __init__(self, max_bucket, period):
self.queue = asyncio.Queue()
super().__init__(max_bucket, period)
def create_listener(self):
q = asyncio.Queue()
self._mic.add(q)
ln = helper.Listener(self._mic, q)
return ln
def start(self):
super().start()
self._queue = asyncio.Queue()
self._ctrl_task = self._create_monitored_task(self._ctrl_coro())
async def test_service_run(hass, caplog):
"""Test running a service with keyword arguments."""
notify_q = asyncio.Queue(0)
await setup_script(
hass,
notify_q,
dt(2020, 7, 1, 11, 59, 59, 999999),
"""
@service
def func1(arg1=1, arg2=2, context=None):
x = 1
x = 2 * x + 3
log.info(f"this is func1 x = {x}, arg1 = {arg1}, arg2 = {arg2}")
pyscript.done = [x, arg1, arg2, str(context)]
# registering twice will cause it to be called twice
@service("other.func2")
@service("other.func2")
@service
def func2(**kwargs):
x = 1
x = 2 * x + 3
log.info(f"this is func1 x = {x}, kwargs = {kwargs}")
has2 = service.has_service("pyscript", "func2")
has3 = service.has_service("pyscript", "func3")
del kwargs["context"]
pyscript.done = [x, kwargs, has2, has3]
@service
def call_service(domain=None, name=None, **kwargs):
if domain == "pyscript" and name == "func1":
task.sleep(0)
pyscript.func1(**kwargs)
else:
service.call(domain, name, **kwargs)
""",
)
context = Context(user_id="1234", parent_id="5678", id="8901")
await hass.services.async_call("pyscript", "func1", {}, context=context)
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [5, 1, 2, str(context)]
assert "this is func1 x = 5" in caplog.text
await hass.services.async_call("pyscript",
"func1", {"arg1": 10},
context=context)
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [5, 10, 2, str(context)]
await hass.services.async_call(
"pyscript",
"call_service",
{
"domain": "pyscript",
"name": "func1",
"arg1": "string1"
},
context=context,
)
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [5, "string1", 2, str(context)]
await hass.services.async_call("pyscript",
"func1", {
"arg1": "string1",
"arg2": 123
},
context=context)
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [5, "string1", 123, str(context)]
await hass.services.async_call("pyscript", "call_service", {
"domain": "pyscript",
"name": "func2"
})
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [5, {"trigger_type": "service"}, 1, 0]
await hass.services.async_call(
"pyscript",
"call_service",
{
"domain": "pyscript",
"name": "func2",
"arg1": "string1"
},
)
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [
5, {
"trigger_type": "service",
"arg1": "string1"
}, 1, 0
]
await hass.services.async_call("pyscript", "func2", {
"arg1": "string1",
"arg2": 456
})
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [
5, {
"trigger_type": "service",
"arg1": "string1",
"arg2": 456
}, 1, 0
]
await hass.services.async_call("other", "func2", {
"arg1": "string1",
"arg2": 123
})
ret = await wait_until_done(notify_q)
assert literal_eval(ret) == [
5, {
"trigger_type": "service",
"arg1": "string1",
"arg2": 123
}, 1, 0
]
assert literal_eval(ret) == [
5, {
"trigger_type": "service",
"arg1": "string1",
"arg2": 123
}, 1, 0
]
"""Load saved devices to provide quick startup."""
import asyncio
from pyinsteon import async_close, async_connect
from pyinsteon.managers.link_manager import async_enter_linking_mode
from samples import _LOGGER, PATH, get_hub_config, set_log_levels
# DEVICE = '/dev/ttyS5'
DEVICE = "COM5"
USERNAME, PASSWORD, HOST = get_hub_config()
done = asyncio.Queue()
async def async_setup_device(address):
"""Set up device."""
from pyinsteon import devices
device = devices[address]
await device.aldb.async_load(refresh=True)
await device.async_read_op_flags()
await device.async_read_ext_properties()
max_wait = 300
wait = 0
sleep_for = 5
while not device.aldb.is_loaded and wait < max_wait:
await asyncio.sleep(sleep_for)
wait += sleep_for
await device.async_add_default_links()
await asyncio.sleep(sleep_for)
await devices.async_save(workdir=PATH)
async def test_reload(hass, caplog):
"""Test reload."""
notify_q = asyncio.Queue(0)
now = dt(2020, 7, 1, 11, 59, 59, 999999)
source0 = """
seq_num = 0
@time_trigger
def func_startup_sync():
global seq_num
seq_num += 1
log.info(f"func_startup_sync setting pyscript.done = {seq_num}")
pyscript.done = seq_num
@service
@state_trigger("pyscript.f1var1 == '1'")
def func9(var_name=None, value=None):
global seq_num
seq_num += 1
log.info(f"func9 var = {var_name}, value = {value}")
pyscript.done = [seq_num, var_name, int(value)]
"""
source1 = """
seq_num = 10
@time_trigger("startup")
def func_startup_sync():
global seq_num
seq_num += 1
log.info(f"func_startup_sync setting pyscript.done = {seq_num}")
pyscript.done = seq_num
@service
@state_trigger("pyscript.f5var1 == '1'")
def func5(var_name=None, value=None):
global seq_num
seq_num += 1
log.info(f"func5 var = {var_name}, value = {value}")
pyscript.done = [seq_num, var_name, int(value)]
"""
await setup_script(hass, notify_q, now, source0)
#
# run and reload 6 times with different source files to make sure seqNum
# gets reset, autostart of func_startup_sync happens and triggers work each time
#
# first time: fire event to startup triggers and run func_startup_sync
#
hass.bus.async_fire(EVENT_HOMEASSISTANT_STARTED)
for i in range(6):
if i & 1:
seq_num = 10
assert not hass.services.has_service("pyscript", "func9")
assert hass.services.has_service("pyscript", "reload")
assert hass.services.has_service("pyscript", "func5")
seq_num += 1
assert literal_eval(await wait_until_done(notify_q)) == seq_num
seq_num += 1
# initialize the trigger and active variables
hass.states.async_set("pyscript.f5var1", 0)
# try some values that shouldn't work, then one that does
hass.states.async_set("pyscript.f5var1", "string")
hass.states.async_set("pyscript.f5var1", 1)
assert literal_eval(await wait_until_done(notify_q)) == [
seq_num,
"pyscript.f5var1",
1,
]
assert "func5 var = pyscript.f5var1, value = 1" in caplog.text
next_source = source0
else:
seq_num = 0
assert hass.services.has_service("pyscript", "func9")
assert hass.services.has_service("pyscript", "reload")
assert not hass.services.has_service("pyscript", "func5")
seq_num += 1
assert literal_eval(await wait_until_done(notify_q)) == seq_num
seq_num += 1
# initialize the trigger and active variables
hass.states.async_set("pyscript.f1var1", 0)
# try some values that shouldn't work, then one that does
hass.states.async_set("pyscript.f1var1", "string")
hass.states.async_set("pyscript.f1var1", 1)
assert literal_eval(await wait_until_done(notify_q)) == [
seq_num,
"pyscript.f1var1",
1,
]
assert "func9 var = pyscript.f1var1, value = 1" in caplog.text
next_source = source1
#
# now reload the other source file
#
scripts = [
"/hello.py",
]
with patch(
"custom_components.pyscript.os.path.isdir", return_value=True
), patch(
"custom_components.pyscript.glob.iglob", return_value=scripts
), patch("custom_components.pyscript.global_ctx.open",
mock_open(read_data=next_source)), patch(
"custom_components.pyscript.open",
mock_open(read_data=next_source)
), patch(
"custom_components.pyscript.trigger.dt_now",
return_value=now
), patch(
"homeassistant.config.load_yaml_config_file",
return_value={}
), patch(
"custom_components.pyscript.os.path.getmtime",
return_value=1000
), patch(
"custom_components.pyscript.global_ctx.os.path.getmtime",
return_value=1000), patch(
"custom_components.pyscript.install_requirements",
return_value=None,
):
reload_param = {}
if i % 2 == 1:
#
# on alternate times, just reload the specific file we are testing with
#
reload_param = {"global_ctx": "file.hello"}
await hass.services.async_call("pyscript",
"reload",
reload_param,
blocking=True)
if i % 3 == 0:
#
# reload a file that doesn't exist; will log error and do nothing
#
await hass.services.async_call(
"pyscript",
"reload", {"global_ctx": "file.nosuchfile"},
blocking=True)
assert "pyscript.reload: no global context 'file.nosuchfile' to reload" in caplog.text
async def run(self):
try:
adinfo = self.session.query(ADInfo).get(self.ad_id)
self.domain_name = str(adinfo.distinguishedName).replace(
',', '.').replace('DC=', '')
qs = self.agent_cnt
self.agent_in_q = asyncio.Queue(qs) #AsyncProcessQueue()
self.agent_out_q = asyncio.Queue(qs) #AsyncProcessQueue(1000)
self.sd_file_path = 'sd_' + datetime.datetime.utcnow().strftime(
"%Y%m%d_%H%M%S") + '.gzip'
self.sd_file = gzip.GzipFile(self.sd_file_path, 'w')
logger.debug('Polling sds')
_, res = await self.prepare_targets()
if res is not None:
raise res
for _ in range(self.agent_cnt):
agent = LDAPGathererAgent(self.ldap_mgr, self.agent_in_q,
self.agent_out_q)
self.agents.append(asyncio.create_task(agent.arun()))
asyncio.create_task(self.start_jobs())
if self.show_progress is True:
self.sds_progress = tqdm(desc='Collecting SDs',
total=self.total_targets,
position=0,
leave=True)
if self.progress_queue is not None:
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.STARTED
msg.adid = self.ad_id
msg.domain_name = self.domain_name
await self.progress_queue.put(msg)
acnt = self.total_targets
last_stat_cnt = 0
while acnt > 0:
try:
res = await self.agent_out_q.get()
res_type, res = res
if res_type == LDAPAgentCommand.SD:
await self.store_sd(res)
if self.show_progress is True:
self.sds_progress.update()
if self.progress_queue is not None:
if acnt % self.progress_step_size == 0:
last_stat_cnt += self.progress_step_size
now = datetime.datetime.utcnow()
td = (now - self.progress_last_updated
).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_targets
msg.total_finished = self.total_targets - acnt
if td > 0:
msg.speed = str(self.progress_step_size //
td)
msg.step_size = self.progress_step_size
await self.progress_queue.put(msg)
elif res_type == LDAPAgentCommand.EXCEPTION:
logger.warning(str(res))
acnt -= 1
except Exception as e:
logger.exception('SDs enumeration error!')
raise e
if self.progress_queue is not None:
now = datetime.datetime.utcnow()
td = (now - self.progress_last_updated).total_seconds()
self.progress_last_updated = now
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.PROGRESS
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.total = self.total_targets
msg.total_finished = self.total_targets
if td > 0:
msg.speed = str((self.total_targets - last_stat_cnt) // td)
msg.step_size = self.total_targets - last_stat_cnt
await self.progress_queue.put(msg)
adinfo = self.session.query(ADInfo).get(self.ad_id)
adinfo.ldap_sds_finished = True
self.session.commit()
return True, None
except Exception as e:
logger.exception('SDs enumeration main error')
if self.progress_queue is not None:
msg = GathererProgress()
msg.type = GathererProgressType.SD
msg.msg_type = MSGTYPE.ERROR
msg.adid = self.ad_id
msg.domain_name = self.domain_name
msg.error = e
await self.progress_queue.put(msg)
return False, e
finally:
await self.stop_sds_collection()
try:
self.session.close()
except:
pass
def __init__(self, pool: dbver.Pool[apsw.Connection]) -> None:
self._pool = pool
self._maybejobs: asyncio.Queue[Awaitable[Optional[Job]]] = asyncio.Queue()
self._task: Optional[asyncio.Task] = None
self._closed = False
self._executor: Optional[concurrent.futures.ThreadPoolExecutor] = None
async def console(self):
self.send_buffer = asyncio.Queue(loop=asyncio.get_running_loop())
asyncio.ensure_future(self.display_incomming_data())
asyncio.ensure_future(self.send_outgoing_data())
await self.execute_hooks()
for i in range(cpu_count()):
blocking_tasks.append(
asyncio.ensure_future(
loop.run_in_executor(executor, resize_img, in_queue,
out_queue)))
completed, pending = await asyncio.wait(blocking_tasks)
results = [t.result() for t in completed]
print(results)
sys.stdout.flush()
await out_queue.coro_put(None)
return
if __name__ == "__main__":
print('starting up...')
queue_urls = asyncio.Queue(50)
queue_images = AsyncProcessQueue(50)
queue_thumbnails = AsyncProcessQueue(50)
loop = asyncio.get_event_loop()
loop.set_debug(True)
loop.create_task(retrieve_url(queue_urls))
loop.create_task(get_image(queue_urls, queue_images))
loop.create_task(process_data(queue_images, queue_thumbnails, loop))
loop.create_task(publish_data(queue_thumbnails))
pending = asyncio.Task.all_tasks()
print('firing up loop...')
loop.run_until_complete(asyncio.gather(*pending))
def __init__(self,
quality,
fname,
max_tasks=6,
chunk_size=524288,
sess_data=None,
timeout=10):
'''
Initialization. If you use the _BaseDownloader directly, it raises NotImplementedError.
:param aid: Av index.
:param quality: Quility index in [112, 74, 80, 64, 32, 16].
:param fname: File to save.
:param max_tasks: Number of corotines.
:param chunk_size: Bytes number per download (524288=0.5M).
:sess_data: SESSDATA in cookies.
:timeout: Timeout of downloads.
'''
# self.aid = aid
self.quality = quality
self.fname = fname
self.max_tasks = max_tasks
self.chunk_size = chunk_size
if os.path.exists(fname):
logging.warning(
'{} already exists, the file will be deleted.'.format(fname))
os.remove(fname)
self.queue = asyncio.Queue()
if sess_data is None:
cookies = None
else:
cookies = {'SESSDATA': sess_data}
self.session = aiohttp.ClientSession(
timeout=aiohttp.ClientTimeout(total=timeout), cookies=cookies)
self.headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:56.0) Gecko/20100101 Firefox/56.0',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'identity',
'Range': 'bytes=0-1',
'Origin': 'https://www.bilibili.com',
'Connection': 'keep-alive'
}
self.cid = ''
self.title = ''
self.subtitle = ''
self.blocks = []
self.content = b''
self._ordered_chunks = deque()
self._timestamp = 0
self._download_start_time = 0
self._current_size = 0
self._size = 0
self._nexts = deque()
self._fileobj = None
raise NotImplementedError('Don\'t use _BaseDownloader directly!')
def __init__(self: 'TelegramClient',
session: 'typing.Union[str, Session]',
api_id: int,
api_hash: str,
*,
connection: 'typing.Type[Connection]' = ConnectionTcpFull,
use_ipv6: bool = False,
proxy: typing.Union[tuple, dict] = None,
timeout: int = 10,
request_retries: int = 5,
connection_retries: int = 5,
retry_delay: int = 1,
auto_reconnect: bool = True,
sequential_updates: bool = False,
flood_sleep_threshold: int = 60,
device_model: str = None,
system_version: str = None,
app_version: str = None,
lang_code: str = 'en',
system_lang_code: str = 'en',
loop: asyncio.AbstractEventLoop = None,
base_logger: typing.Union[str, logging.Logger] = None):
if not api_id or not api_hash:
raise ValueError("Your API ID or Hash cannot be empty or None. "
"Refer to telethon.rtfd.io for more information.")
self._use_ipv6 = use_ipv6
self._loop = loop or asyncio.get_event_loop()
if isinstance(base_logger, str):
base_logger = logging.getLogger(base_logger)
elif not isinstance(base_logger, logging.Logger):
base_logger = __default_log__
class _Loggers(dict):
def __missing__(self, key):
if key.startswith("telethon."):
key = key.split('.', maxsplit=1)[1]
return base_logger.getChild(key)
self._log = _Loggers()
# Determine what session object we have
if isinstance(session, str) or session is None:
try:
session = SQLiteSession(session)
except ImportError:
import warnings
warnings.warn('The sqlite3 module is not available under this '
'Python installation and no custom session '
'instance was given; using MemorySession.\n'
'You will need to re-login every time unless '
'you use another session storage')
session = MemorySession()
elif not isinstance(session, Session):
raise TypeError(
'The given session must be a str or a Session instance.')
# ':' in session.server_address is True if it's an IPv6 address
if (not session.server_address
or (':' in session.server_address) != use_ipv6):
session.set_dc(
DEFAULT_DC_ID,
DEFAULT_IPV6_IP if self._use_ipv6 else DEFAULT_IPV4_IP,
DEFAULT_PORT)
self.flood_sleep_threshold = flood_sleep_threshold
# TODO Use AsyncClassWrapper(session)
# ChatGetter and SenderGetter can use the in-memory _entity_cache
# to avoid network access and the need for await in session files.
#
# The session files only wants the entities to persist
# them to disk, and to save additional useful information.
# TODO Session should probably return all cached
# info of entities, not just the input versions
self.session = session
self._entity_cache = EntityCache()
self.api_id = int(api_id)
self.api_hash = api_hash
self._request_retries = request_retries
self._connection_retries = connection_retries
self._retry_delay = retry_delay or 0
self._proxy = proxy
self._timeout = timeout
self._auto_reconnect = auto_reconnect
assert isinstance(connection, type)
self._connection = connection
init_proxy = None if not issubclass(connection, TcpMTProxy) else \
types.InputClientProxy(*connection.address_info(proxy))
# Used on connection. Capture the variables in a lambda since
# exporting clients need to create this InvokeWithLayerRequest.
system = platform.uname()
self._init_with = lambda x: functions.InvokeWithLayerRequest(
LAYER,
functions.InitConnectionRequest(
api_id=self.api_id,
device_model=device_model or system.system or 'Unknown',
system_version=system_version or system.release or '1.0',
app_version=app_version or self.__version__,
lang_code=lang_code,
system_lang_code=system_lang_code,
lang_pack='', # "langPacks are for official apps only"
query=x,
proxy=init_proxy))
self._sender = MTProtoSender(
self.session.auth_key,
self._loop,
loggers=self._log,
retries=self._connection_retries,
delay=self._retry_delay,
auto_reconnect=self._auto_reconnect,
connect_timeout=self._timeout,
auth_key_callback=self._auth_key_callback,
update_callback=self._handle_update,
auto_reconnect_callback=self._handle_auto_reconnect)
# Remember flood-waited requests to avoid making them again
self._flood_waited_requests = {}
# Cache ``{dc_id: (n, MTProtoSender)}`` for all borrowed senders,
# being ``n`` the amount of borrows a given sender has; once ``n``
# reaches ``0`` it should be disconnected and removed.
self._borrowed_senders = {}
self._borrow_sender_lock = asyncio.Lock(loop=self._loop)
self._updates_handle = None
self._last_request = time.time()
self._channel_pts = {}
if sequential_updates:
self._updates_queue = asyncio.Queue(loop=self._loop)
self._dispatching_updates_queue = asyncio.Event(loop=self._loop)
else:
# Use a set of pending instead of a queue so we can properly
# terminate all pending updates on disconnect.
self._updates_queue = set()
self._dispatching_updates_queue = None
self._authorized = None # None = unknown, False = no, True = yes
# Update state (for catching up after a disconnection)
# TODO Get state from channels too
self._state_cache = StateCache(self.session.get_update_state(0),
self._log)
# Some further state for subclasses
self._event_builders = []
# {chat_id: {Conversation}}
self._conversations = collections.defaultdict(set)
# Default parse mode
self._parse_mode = markdown
# Some fields to easy signing in. Let {phone: hash} be
# a dictionary because the user may change their mind.
self._phone_code_hash = {}
self._phone = None
self._tos = None
# Sometimes we need to know who we are, cache the self peer
self._self_input_peer = None
self._bot = None
def __init__(self, max_batch_size: int, timeout_s: float) -> None:
self.queue = asyncio.Queue()
self.full_batch_event = asyncio.Event()
self.max_batch_size = max_batch_size
self.timeout_s = timeout_s
async def test_skipped_with_no_handlers(registry, settings, selector, resource,
cause_mock, cause_type, caplog,
assert_logs, k8s_mocked):
caplog.set_level(logging.DEBUG)
event_type = None
event_body = {'metadata': {'finalizers': []}}
cause_mock.reason = cause_type
assert not registry._changing.has_handlers(
resource=resource) # prerequisite
registry._changing.append(
ChangingHandler(
reason='a-non-existent-cause-type',
fn=lambda **_: None,
id='id',
param=None,
errors=None,
timeout=None,
retries=None,
backoff=None,
selector=selector,
annotations=None,
labels=None,
when=None,
field=None,
value=None,
old=None,
new=None,
field_needs_change=None,
deleted=None,
initial=None,
requires_finalizer=None,
))
await process_resource_event(
lifecycle=kopf.lifecycles.all_at_once,
registry=registry,
settings=settings,
resource=resource,
indexers=OperatorIndexers(),
memories=ResourceMemories(),
memobase=Memo(),
raw_event={
'type': event_type,
'object': event_body
},
event_queue=asyncio.Queue(),
)
assert not k8s_mocked.sleep.called
assert k8s_mocked.patch.called
# The patch must contain ONLY the last-seen update, and nothing else.
patch = k8s_mocked.patch.call_args_list[0][1]['payload']
assert set(patch.keys()) == {'metadata'}
assert set(patch['metadata'].keys()) == {'annotations'}
assert set(
patch['metadata']['annotations'].keys()) == {LAST_SEEN_ANNOTATION}
assert_logs([
"(Creation|Updating|Resuming|Deletion) is in progress:",
"Patching with:",
],
prohibited=[
"(Creation|Updating|Resuming|Deletion) is processed:",
])
def __init__(self):
self.qq = asyncio.Queue()
users = [
User(i, user_info, dict_bili, task_control, True)
for i, user_info in enumerate(dict_user['users'])
]
danmu_connection = connect.connect(
dict_user['other_control']['default_monitor_roomid'])
list_raffle_connection = [connect.RaffleConnect(i) for i in range(1, 5)]
list_raffle_connection_task = [i.run() for i in list_raffle_connection]
var_super_user = SuperUser()
raffle = RaffleHandler(users, var_super_user)
delayraffle = DelayRaffleHandler(users, var_super_user)
normal_task = Task(users, var_super_user)
queue = asyncio.Queue()
bili_console.Biliconsole(loop, queue, users)
console_thread = threading.Thread(target=bili_console.controler)
console_thread.start()
tasks = [
raffle.join_raffle(),
delayraffle.join_raffle(),
normal_task.heartbeat(),
danmu_connection.run(),
normal_task.run(),
bili_console.Biliconsole().run()
]
loop.run_until_complete(asyncio.wait(tasks + list_raffle_connection_task))
console_thread.join()
from router.simple_router import SimpleRouter as Router
from model import Agent
import modules.connection as connection
import modules.init as init
import serializer.json_serializer as Serializer
import view.site_handlers as site_handlers
if len(sys.argv) == 2 and str.isdigit(sys.argv[1]):
PORT = int(sys.argv[1])
else:
PORT = 8080
LOOP = asyncio.get_event_loop()
AGENT = web.Application(middlewares=[IndexMiddleware()])
AGENT['msg_router'] = Router()
AGENT['msg_receiver'] = Receiver(asyncio.Queue())
AGENT['agent'] = Agent()
# template engine setup
aiohttp_jinja2.setup(
AGENT,
loader=jinja2.FileSystemLoader(
os.path.realpath('view/templates/')
)
)
ROUTES = [
web.get('/', site_handlers.index),
web.post('/indy', AGENT['msg_receiver'].handle_message),
web.post('/indy/request', connection.send_request),
#web.get('/indy/connections', site_handlers.connections),
async def round_socket_handler(request):
users = request.app["users"]
sockets = request.app["lobbysockets"]
seeks = request.app["seeks"]
games = request.app["games"]
db = request.app["db"]
ws = WebSocketResponse(heartbeat=3.0, receive_timeout=10.0)
ws_ready = ws.can_prepare(request)
if not ws_ready.ok:
raise web.HTTPFound("/")
await ws.prepare(request)
session = await aiohttp_session.get_session(request)
session_user = session.get("user_name")
user = users[session_user] if session_user is not None and session_user in users else None
game = None
opp_ws = None
log.debug("-------------------------- NEW round WEBSOCKET by %s" % user)
try:
async for msg in ws:
if msg.type == aiohttp.WSMsgType.TEXT:
if msg.data == "close":
log.debug("Got 'close' msg.")
break
else:
data = json.loads(msg.data)
# log.debug("Websocket (%s) message: %s" % (id(ws), msg))
if data["type"] == "move":
# log.info("Got USER move %s %s %s" % (user.username, data["gameId"], data["move"]))
game = await load_game(request.app, data["gameId"])
move = data["move"]
await play_move(request.app, user, game, move, data["clocks"], data["ply"])
elif data["type"] == "analysis_move":
game = await load_game(request.app, data["gameId"])
await analysis_move(request.app, user, game, data["move"], data["fen"], data["ply"])
elif data["type"] == "ready":
game = await load_game(request.app, data["gameId"])
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
if opp_player.bot:
# Janggi game start have to wait for human player setup!
if game.variant != "janggi" or not (game.bsetup or game.wsetup):
await opp_player.event_queue.put(game.game_start)
response = {"type": "gameStart", "gameId": data["gameId"]}
await ws.send_json(response)
else:
response = {"type": "gameStart", "gameId": data["gameId"]}
await ws.send_json(response)
response = {"type": "user_present", "username": user.username}
await round_broadcast(game, users, game.spectator_list, full=True)
elif data["type"] == "board":
game = await load_game(request.app, data["gameId"])
if game.variant == "janggi":
if (game.bsetup or game.wsetup) and game.status <= STARTED:
if game.bsetup:
await ws.send_json({"type": "setup", "color": "black", "fen": game.board.initial_fen})
elif game.wsetup:
await ws.send_json({"type": "setup", "color": "white", "fen": game.board.initial_fen})
else:
board_response = game.get_board(full=True)
await ws.send_json(board_response)
else:
board_response = game.get_board(full=True)
await ws.send_json(board_response)
elif data["type"] == "setup":
# Janggi game starts with a prelude phase to set up horses and elephants
# First the second player (Red) choses his setup! Then the first player (Blue)
game = await load_game(request.app, data["gameId"])
game.board.initial_fen = data["fen"]
game.initial_fen = game.board.initial_fen
game.board.fen = game.board.initial_fen
# print("--- Got FEN from %s %s" % (data["color"], data["fen"]))
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
game.steps[0]["fen"] = data["fen"]
game.set_dests()
if data["color"] == "black":
game.bsetup = False
response = {"type": "setup", "color": "white", "fen": data["fen"]}
await ws.send_json(response)
if opp_player.bot:
game.board.janggi_setup("w")
game.steps[0]["fen"] = game.board.initial_fen
game.set_dests()
else:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
await opp_ws.send_json(response)
else:
game.wsetup = False
response = game.get_board(full=True)
# log.info("User %s asked board. Server sent: %s" % (user.username, board_response["fen"]))
await ws.send_json(response)
if not opp_player.bot:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
await opp_ws.send_json(response)
if opp_player.bot:
await opp_player.event_queue.put(game.game_start)
elif data["type"] == "analysis":
game = await load_game(request.app, data["gameId"])
# If there is any fishnet client, use it.
if len(request.app["workers"]) > 0:
work_id = "".join(random.choice(string.ascii_letters + string.digits) for x in range(6))
work = {
"work": {
"type": "analysis",
"id": work_id,
},
# or:
# "work": {
# "type": "move",
# "id": "work_id",
# "level": 5 // 1 to 8
# },
"username": data["username"],
"game_id": data["gameId"], # optional
"position": game.board.initial_fen, # start position (X-FEN)
"variant": game.variant,
"chess960": game.chess960,
"moves": " ".join(game.board.move_stack), # moves of the game (UCI)
"nodes": 500000, # optional limit
# "skipPositions": [1, 4, 5] # 0 is the first position
}
request.app["works"][work_id] = work
request.app["fishnet"].put_nowait((ANALYSIS, work_id))
else:
engine = users.get("Fairy-Stockfish")
if (engine is not None) and engine.online():
engine.game_queues[data["gameId"]] = asyncio.Queue()
await engine.event_queue.put(game.analysis_start(data["username"]))
response = {"type": "roundchat", "user": "", "room": "spectator", "message": "Analysis request sent..."}
await ws.send_json(response)
elif data["type"] == "rematch":
game = await load_game(request.app, data["gameId"])
if game is None:
log.debug("Requested game %s not found!")
response = {"type": "game_not_found", "username": user.username, "gameId": data["gameId"]}
await ws.send_json(response)
continue
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
handicap = data["handicap"]
fen = "" if game.variant == "janggi" else game.initial_fen
if opp_player.bot:
if opp_player.username == "Random-Mover":
engine = users.get("Random-Mover")
else:
engine = users.get("Fairy-Stockfish")
if engine is None or not engine.online():
# TODO: message that engine is offline, but capture BOT will play instead
engine = users.get("Random-Mover")
color = "w" if game.wplayer.username == opp_name else "b"
if handicap:
color = "w" if color == "b" else "b"
seek = Seek(
user, game.variant,
fen=fen,
color=color,
base=game.base,
inc=game.inc,
byoyomi_period=game.byoyomi_period,
level=game.level,
rated=game.rated,
chess960=game.chess960)
seeks[seek.id] = seek
response = await new_game(request.app, engine, seek.id)
await ws.send_json(response)
await engine.event_queue.put(challenge(seek, response))
gameId = response["gameId"]
engine.game_queues[gameId] = asyncio.Queue()
else:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
if opp_name in game.rematch_offers:
color = "w" if game.wplayer.username == opp_name else "b"
if handicap:
color = "w" if color == "b" else "b"
seek = Seek(
user, game.variant,
fen=fen,
color=color,
base=game.base,
inc=game.inc,
byoyomi_period=game.byoyomi_period,
level=game.level,
rated=game.rated,
chess960=game.chess960)
seeks[seek.id] = seek
response = await new_game(request.app, opp_player, seek.id)
await ws.send_json(response)
await opp_ws.send_json(response)
else:
game.rematch_offers.add(user.username)
response = {"type": "offer", "message": "Rematch offer sent", "room": "player", "user": ""}
game.messages.append(response)
await ws.send_json(response)
await opp_ws.send_json(response)
elif data["type"] == "draw":
game = await load_game(request.app, data["gameId"])
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
response = await draw(games, data, agreement=opp_name in game.draw_offers)
await ws.send_json(response)
if opp_player.bot:
if game.status > STARTED:
await opp_player.game_queues[data["gameId"]].put(game.game_end)
else:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
await opp_ws.send_json(response)
if opp_name not in game.draw_offers:
game.draw_offers.add(user.username)
await round_broadcast(game, users, response)
elif data["type"] == "logout":
await ws.close()
elif data["type"] == "byoyomi":
game = await load_game(request.app, data["gameId"])
game.byo_correction += game.inc * 1000
game.byoyomi_periods[0 if data["color"] == "white" else 1] = data["period"]
# print("BYOYOMI:", data)
elif data["type"] in ("abort", "resign", "abandone", "flag"):
game = await load_game(request.app, data["gameId"])
if data["type"] == "abort" and (game is not None) and game.board.ply > 2:
continue
response = await game.game_ended(user, data["type"])
await ws.send_json(response)
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
if opp_player.bot:
await opp_player.game_queues[data["gameId"]].put(game.game_end)
else:
if data["gameId"] in users[opp_name].game_sockets:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
await opp_ws.send_json(response)
await round_broadcast(game, users, response)
elif data["type"] == "game_user_connected":
game = await load_game(request.app, data["gameId"])
if session_user is not None:
if data["username"] and data["username"] != session_user:
log.info("+++ Existing game_user %s socket connected as %s." % (session_user, data["username"]))
session_user = data["username"]
if session_user in users:
user = users[session_user]
else:
user = User(request.app, username=data["username"], anon=data["username"].startswith("Anon-"))
users[user.username] = user
# Update logged in users as spactators
if user.username != game.wplayer.username and user.username != game.bplayer.username and game is not None:
game.spectators.add(user)
else:
if session_user in users:
user = users[session_user]
else:
user = User(request.app, username=data["username"], anon=data["username"].startswith("Anon-"))
users[user.username] = user
else:
log.info("+++ Existing game_user %s socket reconnected." % data["username"])
session_user = data["username"]
if session_user in users:
user = users[session_user]
else:
user = User(request.app, username=data["username"], anon=data["username"].startswith("Anon-"))
users[user.username] = user
user.ping_counter = 0
# update websocket
if data["gameId"] in user.game_sockets:
await user.game_sockets[data["gameId"]].close()
user.game_sockets[data["gameId"]] = ws
# remove user seeks
if len(user.lobby_sockets) == 0 or (
game.status <= STARTED and (user.username == game.wplayer.username or user.username == game.bplayer.username)):
await user.clear_seeks(sockets, seeks)
if game is None:
log.debug("Requested game %s not found!")
response = {"type": "game_not_found", "username": user.username, "gameId": data["gameId"]}
await ws.send_json(response)
continue
else:
games[data["gameId"]] = game
if user.username != game.wplayer.username and user.username != game.bplayer.username:
game.spectators.add(user)
await round_broadcast(game, users, game.spectator_list, full=True)
response = {"type": "game_user_connected", "username": user.username, "gameId": data["gameId"], "ply": game.board.ply}
await ws.send_json(response)
response = {"type": "crosstable", "ct": game.crosstable}
await ws.send_json(response)
response = {"type": "fullchat", "lines": list(game.messages)}
await ws.send_json(response)
response = {"type": "user_present", "username": user.username}
await round_broadcast(game, users, response, full=True)
# not connected to lobby socket but connected to game socket
if len(user.game_sockets) == 1 and user.username not in sockets:
response = {"type": "u_cnt", "cnt": online_count(users)}
await lobby_broadcast(sockets, response)
elif data["type"] == "is_user_present":
player_name = data["username"]
player = users.get(player_name)
await asyncio.sleep(1)
if player is not None and data["gameId"] in (player.game_queues if player.bot else player.game_sockets):
response = {"type": "user_present", "username": player_name}
else:
response = {"type": "user_disconnected", "username": player_name}
await ws.send_json(response)
elif data["type"] == "moretime":
# TODO: stop and update game stopwatch time with updated secs
game = await load_game(request.app, data["gameId"])
opp_color = WHITE if user.username == game.bplayer.username else BLACK
if opp_color == game.stopwatch.color:
opp_time = game.stopwatch.stop()
game.stopwatch.restart(opp_time + MORE_TIME)
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
if not opp_player.bot:
opp_ws = users[opp_name].game_sockets[data["gameId"]]
response = {"type": "moretime", "username": opp_name}
await opp_ws.send_json(response)
await round_broadcast(game, users, response)
elif data["type"] == "roundchat":
gameId = data["gameId"]
game = await load_game(request.app, gameId)
# Users running a fishnet worker can ask server side analysis with chat message: !analysis
if data["message"] == "!analysis" and user.username in request.app["fishnet_versions"]:
for step in game.steps:
if "analysis" in step:
del step["analysis"]
await ws.send_json({"type": "request_analysis"})
continue
response = {"type": "roundchat", "user": user.username, "message": data["message"], "room": data["room"]}
game.messages.append(response)
for name in (game.wplayer.username, game.bplayer.username):
player = users[name]
if player.bot:
if gameId in player.game_queues:
await player.game_queues[gameId].put('{"type": "chatLine", "username": "******", "room": "spectator", "text": "%s"}\n' % (user.username, data["message"]))
else:
if gameId in player.game_sockets:
player_ws = player.game_sockets[gameId]
await player_ws.send_json(response)
await round_broadcast(game, users, response)
elif data["type"] == "leave":
response = {"type": "roundchat", "user": "", "message": "%s left the game" % user.username, "room": "player"}
gameId = data["gameId"]
game = await load_game(request.app, gameId)
game.messages.append(response)
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
if not opp_player.bot and gameId in opp_player.game_sockets:
opp_player_ws = opp_player.game_sockets[gameId]
await opp_player_ws.send_json(response)
response = {"type": "user_disconnected", "username": user.username}
await opp_player_ws.send_json(response)
await round_broadcast(game, users, response)
elif data["type"] == "updateTV":
if "profileId" in data and data["profileId"] != "":
gameId = await tv_game_user(db, users, data["profileId"])
else:
gameId = await tv_game(db, request.app)
if gameId != data["gameId"] and gameId is not None:
response = {"type": "updateTV", "gameId": gameId}
await ws.send_json(response)
elif data["type"] == "count":
game = await load_game(request.app, data["gameId"])
cur_player = game.bplayer if game.board.color == BLACK else game.wplayer
opp_name = game.wplayer.username if user.username == game.bplayer.username else game.bplayer.username
opp_player = users[opp_name]
opp_ws = users[opp_name].game_sockets[data["gameId"]]
if user.username == cur_player.username:
if data["mode"] == "start":
game.start_manual_count()
response = {"type": "count", "message": "Board's honor counting started", "room": "player", "user": ""}
await ws.send_json(response)
await opp_ws.send_json(response)
await round_broadcast(game, users, response)
elif data["mode"] == "stop":
game.stop_manual_count()
response = {"type": "count", "message": "Board's honor counting stopped", "room": "player", "user": ""}
await ws.send_json(response)
await opp_ws.send_json(response)
await round_broadcast(game, users, response)
else:
response = {"type": "count", "message": "You can only start/stop board's honor counting on your own turn!", "room": "player", "user": ""}
await ws.send_json(response)
elif msg.type == aiohttp.WSMsgType.CLOSED:
log.debug("--- Round websocket %s msg.type == aiohttp.WSMsgType.CLOSED" % id(ws))
break
elif msg.type == aiohttp.WSMsgType.ERROR:
log.error("--- Round ws %s msg.type == aiohttp.WSMsgType.ERROR" % id(ws))
break
else:
log.debug("--- Round ws other msg.type %s %s" % (msg.type, msg))
except concurrent.futures._base.CancelledError:
# client disconnected
pass
except Exception as e:
log.error("!!! Round ws exception occured: %s" % type(e))
finally:
log.debug("---fianlly: await ws.close()")
await ws.close()
if game is not None and not user.bot:
if game.id in user.game_sockets:
del user.game_sockets[game.id]
if user.username != game.wplayer.username and user.username != game.bplayer.username:
game.spectators.discard(user)
await round_broadcast(game, users, game.spectator_list, full=True)
# not connected to lobby socket and not connected to game socket
if len(user.game_sockets) == 0 and user.username not in sockets:
response = {"type": "u_cnt", "cnt": online_count(users)}
await lobby_broadcast(sockets, response)
if game is not None:
response = {"type": "user_disconnected", "username": user.username}
await round_broadcast(game, users, response, full=True)
return ws
def __init__(self):
self.queue = asyncio.Queue()
self.tasks = [] # workers task list
