def __init__(self):
self._portal = trio.BlockingTrioPortal()
self._request_queue = trio.Queue(1)
self._response_queue = trio.Queue(1)
self._thread = threading.Thread(target=self.thread_fn, daemon=True)
self._thread.start()
self._has_quit = False
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.nursery = context.nursery
self.QueueFull = trio.WouldBlock
self.command_queue = trio.Queue(ca.MAX_COMMAND_BACKLOG)
self.new_command_condition = trio.Condition()
self.subscription_queue = trio.Queue(ca.MAX_TOTAL_SUBSCRIPTION_BACKLOG)
self.write_event = Event()
self.events_on = trio.Event()
def __init__(self, *, branch_from=None):
if branch_from is None:
self._portal = trio.BlockingTrioPortal()
self._request_queue = trio.Queue(1)
self._response_queue = trio.Queue(1)
self._thread = threading.Thread(target=self.thread_fn, daemon=True)
self._thread.start()
else:
self._portal = branch_from._portal
self._request_queue = branch_from._request_queue
self._response_queue = branch_from._response_queue
self._thread = branch_from._thread
self._branched = branch_from is not None
self._has_quit = False
def __init__(self, queue_len=None):
if queue_len is None:
queue_len = 10000
# Processing queue
self._q = trio.Queue(queue_len)
# which files to close?
self._close_files = set()
# set up
super().__init__(_TrioSelector())
# replaced internal data
self._ready = _Clear()
self._scheduled = _Clear()
self._default_executor = TrioExecutor()
self._orig_signals = {}
# we do our own timeout handling
self._timers = []
# Marker whether the loop is actually running
self._stopped = trio.Event()
self._stopped.set()
async def main():
"""
This example showcases `as_completed` which is not available in trio
however we can mimic its behaviour using a trio.Queue storing the results
of each coroutine. Then reading indefinitely from it until no more tasks
are scheduled.
"""
start = time.time()
q = trio.Queue(MAX_CLIENTS)
async def jockey(coro, i):
r = await coro(i)
await q.put(r)
async with trio.open_nursery() as nursery:
"""
Note important aspect of Trio, this parent block is itself a task.
It may not have any code after the `start_soon` calls, which means
the parent task won't do any work and simply wait for child tasks.
But if it does include a checkpoint (the await q.get() in this case)
the code will run like any other task interleaving with the child ones.
"""
for i in range(1, MAX_CLIENTS + 1):
nursery.start_soon(jockey, fetch_async, i)
count = 0
while True:
print('* Parent: Checking for results on queue')
r = await q.get()
print('{} {}'.format(">>" * (count + 1), r))
count += 1
if not nursery.child_tasks:
break
print("Process took: {:.2f} seconds".format(time.time() - start))
class MemoryListener(trio.abc.Listener):
''' This class is copied from trio's own test suite. '''
closed = attr.ib(default=False)
accepted_streams = attr.ib(default=attr.Factory(list))
queued_streams = attr.ib(default=attr.Factory(lambda: trio.Queue(1)))
accept_hook = attr.ib(default=None)
async def connect(self):
assert not self.closed
client, server = trio.testing.memory_stream_pair()
await self.queued_streams.put(server)
return client
async def accept(self):
await trio.hazmat.checkpoint()
assert not self.closed
if self.accept_hook is not None:
await self.accept_hook()
stream = await self.queued_streams.get()
self.accepted_streams.append(stream)
return stream
async def aclose(self):
self.closed = True
await trio.hazmat.checkpoint()
def __init__(self, concurrency=None, overwrite=False, force_scrape=False):
self.overwrite = overwrite
self.force_scrape = force_scrape
if not concurrency:
concurrency = config.SPIDER_DEFAULT_CONCURRENCY
asks.init('trio')
self.session_pool = trio.Queue(concurrency)
for i in range(concurrency):
self.session_pool.put_nowait(AsyncSession())
def _async_handler(self, coro):
queue = trio.Queue(1)
yield (coro, queue)
while True:
try:
return queue.get_nowait()
except trio.WouldBlock:
pass
yield
def __init__(self, conf=None, level=0, logfile=None, debug=False):
self.debug = debug
level = level or 0
if level > logging.NOTSET:
levels = [
logging.CRITICAL,
logging.ERROR,
logging.WARNING,
logging.INFO,
logging.DEBUG,
]
if level >= len(levels):
level = len(levels) - 1
lvl = levels[level]
self.logger.setLevel(lvl)
if lvl > logging.DEBUG:
LOG_FORMAT = '[%(asctime)s] %(levelname)s in %(module)s.%(funcName)s: %(message)s'
else:
LOG_FORMAT = (
'-' * 80 + '\n' +
'%(levelname)s in %(module)s.%(funcName)s [%(pathname)s:%(lineno)d]:\n' +
'%(message)s\n' +
'-' * 80
)
if logfile:
handler = RotatingFileHandler(logfile, maxBytes=1024 * 1024 * 100, backupCount=20)
else:
handler = logging.StreamHandler()
handler.setLevel(lvl)
handler.setFormatter(logging.Formatter(LOG_FORMAT))
self.logger.addHandler(handler)
self.logger.info(f'conf: {conf}')
self.logger.info('level: {}'.format(logging.getLevelName(lvl)))
if not conf:
raise IOError('No configuration file found')
# Raise exception if errors are encountered during parsing
self.conf = config
self.conf.parse(conf, True, BUI_DEFAULTS)
self.conf.default_section('Global')
self.port = self.conf.safe_get('port', 'integer')
self.bind = self.conf.safe_get('bind')
self.ssl = self.conf.safe_get('ssl', 'boolean')
self.sslcert = self.conf.safe_get('sslcert')
self.sslkey = self.conf.safe_get('sslkey')
self.password = self.conf.safe_get('password')
self.pool = self.conf.safe_get('pool', 'integer')
self.burpbin = self.conf.safe_get('burpbin', section='Burp')
self.bconfcli = self.conf.safe_get('bconfcli', section='Burp')
self.timeout = self.conf.safe_get('timeout', 'integer', section='Burp')
self.conf.setdefault('BUI_MONITOR', True)
self.monitor_pool = trio.Queue(self.pool)
def __init__(self, circuit, *, nursery):
self.circuit = circuit # a caproto.VirtualCircuit
self.log = self.circuit.log
self.nursery = nursery
self.channels = {} # map cid to Channel
self.ioids = {} # map ioid to Channel
self.ioid_data = {} # map ioid to server response
self.subscriptionids = {} # map subscriptionid to Channel
self.connected = True
self.socket = None
self.command_queue = trio.Queue(capacity=1000)
self.new_command_condition = trio.Condition()
self._socket_lock = trio.Lock()
def __init__(self, url: str, nursery):
"""
:param url: The gateway URL.
:param nursery: The nursery to use.
"""
super().__init__(url)
self.nursery = nursery
self._portal = trio.BlockingTrioPortal()
self._queue = trio.Queue(capacity=5)
self._cancelled = threading.Event()
self._ws = None # type: WebSocket
async def main():
t0 = datetime.datetime.now()
print(colorama.Fore.WHITE + "App started.", flush=True)
data = trio.Queue(capacity=10)
with trio.move_on_after(25):
async with trio.open_nursery() as nursery:
nursery.start_soon(generate_data, 20, data, name='Prod 1')
nursery.start_soon(generate_data, 20, data, name='Prod 2')
nursery.start_soon(process_data, 40, data, name='Consumer')
dt = datetime.datetime.now() - t0
print(colorama.Fore.WHITE + "App exiting, total time: {:,.2f} sec.".format(
dt.total_seconds()), flush=True)
def __init__(self, *, nursery):
self.nursery = nursery
self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT)
self.log = self.broadcaster.log
self.command_bundle_queue = trio.Queue(capacity=1000)
self.broadcaster_command_condition = trio.Condition()
self._cleanup_condition = trio.Condition()
self._cleanup_event = trio.Event()
# UDP socket broadcasting to CA servers
self.udp_sock = None
self.registered = False # refers to RepeaterRegisterRequest
self.unanswered_searches = {} # map search id (cid) to name
self.search_results = {} # map name to address
self.new_id = ThreadsafeCounter(
dont_clash_with=self.unanswered_searches)
def __init__(self, sock: trio.SocketStream, callable: bigsig):
"""
:param sock: The :class:`trio.SocketStream` this server is listening on.
"""
self._callable = callable
self._connection = h11.Connection(h11.SERVER)
self._sock = sock
# event queue, because asgi is not very good
self._event_queue = trio.Queue(capacity=1)
# used to store the cancel scopes
self._cancels = set()
self._keep_alive = None
self._canceller = None
self._buffer = BytesIO()
async def subscribe(self, *args, **kwargs):
"Start a new subscription and spawn an async task to receive readings."
command = self.channel.subscribe(*args, **kwargs)
# Stash the subscriptionid to match the response to the request.
queue = trio.Queue(capacity=100)
self.circuit.subscriptionids[command.subscriptionid] = queue
await self.circuit.send(command)
async def _queue_loop(task_status):
task_status.started()
while True:
command = await queue.get()
if command is ca.DISCONNECTED:
break
self.process_subscription(command)
await self.nursery.start(_queue_loop)
return command.subscriptionid
def __init__(self, stream, wsproto):
'''
Constructor.
:param SocketStream stream:
:param wsproto: a WSConnection instance
:param client: a Trio cancel scope (only used by the server)
'''
self._closed = trio.Event()
self._close_reason = None
self._id = next(self.__class__.CONNECTION_ID)
self._message_queue = trio.Queue(0)
self._stream = stream
self._stream_lock = trio.StrictFIFOLock()
self._wsproto = wsproto
self._bytes_message = b''
self._str_message = ''
self._reader_running = True
async def main():
"""
This example showcases `FIRST_COMPLETED` which is not available in trio
however we can mimic its behaviour using a trio.Queue storing only one result
from the first coroutine to finish, then cancelling the other tasks.
"""
q = trio.Queue(1)
async def jockey(coro, service):
r = await coro(service)
await q.put(r)
async with trio.open_nursery() as nursery:
for service in SERVICES:
nursery.start_soon(jockey, fetch_ip, service)
print(await q.get())
nursery.cancel_scope.cancel()
async def main():
t0 = datetime.datetime.now()
print(colorama.Fore.WHITE + "App started.", flush=True)
# loop = asyncio.get_event_loop()
data = trio.Queue(capacity=10) # 10 items in queue before it blocks
# cancel all tasks (and their recursive children etc) if running after 5s
with trio.move_on_after(5):
async with trio.open_nursery() as nursery:
# kick off tasks
nursery.start_soon(generate_data, 20, data, name='Prod 1')
nursery.start_soon(generate_data, 20, data, name='Prod 2')
nursery.start_soon(process_data, 20, data, name='Consumer')
dt = datetime.datetime.now() - t0
print(colorama.Fore.WHITE +
"App exiting, total time: {:,.2f} sec.".format(dt.total_seconds()),
flush=True)
async def main(timeout):
response = {"message": "Result from asynchronous.", "ip": "not available"}
q = trio.Queue(1)
async def jockey(coro, service, timeout):
with trio.move_on_after(timeout) as cs:
await q.put(await coro(service))
await q.put(None)
async with trio.open_nursery() as nursery:
for service in SERVICES:
nursery.start_soon(jockey, fetch_ip, service, timeout)
r = await q.get()
if r is not None:
response["ip"] = r
nursery.cancel_scope.cancel()
print(response)
async def main(directories, redis_conn):
directory_check_interval = 10
# path/filename : contents hash
processed_sources = {}
key_q = trio.Queue(1)
try:
while True:
async with trio.open_nursery() as nursery:
nursery.start_soon(handle_key_events, redis_conn, key_q)
for directory in directories:
print("checking directory: {}".format(directory))
nursery.start_soon(source_from, directory, directory_check_interval, processed_sources)
nursery.start_soon(ingest_from)
sources_from_directory = await q.get()
processed_sources.update(sources_from_directory)
if processed_sources:
redis_conn.hmset("vizaviz:{server_id}:sources".format(server_id=SERVER_ID), processed_sources)
key_q.put_nowait("stop")
except Exception as ex:
# trio works with ctrl-c
print(ex, "exiting...")
async def on_connection(stream):
''' Handle a new TCP connection. '''
connection_id = next(connection_counter)
remote = stream.socket.getpeername()
logger.info('Connection #%d from %s', connection_id, remote[0])
try:
name = await get_name(stream)
logger.info('Connection #%d: name set to "%s"', connection_id, name)
message_queue = trio.Queue(3)
message_queues[connection_id] = message_queue
async with trio.open_nursery() as nursery:
nursery.start_soon(chat_reader, stream, connection_id,
message_queue)
nursery.start_soon(chat_writer, stream, name)
except ClosedByPeer:
logger.info('Connection #%d: closed by peer', connection_id)
except trio.BrokenStreamError:
logger.info('Connection #%d: reset by peer', connection_id)
except Exception:
logger.exception('Connection #%d: uncaught exception', connection_id)
finally:
logger.info('Connection #%d: closed', connection_id)
message_queues.pop(connection_id, None)
class LoggedClientContext:
transport = attr.ib()
organization_id = attr.ib()
device_id = attr.ib()
public_key = attr.ib()
verify_key = attr.ib()
conn_id = attr.ib(init=False)
logger = attr.ib(init=False)
subscribed_events = attr.ib(factory=dict)
events = attr.ib(factory=lambda: trio.Queue(100))
def __attrs_post_init__(self):
self.conn_id = self.transport.conn_id
self.logger = self.transport.logger = self.transport.logger.bind(
organization_id=str(self.organization_id),
client_id=str(self.device_id))
@property
def user_id(self):
return self.device_id.user_id
@property
def device_name(self):
return self.device_id.device_name
async def __aenter__(self):
self.connecting = trio.Event()
self.connection_closed = trio.Event()
self.state = CONNECTING
self.version_major = None
self.version_minor = None
self.server_properties = None
self.server_mechanisms = None
self.server_locales = None
self.server_heartbeat = None
self.channels = {}
self.server_frame_max = None
self.server_channel_max = None
self.channels_ids_ceil = 0
self.channels_ids_free = set()
self._send_queue = trio.Queue(1)
if self._ssl:
ssl_context = ssl.create_default_context()
if not verify_ssl:
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
port = self._port
if port is None:
if self._ssl:
port = 5671
else:
port = 5672
if self._ssl:
stream = await trio.open_ssl_over_tcp_stream(
self._host, port, ssl_context=ssl_context)
sock = stream.transport_stream
else:
sock = stream = await trio.open_tcp_stream(self._host, port)
# these 2 flags *may* show up in sock.type. They are only available on linux
# see https://bugs.python.org/issue21327
nonblock = getattr(sock, 'SOCK_NONBLOCK', 0)
cloexec = getattr(sock, 'SOCK_CLOEXEC', 0)
if sock is not None and (sock.socket.type & ~nonblock
& ~cloexec) == socket.SOCK_STREAM:
sock.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._stream = stream
self._nursery_mgr = trio.open_nursery()
self._nursery = await self._nursery_mgr.__aenter__()
# we will need the writer loop running since the beginning
self._nursery.start_soon(self._writer_loop)
try:
await self._stream.send_all(amqp_constants.PROTOCOL_HEADER)
# Wait 'start' method from the server
await self.dispatch_frame()
if self.version_major is None:
raise RuntimeError("Server didn't start with a START packet")
client_properties = {
'capabilities': {
'consumer_cancel_notify': True,
'connection.blocked': False,
},
'copyright': 'BSD',
'product': version.__package__,
'product_version': version.__version__,
}
client_properties.update(self.client_properties)
# waiting reply start with credentions and co
await self.start_ok(client_properties, 'AMQPLAIN', self._auth,
self.server_locales[0])
# wait for a "tune" reponse
await self.dispatch_frame()
if self.server_channel_max is None:
raise RuntimeError("Server didn't send a TUNE packet")
tune_ok = {
'channel_max':
self.connection_tunning.get('channel_max',
self.server_channel_max),
'frame_max':
self.connection_tunning.get('frame_max',
self.server_frame_max),
'heartbeat':
self.connection_tunning.get('heartbeat',
self.server_heartbeat),
}
# "tune" the connexion with max channel, max frame, heartbeat
await self.tune_ok(**tune_ok)
# update connection tunning values
self.server_frame_max = tune_ok['frame_max']
self.server_channel_max = tune_ok['channel_max']
self.server_heartbeat = tune_ok['heartbeat']
# open a virtualhost
await self.open(self._virtualhost,
capabilities='',
insist=self._insist)
# wait for open-ok
await self.dispatch_frame()
if self.state != OPEN:
raise exceptions.AmqpClosedConnection()
except BaseException as exc:
await self.aclose(no_wait=True)
raise
# read other server's responses asynchronously
await self._nursery.start(self.run)
return self
def __init__(self, pvdb, interfaces=None):
super().__init__(pvdb, interfaces)
self.nursery = None
self.command_bundle_queue = trio.Queue(1000)
self.subscription_queue = trio.Queue(1000)
self.beacon_sock = ca.bcast_socket(socket)
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.nursery = context.nursery
self.command_queue = trio.Queue(1000)
self.new_command_condition = trio.Condition()
def __init__(self):
self.queue = trio.Queue(max_len)
self.portal = portal
async def __aenter__(self):
self.connecting = trio.Event()
self.connection_closed = trio.Event()
self.state = CONNECTING
self.version_major = None
self.version_minor = None
self.server_properties = None
self.server_mechanisms = None
self.server_locales = None
self.server_heartbeat = None
self.channels = {}
self.server_frame_max = None
self.server_channel_max = None
self.channels_ids_ceil = 0
self.channels_ids_free = set()
self._send_queue = trio.Queue(1)
if self._ssl:
if self._ssl is True:
ssl_context = ssl.create_default_context()
else:
ssl_context = self._ssl
port = self._port
if port is None:
if self._ssl:
port = 5671
else:
port = 5672
if self._ssl:
stream = await trio.open_ssl_over_tcp_stream(self._host, port, ssl_context=ssl_context)
sock = stream.transport_stream
else:
sock = stream = await trio.open_tcp_stream(self._host, port)
sock.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._stream = stream
# the writer loop needs to run since the beginning
await self._nursery.start(self._writer_loop)
try:
await self._stream.send_all(amqp_constants.PROTOCOL_HEADER)
# Wait 'start' method from the server
await self.dispatch_frame()
if self.version_major is None:
raise RuntimeError("Server didn't start with a START packet")
client_properties = {
'capabilities': {
'consumer_cancel_notify': True,
'connection.blocked': False,
},
'copyright': 'BSD',
'product': _version.__package__,
'product_version': _version.__version__,
}
client_properties.update(self.client_properties)
# waiting reply start with credentions and co
await self.start_ok(client_properties, 'AMQPLAIN', self._auth, self.server_locales[0])
# wait for a "tune" reponse
await self.dispatch_frame()
if self.server_channel_max is None:
raise RuntimeError("Server didn't send a TUNE packet")
tune_ok = {
'channel_max':
self.connection_tunning.get('channel_max', self.server_channel_max),
'frame_max':
self.connection_tunning.get('frame_max', self.server_frame_max),
'heartbeat':
self.connection_tunning.get('heartbeat', self.server_heartbeat),
}
# "tune" the connexion with max channel, max frame, heartbeat
await self.tune_ok(**tune_ok)
# update connection tunning values
self.server_frame_max = tune_ok['frame_max']
self.server_channel_max = tune_ok['channel_max']
self.server_heartbeat = tune_ok['heartbeat']
# open a virtualhost
await self.open(self._virtualhost, capabilities='', insist=self._insist)
# wait for open-ok
await self.dispatch_frame()
if self.state != OPEN:
if self._close_reason is not None:
raise exceptions.AmqpClosedConnection(self._close_reason['text'])
else:
raise exceptions.AmqpClosedConnection()
# read the other server's responses asynchronously
await self._nursery.start(self._reader_loop)
except BaseException as exc:
await self.aclose(no_wait=True)
raise
return self
def __aiter__(self):
if self._q is None:
self._q = trio.Queue(30) # TODO: 2 + possible prefetch
return self
async def __aenter__(self):
await self.channel.basic_consume(self._data, consumer_tag=self.consumer_tag, **self.kwargs)
self._q = trio.Queue(30) # TODO: 2 + possible prefetch
return self
PICS_EXT = ".jpg"
PICS_DIR = "pics"
# 每次请求延迟（秒）
DELAY_TIME = 0.35
# 最大并发数，尽量不要设置得过大
MAX_CONCURRENCY = 64
# 最多重试次数
MAX_RETRY = 5
# 队列容量
MAX_QSIZE = 180000
# 日志等级
LOG_LEVEL = logging.INFO
USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36")
Q = trio.Queue(MAX_QSIZE)
def get_logger():
"""
获取 logger 实例
"""
formatter = logging.Formatter("%(asctime)s - %(message)s")
logger = logging.getLogger("monitor")
logger.setLevel(LOG_LEVEL)
sh = logging.StreamHandler()
sh.setFormatter(formatter)
logger.addHandler(sh)
return logger
