def __init__(self): self._portal = trio.BlockingTrioPortal() self._request_queue = trio.Queue(1) self._response_queue = trio.Queue(1) self._thread = threading.Thread(target=self.thread_fn, daemon=True) self._thread.start() self._has_quit = False
def __init__(self, circuit, client, context): super().__init__(circuit, client, context) self.nursery = context.nursery self.QueueFull = trio.WouldBlock self.command_queue = trio.Queue(ca.MAX_COMMAND_BACKLOG) self.new_command_condition = trio.Condition() self.subscription_queue = trio.Queue(ca.MAX_TOTAL_SUBSCRIPTION_BACKLOG) self.write_event = Event() self.events_on = trio.Event()
def __init__(self, *, branch_from=None): if branch_from is None: self._portal = trio.BlockingTrioPortal() self._request_queue = trio.Queue(1) self._response_queue = trio.Queue(1) self._thread = threading.Thread(target=self.thread_fn, daemon=True) self._thread.start() else: self._portal = branch_from._portal self._request_queue = branch_from._request_queue self._response_queue = branch_from._response_queue self._thread = branch_from._thread self._branched = branch_from is not None self._has_quit = False
def __init__(self, queue_len=None): if queue_len is None: queue_len = 10000 # Processing queue self._q = trio.Queue(queue_len) # which files to close? self._close_files = set() # set up super().__init__(_TrioSelector()) # replaced internal data self._ready = _Clear() self._scheduled = _Clear() self._default_executor = TrioExecutor() self._orig_signals = {} # we do our own timeout handling self._timers = [] # Marker whether the loop is actually running self._stopped = trio.Event() self._stopped.set()
async def main(): """ This example showcases `as_completed` which is not available in trio however we can mimic its behaviour using a trio.Queue storing the results of each coroutine. Then reading indefinitely from it until no more tasks are scheduled. """ start = time.time() q = trio.Queue(MAX_CLIENTS) async def jockey(coro, i): r = await coro(i) await q.put(r) async with trio.open_nursery() as nursery: """ Note important aspect of Trio, this parent block is itself a task. It may not have any code after the `start_soon` calls, which means the parent task won't do any work and simply wait for child tasks. But if it does include a checkpoint (the await q.get() in this case) the code will run like any other task interleaving with the child ones. """ for i in range(1, MAX_CLIENTS + 1): nursery.start_soon(jockey, fetch_async, i) count = 0 while True: print('* Parent: Checking for results on queue') r = await q.get() print('{} {}'.format(">>" * (count + 1), r)) count += 1 if not nursery.child_tasks: break print("Process took: {:.2f} seconds".format(time.time() - start))
class MemoryListener(trio.abc.Listener): ''' This class is copied from trio's own test suite. ''' closed = attr.ib(default=False) accepted_streams = attr.ib(default=attr.Factory(list)) queued_streams = attr.ib(default=attr.Factory(lambda: trio.Queue(1))) accept_hook = attr.ib(default=None) async def connect(self): assert not self.closed client, server = trio.testing.memory_stream_pair() await self.queued_streams.put(server) return client async def accept(self): await trio.hazmat.checkpoint() assert not self.closed if self.accept_hook is not None: await self.accept_hook() stream = await self.queued_streams.get() self.accepted_streams.append(stream) return stream async def aclose(self): self.closed = True await trio.hazmat.checkpoint()
def __init__(self, concurrency=None, overwrite=False, force_scrape=False): self.overwrite = overwrite self.force_scrape = force_scrape if not concurrency: concurrency = config.SPIDER_DEFAULT_CONCURRENCY asks.init('trio') self.session_pool = trio.Queue(concurrency) for i in range(concurrency): self.session_pool.put_nowait(AsyncSession())
def _async_handler(self, coro): queue = trio.Queue(1) yield (coro, queue) while True: try: return queue.get_nowait() except trio.WouldBlock: pass yield
def __init__(self, conf=None, level=0, logfile=None, debug=False): self.debug = debug level = level or 0 if level > logging.NOTSET: levels = [ logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG, ] if level >= len(levels): level = len(levels) - 1 lvl = levels[level] self.logger.setLevel(lvl) if lvl > logging.DEBUG: LOG_FORMAT = '[%(asctime)s] %(levelname)s in %(module)s.%(funcName)s: %(message)s' else: LOG_FORMAT = ( '-' * 80 + '\n' + '%(levelname)s in %(module)s.%(funcName)s [%(pathname)s:%(lineno)d]:\n' + '%(message)s\n' + '-' * 80 ) if logfile: handler = RotatingFileHandler(logfile, maxBytes=1024 * 1024 * 100, backupCount=20) else: handler = logging.StreamHandler() handler.setLevel(lvl) handler.setFormatter(logging.Formatter(LOG_FORMAT)) self.logger.addHandler(handler) self.logger.info(f'conf: {conf}') self.logger.info('level: {}'.format(logging.getLevelName(lvl))) if not conf: raise IOError('No configuration file found') # Raise exception if errors are encountered during parsing self.conf = config self.conf.parse(conf, True, BUI_DEFAULTS) self.conf.default_section('Global') self.port = self.conf.safe_get('port', 'integer') self.bind = self.conf.safe_get('bind') self.ssl = self.conf.safe_get('ssl', 'boolean') self.sslcert = self.conf.safe_get('sslcert') self.sslkey = self.conf.safe_get('sslkey') self.password = self.conf.safe_get('password') self.pool = self.conf.safe_get('pool', 'integer') self.burpbin = self.conf.safe_get('burpbin', section='Burp') self.bconfcli = self.conf.safe_get('bconfcli', section='Burp') self.timeout = self.conf.safe_get('timeout', 'integer', section='Burp') self.conf.setdefault('BUI_MONITOR', True) self.monitor_pool = trio.Queue(self.pool)
def __init__(self, circuit, *, nursery): self.circuit = circuit # a caproto.VirtualCircuit self.log = self.circuit.log self.nursery = nursery self.channels = {} # map cid to Channel self.ioids = {} # map ioid to Channel self.ioid_data = {} # map ioid to server response self.subscriptionids = {} # map subscriptionid to Channel self.connected = True self.socket = None self.command_queue = trio.Queue(capacity=1000) self.new_command_condition = trio.Condition() self._socket_lock = trio.Lock()
def __init__(self, url: str, nursery): """ :param url: The gateway URL. :param nursery: The nursery to use. """ super().__init__(url) self.nursery = nursery self._portal = trio.BlockingTrioPortal() self._queue = trio.Queue(capacity=5) self._cancelled = threading.Event() self._ws = None # type: WebSocket
async def main(): t0 = datetime.datetime.now() print(colorama.Fore.WHITE + "App started.", flush=True) data = trio.Queue(capacity=10) with trio.move_on_after(25): async with trio.open_nursery() as nursery: nursery.start_soon(generate_data, 20, data, name='Prod 1') nursery.start_soon(generate_data, 20, data, name='Prod 2') nursery.start_soon(process_data, 40, data, name='Consumer') dt = datetime.datetime.now() - t0 print(colorama.Fore.WHITE + "App exiting, total time: {:,.2f} sec.".format( dt.total_seconds()), flush=True)
def __init__(self, *, nursery): self.nursery = nursery self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT) self.log = self.broadcaster.log self.command_bundle_queue = trio.Queue(capacity=1000) self.broadcaster_command_condition = trio.Condition() self._cleanup_condition = trio.Condition() self._cleanup_event = trio.Event() # UDP socket broadcasting to CA servers self.udp_sock = None self.registered = False # refers to RepeaterRegisterRequest self.unanswered_searches = {} # map search id (cid) to name self.search_results = {} # map name to address self.new_id = ThreadsafeCounter( dont_clash_with=self.unanswered_searches)
def __init__(self, sock: trio.SocketStream, callable: bigsig): """ :param sock: The :class:`trio.SocketStream` this server is listening on. """ self._callable = callable self._connection = h11.Connection(h11.SERVER) self._sock = sock # event queue, because asgi is not very good self._event_queue = trio.Queue(capacity=1) # used to store the cancel scopes self._cancels = set() self._keep_alive = None self._canceller = None self._buffer = BytesIO()
async def subscribe(self, *args, **kwargs): "Start a new subscription and spawn an async task to receive readings." command = self.channel.subscribe(*args, **kwargs) # Stash the subscriptionid to match the response to the request. queue = trio.Queue(capacity=100) self.circuit.subscriptionids[command.subscriptionid] = queue await self.circuit.send(command) async def _queue_loop(task_status): task_status.started() while True: command = await queue.get() if command is ca.DISCONNECTED: break self.process_subscription(command) await self.nursery.start(_queue_loop) return command.subscriptionid
def __init__(self, stream, wsproto): ''' Constructor. :param SocketStream stream: :param wsproto: a WSConnection instance :param client: a Trio cancel scope (only used by the server) ''' self._closed = trio.Event() self._close_reason = None self._id = next(self.__class__.CONNECTION_ID) self._message_queue = trio.Queue(0) self._stream = stream self._stream_lock = trio.StrictFIFOLock() self._wsproto = wsproto self._bytes_message = b'' self._str_message = '' self._reader_running = True
async def main(): """ This example showcases `FIRST_COMPLETED` which is not available in trio however we can mimic its behaviour using a trio.Queue storing only one result from the first coroutine to finish, then cancelling the other tasks. """ q = trio.Queue(1) async def jockey(coro, service): r = await coro(service) await q.put(r) async with trio.open_nursery() as nursery: for service in SERVICES: nursery.start_soon(jockey, fetch_ip, service) print(await q.get()) nursery.cancel_scope.cancel()
async def main(): t0 = datetime.datetime.now() print(colorama.Fore.WHITE + "App started.", flush=True) # loop = asyncio.get_event_loop() data = trio.Queue(capacity=10) # 10 items in queue before it blocks # cancel all tasks (and their recursive children etc) if running after 5s with trio.move_on_after(5): async with trio.open_nursery() as nursery: # kick off tasks nursery.start_soon(generate_data, 20, data, name='Prod 1') nursery.start_soon(generate_data, 20, data, name='Prod 2') nursery.start_soon(process_data, 20, data, name='Consumer') dt = datetime.datetime.now() - t0 print(colorama.Fore.WHITE + "App exiting, total time: {:,.2f} sec.".format(dt.total_seconds()), flush=True)
async def main(timeout): response = {"message": "Result from asynchronous.", "ip": "not available"} q = trio.Queue(1) async def jockey(coro, service, timeout): with trio.move_on_after(timeout) as cs: await q.put(await coro(service)) await q.put(None) async with trio.open_nursery() as nursery: for service in SERVICES: nursery.start_soon(jockey, fetch_ip, service, timeout) r = await q.get() if r is not None: response["ip"] = r nursery.cancel_scope.cancel() print(response)
async def main(directories, redis_conn): directory_check_interval = 10 # path/filename : contents hash processed_sources = {} key_q = trio.Queue(1) try: while True: async with trio.open_nursery() as nursery: nursery.start_soon(handle_key_events, redis_conn, key_q) for directory in directories: print("checking directory: {}".format(directory)) nursery.start_soon(source_from, directory, directory_check_interval, processed_sources) nursery.start_soon(ingest_from) sources_from_directory = await q.get() processed_sources.update(sources_from_directory) if processed_sources: redis_conn.hmset("vizaviz:{server_id}:sources".format(server_id=SERVER_ID), processed_sources) key_q.put_nowait("stop") except Exception as ex: # trio works with ctrl-c print(ex, "exiting...")
async def on_connection(stream): ''' Handle a new TCP connection. ''' connection_id = next(connection_counter) remote = stream.socket.getpeername() logger.info('Connection #%d from %s', connection_id, remote[0]) try: name = await get_name(stream) logger.info('Connection #%d: name set to "%s"', connection_id, name) message_queue = trio.Queue(3) message_queues[connection_id] = message_queue async with trio.open_nursery() as nursery: nursery.start_soon(chat_reader, stream, connection_id, message_queue) nursery.start_soon(chat_writer, stream, name) except ClosedByPeer: logger.info('Connection #%d: closed by peer', connection_id) except trio.BrokenStreamError: logger.info('Connection #%d: reset by peer', connection_id) except Exception: logger.exception('Connection #%d: uncaught exception', connection_id) finally: logger.info('Connection #%d: closed', connection_id) message_queues.pop(connection_id, None)
class LoggedClientContext: transport = attr.ib() organization_id = attr.ib() device_id = attr.ib() public_key = attr.ib() verify_key = attr.ib() conn_id = attr.ib(init=False) logger = attr.ib(init=False) subscribed_events = attr.ib(factory=dict) events = attr.ib(factory=lambda: trio.Queue(100)) def __attrs_post_init__(self): self.conn_id = self.transport.conn_id self.logger = self.transport.logger = self.transport.logger.bind( organization_id=str(self.organization_id), client_id=str(self.device_id)) @property def user_id(self): return self.device_id.user_id @property def device_name(self): return self.device_id.device_name
async def __aenter__(self): self.connecting = trio.Event() self.connection_closed = trio.Event() self.state = CONNECTING self.version_major = None self.version_minor = None self.server_properties = None self.server_mechanisms = None self.server_locales = None self.server_heartbeat = None self.channels = {} self.server_frame_max = None self.server_channel_max = None self.channels_ids_ceil = 0 self.channels_ids_free = set() self._send_queue = trio.Queue(1) if self._ssl: ssl_context = ssl.create_default_context() if not verify_ssl: ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE port = self._port if port is None: if self._ssl: port = 5671 else: port = 5672 if self._ssl: stream = await trio.open_ssl_over_tcp_stream( self._host, port, ssl_context=ssl_context) sock = stream.transport_stream else: sock = stream = await trio.open_tcp_stream(self._host, port) # these 2 flags *may* show up in sock.type. They are only available on linux # see https://bugs.python.org/issue21327 nonblock = getattr(sock, 'SOCK_NONBLOCK', 0) cloexec = getattr(sock, 'SOCK_CLOEXEC', 0) if sock is not None and (sock.socket.type & ~nonblock & ~cloexec) == socket.SOCK_STREAM: sock.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self._stream = stream self._nursery_mgr = trio.open_nursery() self._nursery = await self._nursery_mgr.__aenter__() # we will need the writer loop running since the beginning self._nursery.start_soon(self._writer_loop) try: await self._stream.send_all(amqp_constants.PROTOCOL_HEADER) # Wait 'start' method from the server await self.dispatch_frame() if self.version_major is None: raise RuntimeError("Server didn't start with a START packet") client_properties = { 'capabilities': { 'consumer_cancel_notify': True, 'connection.blocked': False, }, 'copyright': 'BSD', 'product': version.__package__, 'product_version': version.__version__, } client_properties.update(self.client_properties) # waiting reply start with credentions and co await self.start_ok(client_properties, 'AMQPLAIN', self._auth, self.server_locales[0]) # wait for a "tune" reponse await self.dispatch_frame() if self.server_channel_max is None: raise RuntimeError("Server didn't send a TUNE packet") tune_ok = { 'channel_max': self.connection_tunning.get('channel_max', self.server_channel_max), 'frame_max': self.connection_tunning.get('frame_max', self.server_frame_max), 'heartbeat': self.connection_tunning.get('heartbeat', self.server_heartbeat), } # "tune" the connexion with max channel, max frame, heartbeat await self.tune_ok(**tune_ok) # update connection tunning values self.server_frame_max = tune_ok['frame_max'] self.server_channel_max = tune_ok['channel_max'] self.server_heartbeat = tune_ok['heartbeat'] # open a virtualhost await self.open(self._virtualhost, capabilities='', insist=self._insist) # wait for open-ok await self.dispatch_frame() if self.state != OPEN: raise exceptions.AmqpClosedConnection() except BaseException as exc: await self.aclose(no_wait=True) raise # read other server's responses asynchronously await self._nursery.start(self.run) return self
def __init__(self, pvdb, interfaces=None): super().__init__(pvdb, interfaces) self.nursery = None self.command_bundle_queue = trio.Queue(1000) self.subscription_queue = trio.Queue(1000) self.beacon_sock = ca.bcast_socket(socket)
def __init__(self, circuit, client, context): super().__init__(circuit, client, context) self.nursery = context.nursery self.command_queue = trio.Queue(1000) self.new_command_condition = trio.Condition()
def __init__(self): self.queue = trio.Queue(max_len) self.portal = portal
async def __aenter__(self): self.connecting = trio.Event() self.connection_closed = trio.Event() self.state = CONNECTING self.version_major = None self.version_minor = None self.server_properties = None self.server_mechanisms = None self.server_locales = None self.server_heartbeat = None self.channels = {} self.server_frame_max = None self.server_channel_max = None self.channels_ids_ceil = 0 self.channels_ids_free = set() self._send_queue = trio.Queue(1) if self._ssl: if self._ssl is True: ssl_context = ssl.create_default_context() else: ssl_context = self._ssl port = self._port if port is None: if self._ssl: port = 5671 else: port = 5672 if self._ssl: stream = await trio.open_ssl_over_tcp_stream(self._host, port, ssl_context=ssl_context) sock = stream.transport_stream else: sock = stream = await trio.open_tcp_stream(self._host, port) sock.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self._stream = stream # the writer loop needs to run since the beginning await self._nursery.start(self._writer_loop) try: await self._stream.send_all(amqp_constants.PROTOCOL_HEADER) # Wait 'start' method from the server await self.dispatch_frame() if self.version_major is None: raise RuntimeError("Server didn't start with a START packet") client_properties = { 'capabilities': { 'consumer_cancel_notify': True, 'connection.blocked': False, }, 'copyright': 'BSD', 'product': _version.__package__, 'product_version': _version.__version__, } client_properties.update(self.client_properties) # waiting reply start with credentions and co await self.start_ok(client_properties, 'AMQPLAIN', self._auth, self.server_locales[0]) # wait for a "tune" reponse await self.dispatch_frame() if self.server_channel_max is None: raise RuntimeError("Server didn't send a TUNE packet") tune_ok = { 'channel_max': self.connection_tunning.get('channel_max', self.server_channel_max), 'frame_max': self.connection_tunning.get('frame_max', self.server_frame_max), 'heartbeat': self.connection_tunning.get('heartbeat', self.server_heartbeat), } # "tune" the connexion with max channel, max frame, heartbeat await self.tune_ok(**tune_ok) # update connection tunning values self.server_frame_max = tune_ok['frame_max'] self.server_channel_max = tune_ok['channel_max'] self.server_heartbeat = tune_ok['heartbeat'] # open a virtualhost await self.open(self._virtualhost, capabilities='', insist=self._insist) # wait for open-ok await self.dispatch_frame() if self.state != OPEN: if self._close_reason is not None: raise exceptions.AmqpClosedConnection(self._close_reason['text']) else: raise exceptions.AmqpClosedConnection() # read the other server's responses asynchronously await self._nursery.start(self._reader_loop) except BaseException as exc: await self.aclose(no_wait=True) raise return self
def __aiter__(self): if self._q is None: self._q = trio.Queue(30) # TODO: 2 + possible prefetch return self
async def __aenter__(self): await self.channel.basic_consume(self._data, consumer_tag=self.consumer_tag, **self.kwargs) self._q = trio.Queue(30) # TODO: 2 + possible prefetch return self
PICS_EXT = ".jpg" PICS_DIR = "pics" # 每次请求延迟(秒) DELAY_TIME = 0.35 # 最大并发数,尽量不要设置得过大 MAX_CONCURRENCY = 64 # 最多重试次数 MAX_RETRY = 5 # 队列容量 MAX_QSIZE = 180000 # 日志等级 LOG_LEVEL = logging.INFO USER_AGENT = ("Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 " "(KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36") Q = trio.Queue(MAX_QSIZE) def get_logger(): """ 获取 logger 实例 """ formatter = logging.Formatter("%(asctime)s - %(message)s") logger = logging.getLogger("monitor") logger.setLevel(LOG_LEVEL) sh = logging.StreamHandler() sh.setFormatter(formatter) logger.addHandler(sh) return logger