def __init__(self, dsn, minsize, maxsize, loop, timeout, *, enable_json, enable_hstore, echo, **kwargs): if minsize < 0: raise ValueError("minsize should be zero or greater") if maxsize < minsize: raise ValueError("maxsize should be not less than minsize") self._dsn = dsn self._minsize = minsize self._loop = loop self._timeout = timeout self._enable_json = enable_json self._enable_hstore = enable_hstore self._echo = echo self._conn_kwargs = kwargs self._acquiring = 0 self._free = collections.deque(maxlen=maxsize) self._cond = asyncio.Condition(loop=loop) self._used = set() self._terminated = set() self._closing = False self._closed = False
async def await_all(generator): if isinstance(generator, GeneratorType): await asyncio.gather(*generator) elif isinstance(generator, AsyncGeneratorType): running = 0 cv = asyncio.Condition() async def finish_task(the_cor): nonlocal running, cv await the_cor running -= 1 async with cv: cv.notify() async for cor in generator: running += 1 asyncio.create_task(finish_task(cor)) async with cv: await cv.wait_for(lambda: running == 0) else: raise ValueError(f'not a generator: {generator}')
def test_context_manager_async_with(self): primitives = [ asyncio.Lock(loop=self.loop), asyncio.Condition(loop=self.loop), asyncio.Semaphore(loop=self.loop), asyncio.BoundedSemaphore(loop=self.loop), ] async def test(lock): await asyncio.sleep(0.01, loop=self.loop) self.assertFalse(lock.locked()) async with lock as _lock: self.assertIs(_lock, None) self.assertTrue(lock.locked()) await asyncio.sleep(0.01, loop=self.loop) self.assertTrue(lock.locked()) self.assertFalse(lock.locked()) for primitive in primitives: self.loop.run_until_complete(test(primitive)) self.assertFalse(primitive.locked())
async def test_long_func(loop): counter = 0 condition = asyncio.Condition() async def task(): nonlocal counter async with condition: await asyncio.sleep(1) counter += 1 condition.notify_all() cron = CronCallback(task) cron.start("* * * * * *", loop) async with condition: await condition.wait_for(lambda: counter >= 1) with pytest.raises(asyncio.CancelledError): await cron.stop() assert counter == 1
def pipe(*init: int) -> Tuple[Reader, Writer]: """ Provides a Reader/Writer pair that are connected to each other so that multiple programs can be connected together """ buffer = deque(list(init)) cond = asyncio.Condition() async def reader() -> AsyncGenerator[int, None]: while True: async with cond: while len(buffer) == 0: await cond.wait() yield buffer.popleft() async def writer(val: int) -> None: async with cond: buffer.append(val) cond.notify_all() return reader, writer
def test_context_manager_with_await(self): primitives = [ asyncio.Lock(), asyncio.Condition(), asyncio.Semaphore(), asyncio.BoundedSemaphore(), ] async def test(lock): await asyncio.sleep(0.01) self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, "can't be used in 'await' expression" ): with await lock: pass for primitive in primitives: self.loop.run_until_complete(test(primitive)) self.assertFalse(primitive.locked())
def __init__(self, address, db=None, password=None, encoding=None, *, minsize, maxsize, ssl=None, parser=None, create_connection_timeout=None, connection_cls=None, loop=None): assert isinstance(minsize, int) and minsize >= 0, ("minsize must be int >= 0", minsize, type(minsize)) assert maxsize is not None, "Arbitrary pool size is disallowed." assert isinstance( maxsize, int) and maxsize > 0, ("maxsize must be int > 0", maxsize, type(maxsize)) assert minsize <= maxsize, ("Invalid pool min/max sizes", minsize, maxsize) if loop is not None and sys.version_info >= (3, 8): warnings.warn("The loop argument is deprecated", DeprecationWarning) self._address = address self._db = db self._password = password self._ssl = ssl self._encoding = encoding self._parser_class = parser self._minsize = minsize self._create_connection_timeout = create_connection_timeout self._pool = collections.deque(maxlen=maxsize) self._used = set() self._acquiring = 0 self._cond = asyncio.Condition(lock=Lock()) self._close_state = CloseEvent(self._do_close) self._pubsub_conn = None self._connection_cls = connection_cls
async def test_async_ttl_cache_returns_in_flight_future(): return_values = iter(range(10)) event = asyncio.Event() condition = asyncio.Condition() num_waiting_coroutines = DataHolder(value=0) # Wait until we have enough coroutines waiting to return a result. This # ensures that dependent coroutines have a chance to get a future out of # the cache @async_ttl_cache(ttl=0) async def range_coroutine(): await event.wait() return next(return_values) # Wait until we have enough coroutines waiting on range_coroutine, then # wake range_coroutine async def event_setter(): async with condition: while num_waiting_coroutines.value != 2: await condition.wait() event.set() # Keep track of how many waiting range_coroutines we have to ensure both # have had a chance to get the in-flight future out of the cache. This has # to be separate from range_coroutine since we only end up with one # invocation of that method due to caching. It also has to be separate # from event_setter to ensure that the event is not set until both # coroutines are waiting. async def cache_waiter(): async with condition: num_waiting_coroutines.value += 1 condition.notify_all() return await range_coroutine() event_setter_future = asyncio.ensure_future(event_setter()) future1 = asyncio.ensure_future(cache_waiter()) future2 = asyncio.ensure_future(cache_waiter()) await asyncio.wait([event_setter_future, future1, future2]) assert future1.result() == future2.result() == 0
async def test_periodic(loop): condition = asyncio.Condition() counter = 0 async def task(): nonlocal counter counter += 1 async with condition: condition.notify_all() periodic = aiomisc.PeriodicCallback(task) periodic.start(0.1, loop) async with condition: await asyncio.wait_for( condition.wait_for(lambda: counter >= 5), timeout=5, ) with pytest.raises(asyncio.CancelledError): await periodic.stop()
def __init__(self, host, port, loop=None): super(TCPServer, self).__init__(loop) self._host = host self._port = port self._server = None self._connections = {} self._client_id = 0 self._state = self._STOPPED # This condition notified whenever client disconnects / its socket # closes. The `stop` method won't finish until # all clients disconnected. self._stop_event = asyncio.Condition() self._logger = LoggerAdapter( self._logger, extra=dict(host=host, port=port) )
def __init__(self, url, minsize, maxsize, loop, timeout, *, pool_recycle, **kwargs): if minsize < 0: raise ValueError("minsize should be zero or greater") if maxsize < minsize and maxsize != 0: raise ValueError("maxsize should be not less than minsize") self._url = url self._minsize = minsize self._loop = loop self._timeout = timeout self._recycle = pool_recycle self._on_connect = None # on_connect self._conn_kwargs = kwargs self._acquiring = 0 self._free = collections.deque(maxlen=maxsize or None) self._cond = asyncio.Condition(loop=loop) self._used = set() self._terminated = set() self._connect_message_counter = collections.Counter() self._closing = False self._closed = False
def __init__( self, dsn: str, minsize: int, maxsize: int, timeout: float, *, enable_json: bool, enable_hstore: bool, enable_uuid: bool, echo: bool, on_connect: Optional[Callable[[Connection], Awaitable[None]]], pool_recycle: float, **kwargs: Any, ): if minsize < 0: raise ValueError("minsize should be zero or greater") if maxsize < minsize and maxsize != 0: raise ValueError("maxsize should be not less than minsize") self._dsn = dsn self._minsize = minsize self._loop = get_running_loop() self._timeout = timeout self._recycle = pool_recycle self._enable_json = enable_json self._enable_hstore = enable_hstore self._enable_uuid = enable_uuid self._echo = echo self._on_connect = on_connect self._conn_kwargs = kwargs self._acquiring = 0 self._free: Deque[Connection] = collections.deque( maxlen=maxsize or None ) self._cond = asyncio.Condition() self._used: Set[Connection] = set() self._terminated: Set[Connection] = set() self._closing = False self._closed = False
def mock_shepherd(): def status_gen(): yield "bare_sheep", SheepModel({ "running": False, "model": { "name": "model_1", "version": "latest" } }) async def ready(*args): return args[0] == 'uuid-ready' async def nothing(*args, **kwargs): return None m = mock.create_autospec(Shepherd) m.get_status.side_effect = status_gen m.is_job_done.side_effect = ready m.job_done_condition = asyncio.Condition() m.enqueue_job.side_effect = nothing yield m
def test_context_manager_with_await(self): primitives = [ asyncio.Lock(loop=self.loop), asyncio.Condition(loop=self.loop), asyncio.Semaphore(loop=self.loop), asyncio.BoundedSemaphore(loop=self.loop), ] async def test(lock): await asyncio.sleep(0.01) self.assertFalse(lock.locked()) with self.assertWarns(DeprecationWarning): with await lock as _lock: self.assertIs(_lock, None) self.assertTrue(lock.locked()) await asyncio.sleep(0.01) self.assertTrue(lock.locked()) self.assertFalse(lock.locked()) for primitive in primitives: self.loop.run_until_complete(test(primitive)) self.assertFalse(primitive.locked())
async def _send_event(self, request): event_proto = request.event event = BaseEvent(key=event_proto.key, value=event_proto.value, event_type=event_proto.event_type) key = event.key # Lock conditions dict for get/check/update of key await self.lock.acquire() if self.notification_conditions.get(key) is None: self.notification_conditions.update({(key, asyncio.Condition())}) # Release lock after check/update key of notification conditions dict self.lock.release() async with self.notification_conditions.get(key), self.write_condition: event: BaseEvent = self.storage.add_event(event) self.notification_conditions.get(key).notify_all() self.write_condition.notify_all() result_event_proto = event_to_proto(event) return notification_service_pb2.SendEventsResponse( event=result_event_proto, return_code=str(notification_service_pb2.ReturnStatus.SUCCESS), return_msg='')
async def _list_events(self, request): event_proto = request.event key = event_proto.key version = event_proto.version timeout_seconds = request.timeout_seconds if timeout_seconds == 0: event_models = self._query_events(key, version) event_proto_list = event_list_to_proto(event_models) return notification_service_pb2.ListEventsResponse( return_code=str(notification_service_pb2.ReturnStatus.SUCCESS), return_msg='', events=event_proto_list) else: start = time.time() # Lock conditions dict for get/check/update of key await self.lock.acquire() if self.notification_conditions.get(key) is None: self.notification_conditions.update({(key, asyncio.Condition()) }) # Release lock after check/update key of notification conditions dict self.lock.release() event_models = self._query_events(key, version) async with self.notification_conditions.get(key): while time.time() - start < timeout_seconds and len( event_models) == 0: try: await asyncio.wait_for( self.notification_conditions.get(key).wait(), timeout_seconds - time.time() + start) event_models = self._query_events(key, version) except asyncio.TimeoutError: pass event_proto_list = event_list_to_proto(event_models) return notification_service_pb2.ListEventsResponse( return_code=str(notification_service_pb2.ReturnStatus.SUCCESS), return_msg='', events=event_proto_list)
def __init__(self, address, db=0, password=None, encoding=None, *, minsize, maxsize, commands_factory, ssl=None, create_connection_timeout=None, loop=None): assert isinstance(minsize, int) and minsize >= 0, ("minsize must be int >= 0", minsize, type(minsize)) assert maxsize is not None, "Arbitrary pool size is disallowed." assert isinstance( maxsize, int) and maxsize > 0, ("maxsize must be int > 0", maxsize, type(maxsize)) assert minsize <= maxsize, ("Invalid pool min/max sizes", minsize, maxsize) if loop is None: loop = asyncio.get_event_loop() self._address = address self._db = db self._password = password self._ssl = ssl self._encoding = encoding self._minsize = minsize self._factory = commands_factory self._create_connection_timeout = create_connection_timeout self._loop = loop self._pool = collections.deque(maxlen=maxsize) self._used = set() self._acquiring = 0 self._cond = asyncio.Condition(lock=Lock(loop=loop), loop=loop) self._close_state = asyncio.Event(loop=loop) self._close_waiter = None
def __init__( self, latitude: float, longitude: float, delta: float, country: str ) -> None: """ Initialize the component. This constructor must be run in the event loop. """ super().__init__() self._name = "Buienradar" # dimension (x and y) of returned radar image self._dimension = DEFAULT_DIMENSION # time a cached image stays valid for self._delta = delta # country location self._country = country # Condition that guards the loading indicator. # # Ensures that only one reader can cause an http request at the same # time, and that all readers are notified after this request completes. # # invariant: this condition is private to and owned by this instance. self._condition = asyncio.Condition() self._last_image: bytes | None = None # value of the last seen last modified header self._last_modified: str | None = None # loading status self._loading = False # deadline for image refresh - self.delta after last successful load self._deadline: datetime | None = None self._unique_id = f"{latitude:2.6f}{longitude:2.6f}"
async def main(): """ main task """ # set start time start_time = datetime.datetime.now() # get movie urls get_movie_urls() # get proxies get_proxies() print('代理总数: %d\n' % len(proxies)) if len(proxies) == 0: print('代理列表没代理啦,凉凉= =') sys.exit(0) # create client session connected to the internet async with aiohttp.ClientSession() as session: num_urls = len(movie_urls) # set condition variable global cond cond = asyncio.Condition() # generate tasks for spider tasks = list() for i in range(num_urls): tasks.append(crawl_movie_url(session, movie_urls[i], i + 1)) # put proxy allocation task to the last one tasks.append(allocate_proxy(len(tasks))) # execute tasks await asyncio.gather(*tasks) # write result to file with open('movie_info_async.txt', 'w', encoding='utf-8') as movie_file: global results results = sorted(results, key=lambda k: k['number']) movie_file.write(json.dumps(results, indent=2, ensure_ascii=False)) movie_file.close() # calculate task time time_period = datetime.datetime.now() - start_time print('获取结果数:%d个!' % len(results)) print('完成时间:%d秒!' % time_period.seconds)
async def test_restart(loop): counter = 0 condition = asyncio.Condition() async def task(): nonlocal counter, condition counter += 1 async with condition: condition.notify_all() cron = CronCallback(task) cron.start("* * * * * *", loop) async with condition: await condition.wait_for(lambda: counter >= 2) with pytest.raises(asyncio.CancelledError): await cron.stop() assert counter == 2 await asyncio.sleep(2) assert counter == 2 cron.start("* * * * * *", loop) async with condition: await condition.wait_for(lambda: counter >= 4) with pytest.raises(asyncio.CancelledError): await cron.stop() assert counter == 4 await asyncio.sleep(2) assert counter == 4
def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) primitives = [ asyncio.Lock(), asyncio.Condition(), asyncio.Semaphore(), asyncio.BoundedSemaphore(), ] async def test(lock): await asyncio.sleep(0.01) self.assertFalse(lock.locked()) with self.assertRaisesRegex( TypeError, r"object \w+ can't be used in 'await' expression"): with await lock: pass self.assertFalse(lock.locked()) for primitive in primitives: loop.run_until_complete(test(primitive)) self.assertFalse(primitive.locked())
async def test_wait_condition_or_timeout_times_out() -> None: """Test wait_condition_or_timeout will timeout.""" test_cond = asyncio.Condition() async with test_cond: await aioutils.wait_condition_or_timeout(test_cond, 0.1) async def _hold_condition(): async with test_cond: await test_cond.wait() task = asyncio.ensure_future(_hold_condition()) await asyncio.sleep(0.1) async def _async_wait_or_timeout(): async with test_cond: await aioutils.wait_condition_or_timeout(test_cond, 0.1) # Test high lock contention await asyncio.gather(*[_async_wait_or_timeout() for _ in range(100)]) task.cancel() with contextlib.suppress(asyncio.CancelledError): await task
async def test_no_deadline(loop, max_sleep): mana = 0 condition = asyncio.Condition() @aiomisc.asyncbackoff(0.15, None, 0, Exception) async def test(): nonlocal mana mana += 1 await asyncio.sleep(max_sleep - (mana - 1) * 0.1) async with condition: condition.notify_all() task = loop.create_task(test()) async with condition: await asyncio.wait_for( condition.wait_for(lambda: mana == max_sleep * 10), timeout=max_sleep * 10, ) await asyncio.wait_for(task, timeout=max_sleep)
def __init__(self, service, name=None, executor=None): super().__init__(service, name) self._state = State.CREATE # A Task may want to run blocking calls in separate thread. To run a # method in separate thread, task can use the _run_in_executor() method. # User can create their own executor instead using the default one # created by the asyncio. This allows user control over the type of # executor (task/threads) and its properties (e.g. num_workers) self._executor = executor # _update_event can be used to notify coroutines about the change in # state in this service. e.g. run() has completed self._update_event = asyncio.Condition(loop=self.loop) self.set_state(State.INIT) coro = self.start() # fixup task name to show actual task in logs coro.__qualname__ = self._objname self._task = asyncio.ensure_future(coro, loop=self.loop) self._ALL_TASKS[self._objname] = self
async def call(self, func: Callable, *args, is_async: Optional[bool] = None, **kwargs) -> Any: # Push call details to queue when_ready = asyncio.Condition() result = {'ret': None, 'exc_info': (None, None, None)} # Allow QueueFull exceptions to be propagated to caller self._queue.put_nowait( (func, is_async, args, kwargs, when_ready, result)) # Wait for call to be ready async with when_ready: await when_ready.wait() # If an exception was raised, re-raise it typ, val, tb = result['exc_info'] if typ: raise val return result['ret']
async def test_multiple_reader_allowed(zk, path): WORKERS = 8 counter = 0 cond = asyncio.Condition() async def start_worker(): nonlocal counter async with zk.recipes.SharedLock(path).reader_lock: counter += 1 async with cond: cond.notify() async with zk.recipes.SharedLock(path).reader_lock: try: for _ in range(WORKERS): asyncio.create_task(start_worker()) async with cond: await asyncio.wait_for( cond.wait_for(lambda: counter == WORKERS), timeout=2) finally: await zk.deleteall(path) assert counter == WORKERS
def get_status(self): if self.status is not None: return self.status result = None condition = asyncio.Condition(loop=self.loop) @asyncio.coroutine def callback(status): nonlocal result result = status with (yield from condition): condition.notify() self.add_status_change_callback(callback) try: yield from self._get_connection() with (yield from condition): yield from condition.wait_for(lambda: result is not None) return result except (ConnectionError, ConnectionRefusedError): return 'offline' finally: self.remove_status_change_callback(callback)
async def test_long_func(loop): counter = 0 condition = asyncio.Condition() async def task(): nonlocal counter async with condition: counter += 1 await asyncio.sleep(0.5) condition.notify_all() periodic = aiomisc.PeriodicCallback(task) periodic.start(0.1, loop) await asyncio.sleep(1) with pytest.raises(asyncio.CancelledError): await periodic.stop() async with condition: await asyncio.wait_for( condition.wait_for(lambda: counter == 2), timeout=2, )
async def test_lock_by_with_statement(self): primitives = [ asyncio.Lock(), asyncio.Condition(), asyncio.Semaphore(), asyncio.BoundedSemaphore(), ] with self.assertWarns(DeprecationWarning): @asyncio.coroutine def test(lock): yield from asyncio.sleep(0.01) self.assertFalse(lock.locked()) with self.assertRaisesRegex(TypeError, "object is not iterable"): with (yield from lock): pass self.assertFalse(lock.locked()) for lock in primitives: await test(lock) self.assertFalse(lock.locked())
def __init__(self, service, address, *, minsize, maxsize, loop=None, timeout=None): assert isinstance(minsize, int) and minsize >= 0, ( "minsize must be int >= 0", minsize, type(minsize)) assert maxsize is not None, "Arbitrary pool size is disallowed." assert isinstance(maxsize, int) and maxsize > 0, ( "maxsize must be int > 0", maxsize, type(maxsize)) assert minsize <= maxsize, ( "Invalid pool min/max sizes", minsize, maxsize) if loop is None: loop = asyncio.get_event_loop() self._address = address self.minsize = minsize self.maxsize = maxsize self._loop = loop self._pool = collections.deque(maxlen=maxsize) self._used = set() self._acquiring = 0 self._cond = asyncio.Condition(loop=loop) self._service = service self._timeout = timeout self.closed = False self._release_tasks = set()