def __init__(self, dsn, minsize, maxsize, loop, timeout, *, enable_json,
enable_hstore, echo, **kwargs):
if minsize < 0:
raise ValueError("minsize should be zero or greater")
if maxsize < minsize:
raise ValueError("maxsize should be not less than minsize")
self._dsn = dsn
self._minsize = minsize
self._loop = loop
self._timeout = timeout
self._enable_json = enable_json
self._enable_hstore = enable_hstore
self._echo = echo
self._conn_kwargs = kwargs
self._acquiring = 0
self._free = collections.deque(maxlen=maxsize)
self._cond = asyncio.Condition(loop=loop)
self._used = set()
self._terminated = set()
self._closing = False
self._closed = False
async def await_all(generator):
if isinstance(generator, GeneratorType):
await asyncio.gather(*generator)
elif isinstance(generator, AsyncGeneratorType):
running = 0
cv = asyncio.Condition()
async def finish_task(the_cor):
nonlocal running, cv
await the_cor
running -= 1
async with cv:
cv.notify()
async for cor in generator:
running += 1
asyncio.create_task(finish_task(cor))
async with cv:
await cv.wait_for(lambda: running == 0)
else:
raise ValueError(f'not a generator: {generator}')
def test_context_manager_async_with(self):
primitives = [
asyncio.Lock(loop=self.loop),
asyncio.Condition(loop=self.loop),
asyncio.Semaphore(loop=self.loop),
asyncio.BoundedSemaphore(loop=self.loop),
]
async def test(lock):
await asyncio.sleep(0.01, loop=self.loop)
self.assertFalse(lock.locked())
async with lock as _lock:
self.assertIs(_lock, None)
self.assertTrue(lock.locked())
await asyncio.sleep(0.01, loop=self.loop)
self.assertTrue(lock.locked())
self.assertFalse(lock.locked())
for primitive in primitives:
self.loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
async def test_long_func(loop):
counter = 0
condition = asyncio.Condition()
async def task():
nonlocal counter
async with condition:
await asyncio.sleep(1)
counter += 1
condition.notify_all()
cron = CronCallback(task)
cron.start("* * * * * *", loop)
async with condition:
await condition.wait_for(lambda: counter >= 1)
with pytest.raises(asyncio.CancelledError):
await cron.stop()
assert counter == 1
def pipe(*init: int) -> Tuple[Reader, Writer]:
"""
Provides a Reader/Writer pair that are connected to each other so that multiple programs
can be connected together
"""
buffer = deque(list(init))
cond = asyncio.Condition()
async def reader() -> AsyncGenerator[int, None]:
while True:
async with cond:
while len(buffer) == 0:
await cond.wait()
yield buffer.popleft()
async def writer(val: int) -> None:
async with cond:
buffer.append(val)
cond.notify_all()
return reader, writer
def test_context_manager_with_await(self):
primitives = [
asyncio.Lock(),
asyncio.Condition(),
asyncio.Semaphore(),
asyncio.BoundedSemaphore(),
]
async def test(lock):
await asyncio.sleep(0.01)
self.assertFalse(lock.locked())
with self.assertRaisesRegex(
TypeError,
"can't be used in 'await' expression"
):
with await lock:
pass
for primitive in primitives:
self.loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
def __init__(self,
address,
db=None,
password=None,
encoding=None,
*,
minsize,
maxsize,
ssl=None,
parser=None,
create_connection_timeout=None,
connection_cls=None,
loop=None):
assert isinstance(minsize,
int) and minsize >= 0, ("minsize must be int >= 0",
minsize, type(minsize))
assert maxsize is not None, "Arbitrary pool size is disallowed."
assert isinstance(
maxsize, int) and maxsize > 0, ("maxsize must be int > 0", maxsize,
type(maxsize))
assert minsize <= maxsize, ("Invalid pool min/max sizes", minsize,
maxsize)
if loop is not None and sys.version_info >= (3, 8):
warnings.warn("The loop argument is deprecated",
DeprecationWarning)
self._address = address
self._db = db
self._password = password
self._ssl = ssl
self._encoding = encoding
self._parser_class = parser
self._minsize = minsize
self._create_connection_timeout = create_connection_timeout
self._pool = collections.deque(maxlen=maxsize)
self._used = set()
self._acquiring = 0
self._cond = asyncio.Condition(lock=Lock())
self._close_state = CloseEvent(self._do_close)
self._pubsub_conn = None
self._connection_cls = connection_cls
async def test_async_ttl_cache_returns_in_flight_future():
return_values = iter(range(10))
event = asyncio.Event()
condition = asyncio.Condition()
num_waiting_coroutines = DataHolder(value=0)
# Wait until we have enough coroutines waiting to return a result. This
# ensures that dependent coroutines have a chance to get a future out of
# the cache
@async_ttl_cache(ttl=0)
async def range_coroutine():
await event.wait()
return next(return_values)
# Wait until we have enough coroutines waiting on range_coroutine, then
# wake range_coroutine
async def event_setter():
async with condition:
while num_waiting_coroutines.value != 2:
await condition.wait()
event.set()
# Keep track of how many waiting range_coroutines we have to ensure both
# have had a chance to get the in-flight future out of the cache. This has
# to be separate from range_coroutine since we only end up with one
# invocation of that method due to caching. It also has to be separate
# from event_setter to ensure that the event is not set until both
# coroutines are waiting.
async def cache_waiter():
async with condition:
num_waiting_coroutines.value += 1
condition.notify_all()
return await range_coroutine()
event_setter_future = asyncio.ensure_future(event_setter())
future1 = asyncio.ensure_future(cache_waiter())
future2 = asyncio.ensure_future(cache_waiter())
await asyncio.wait([event_setter_future, future1, future2])
assert future1.result() == future2.result() == 0
async def test_periodic(loop):
condition = asyncio.Condition()
counter = 0
async def task():
nonlocal counter
counter += 1
async with condition:
condition.notify_all()
periodic = aiomisc.PeriodicCallback(task)
periodic.start(0.1, loop)
async with condition:
await asyncio.wait_for(
condition.wait_for(lambda: counter >= 5),
timeout=5,
)
with pytest.raises(asyncio.CancelledError):
await periodic.stop()
def __init__(self, host, port, loop=None):
super(TCPServer, self).__init__(loop)
self._host = host
self._port = port
self._server = None
self._connections = {}
self._client_id = 0
self._state = self._STOPPED
# This condition notified whenever client disconnects / its socket
# closes. The `stop` method won't finish until
# all clients disconnected.
self._stop_event = asyncio.Condition()
self._logger = LoggerAdapter(
self._logger,
extra=dict(host=host, port=port)
)
def __init__(self, url, minsize, maxsize, loop,
timeout, *, pool_recycle, **kwargs):
if minsize < 0:
raise ValueError("minsize should be zero or greater")
if maxsize < minsize and maxsize != 0:
raise ValueError("maxsize should be not less than minsize")
self._url = url
self._minsize = minsize
self._loop = loop
self._timeout = timeout
self._recycle = pool_recycle
self._on_connect = None # on_connect
self._conn_kwargs = kwargs
self._acquiring = 0
self._free = collections.deque(maxlen=maxsize or None)
self._cond = asyncio.Condition(loop=loop)
self._used = set()
self._terminated = set()
self._connect_message_counter = collections.Counter()
self._closing = False
self._closed = False
def __init__(
self,
dsn: str,
minsize: int,
maxsize: int,
timeout: float,
*,
enable_json: bool,
enable_hstore: bool,
enable_uuid: bool,
echo: bool,
on_connect: Optional[Callable[[Connection], Awaitable[None]]],
pool_recycle: float,
**kwargs: Any,
):
if minsize < 0:
raise ValueError("minsize should be zero or greater")
if maxsize < minsize and maxsize != 0:
raise ValueError("maxsize should be not less than minsize")
self._dsn = dsn
self._minsize = minsize
self._loop = get_running_loop()
self._timeout = timeout
self._recycle = pool_recycle
self._enable_json = enable_json
self._enable_hstore = enable_hstore
self._enable_uuid = enable_uuid
self._echo = echo
self._on_connect = on_connect
self._conn_kwargs = kwargs
self._acquiring = 0
self._free: Deque[Connection] = collections.deque(
maxlen=maxsize or None
)
self._cond = asyncio.Condition()
self._used: Set[Connection] = set()
self._terminated: Set[Connection] = set()
self._closing = False
self._closed = False
def mock_shepherd():
def status_gen():
yield "bare_sheep", SheepModel({
"running": False,
"model": {
"name": "model_1",
"version": "latest"
}
})
async def ready(*args):
return args[0] == 'uuid-ready'
async def nothing(*args, **kwargs):
return None
m = mock.create_autospec(Shepherd)
m.get_status.side_effect = status_gen
m.is_job_done.side_effect = ready
m.job_done_condition = asyncio.Condition()
m.enqueue_job.side_effect = nothing
yield m
def test_context_manager_with_await(self):
primitives = [
asyncio.Lock(loop=self.loop),
asyncio.Condition(loop=self.loop),
asyncio.Semaphore(loop=self.loop),
asyncio.BoundedSemaphore(loop=self.loop),
]
async def test(lock):
await asyncio.sleep(0.01)
self.assertFalse(lock.locked())
with self.assertWarns(DeprecationWarning):
with await lock as _lock:
self.assertIs(_lock, None)
self.assertTrue(lock.locked())
await asyncio.sleep(0.01)
self.assertTrue(lock.locked())
self.assertFalse(lock.locked())
for primitive in primitives:
self.loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
async def _send_event(self, request):
event_proto = request.event
event = BaseEvent(key=event_proto.key,
value=event_proto.value,
event_type=event_proto.event_type)
key = event.key
# Lock conditions dict for get/check/update of key
await self.lock.acquire()
if self.notification_conditions.get(key) is None:
self.notification_conditions.update({(key, asyncio.Condition())})
# Release lock after check/update key of notification conditions dict
self.lock.release()
async with self.notification_conditions.get(key), self.write_condition:
event: BaseEvent = self.storage.add_event(event)
self.notification_conditions.get(key).notify_all()
self.write_condition.notify_all()
result_event_proto = event_to_proto(event)
return notification_service_pb2.SendEventsResponse(
event=result_event_proto,
return_code=str(notification_service_pb2.ReturnStatus.SUCCESS),
return_msg='')
async def _list_events(self, request):
event_proto = request.event
key = event_proto.key
version = event_proto.version
timeout_seconds = request.timeout_seconds
if timeout_seconds == 0:
event_models = self._query_events(key, version)
event_proto_list = event_list_to_proto(event_models)
return notification_service_pb2.ListEventsResponse(
return_code=str(notification_service_pb2.ReturnStatus.SUCCESS),
return_msg='',
events=event_proto_list)
else:
start = time.time()
# Lock conditions dict for get/check/update of key
await self.lock.acquire()
if self.notification_conditions.get(key) is None:
self.notification_conditions.update({(key, asyncio.Condition())
})
# Release lock after check/update key of notification conditions dict
self.lock.release()
event_models = self._query_events(key, version)
async with self.notification_conditions.get(key):
while time.time() - start < timeout_seconds and len(
event_models) == 0:
try:
await asyncio.wait_for(
self.notification_conditions.get(key).wait(),
timeout_seconds - time.time() + start)
event_models = self._query_events(key, version)
except asyncio.TimeoutError:
pass
event_proto_list = event_list_to_proto(event_models)
return notification_service_pb2.ListEventsResponse(
return_code=str(notification_service_pb2.ReturnStatus.SUCCESS),
return_msg='',
events=event_proto_list)
def __init__(self,
address,
db=0,
password=None,
encoding=None,
*,
minsize,
maxsize,
commands_factory,
ssl=None,
create_connection_timeout=None,
loop=None):
assert isinstance(minsize,
int) and minsize >= 0, ("minsize must be int >= 0",
minsize, type(minsize))
assert maxsize is not None, "Arbitrary pool size is disallowed."
assert isinstance(
maxsize, int) and maxsize > 0, ("maxsize must be int > 0", maxsize,
type(maxsize))
assert minsize <= maxsize, ("Invalid pool min/max sizes", minsize,
maxsize)
if loop is None:
loop = asyncio.get_event_loop()
self._address = address
self._db = db
self._password = password
self._ssl = ssl
self._encoding = encoding
self._minsize = minsize
self._factory = commands_factory
self._create_connection_timeout = create_connection_timeout
self._loop = loop
self._pool = collections.deque(maxlen=maxsize)
self._used = set()
self._acquiring = 0
self._cond = asyncio.Condition(lock=Lock(loop=loop), loop=loop)
self._close_state = asyncio.Event(loop=loop)
self._close_waiter = None
def __init__(
self, latitude: float, longitude: float, delta: float, country: str
) -> None:
"""
Initialize the component.
This constructor must be run in the event loop.
"""
super().__init__()
self._name = "Buienradar"
# dimension (x and y) of returned radar image
self._dimension = DEFAULT_DIMENSION
# time a cached image stays valid for
self._delta = delta
# country location
self._country = country
# Condition that guards the loading indicator.
#
# Ensures that only one reader can cause an http request at the same
# time, and that all readers are notified after this request completes.
#
# invariant: this condition is private to and owned by this instance.
self._condition = asyncio.Condition()
self._last_image: bytes | None = None
# value of the last seen last modified header
self._last_modified: str | None = None
# loading status
self._loading = False
# deadline for image refresh - self.delta after last successful load
self._deadline: datetime | None = None
self._unique_id = f"{latitude:2.6f}{longitude:2.6f}"
async def main():
"""
main task
"""
# set start time
start_time = datetime.datetime.now()
# get movie urls
get_movie_urls()
# get proxies
get_proxies()
print('代理总数: %d\n' % len(proxies))
if len(proxies) == 0:
print('代理列表没代理啦，凉凉= =')
sys.exit(0)
# create client session connected to the internet
async with aiohttp.ClientSession() as session:
num_urls = len(movie_urls)
# set condition variable
global cond
cond = asyncio.Condition()
# generate tasks for spider
tasks = list()
for i in range(num_urls):
tasks.append(crawl_movie_url(session, movie_urls[i], i + 1))
# put proxy allocation task to the last one
tasks.append(allocate_proxy(len(tasks)))
# execute tasks
await asyncio.gather(*tasks)
# write result to file
with open('movie_info_async.txt', 'w', encoding='utf-8') as movie_file:
global results
results = sorted(results, key=lambda k: k['number'])
movie_file.write(json.dumps(results, indent=2, ensure_ascii=False))
movie_file.close()
# calculate task time
time_period = datetime.datetime.now() - start_time
print('获取结果数：%d个！' % len(results))
print('完成时间：%d秒!' % time_period.seconds)
async def test_restart(loop):
counter = 0
condition = asyncio.Condition()
async def task():
nonlocal counter, condition
counter += 1
async with condition:
condition.notify_all()
cron = CronCallback(task)
cron.start("* * * * * *", loop)
async with condition:
await condition.wait_for(lambda: counter >= 2)
with pytest.raises(asyncio.CancelledError):
await cron.stop()
assert counter == 2
await asyncio.sleep(2)
assert counter == 2
cron.start("* * * * * *", loop)
async with condition:
await condition.wait_for(lambda: counter >= 4)
with pytest.raises(asyncio.CancelledError):
await cron.stop()
assert counter == 4
await asyncio.sleep(2)
assert counter == 4
def test_lock_by_with_statement(self):
loop = asyncio.new_event_loop() # don't use TestLoop quirks
self.set_event_loop(loop)
primitives = [
asyncio.Lock(),
asyncio.Condition(),
asyncio.Semaphore(),
asyncio.BoundedSemaphore(),
]
async def test(lock):
await asyncio.sleep(0.01)
self.assertFalse(lock.locked())
with self.assertRaisesRegex(
TypeError,
r"object \w+ can't be used in 'await' expression"):
with await lock:
pass
self.assertFalse(lock.locked())
for primitive in primitives:
loop.run_until_complete(test(primitive))
self.assertFalse(primitive.locked())
async def test_wait_condition_or_timeout_times_out() -> None:
"""Test wait_condition_or_timeout will timeout."""
test_cond = asyncio.Condition()
async with test_cond:
await aioutils.wait_condition_or_timeout(test_cond, 0.1)
async def _hold_condition():
async with test_cond:
await test_cond.wait()
task = asyncio.ensure_future(_hold_condition())
await asyncio.sleep(0.1)
async def _async_wait_or_timeout():
async with test_cond:
await aioutils.wait_condition_or_timeout(test_cond, 0.1)
# Test high lock contention
await asyncio.gather(*[_async_wait_or_timeout() for _ in range(100)])
task.cancel()
with contextlib.suppress(asyncio.CancelledError):
await task
async def test_no_deadline(loop, max_sleep):
mana = 0
condition = asyncio.Condition()
@aiomisc.asyncbackoff(0.15, None, 0, Exception)
async def test():
nonlocal mana
mana += 1
await asyncio.sleep(max_sleep - (mana - 1) * 0.1)
async with condition:
condition.notify_all()
task = loop.create_task(test())
async with condition:
await asyncio.wait_for(
condition.wait_for(lambda: mana == max_sleep * 10),
timeout=max_sleep * 10,
)
await asyncio.wait_for(task, timeout=max_sleep)
def __init__(self, service, name=None, executor=None):
super().__init__(service, name)
self._state = State.CREATE
# A Task may want to run blocking calls in separate thread. To run a
# method in separate thread, task can use the _run_in_executor() method.
# User can create their own executor instead using the default one
# created by the asyncio. This allows user control over the type of
# executor (task/threads) and its properties (e.g. num_workers)
self._executor = executor
# _update_event can be used to notify coroutines about the change in
# state in this service. e.g. run() has completed
self._update_event = asyncio.Condition(loop=self.loop)
self.set_state(State.INIT)
coro = self.start()
# fixup task name to show actual task in logs
coro.__qualname__ = self._objname
self._task = asyncio.ensure_future(coro, loop=self.loop)
self._ALL_TASKS[self._objname] = self
async def call(self,
func: Callable,
*args,
is_async: Optional[bool] = None,
**kwargs) -> Any:
# Push call details to queue
when_ready = asyncio.Condition()
result = {'ret': None, 'exc_info': (None, None, None)}
# Allow QueueFull exceptions to be propagated to caller
self._queue.put_nowait(
(func, is_async, args, kwargs, when_ready, result))
# Wait for call to be ready
async with when_ready:
await when_ready.wait()
# If an exception was raised, re-raise it
typ, val, tb = result['exc_info']
if typ:
raise val
return result['ret']
async def test_multiple_reader_allowed(zk, path):
WORKERS = 8
counter = 0
cond = asyncio.Condition()
async def start_worker():
nonlocal counter
async with zk.recipes.SharedLock(path).reader_lock:
counter += 1
async with cond:
cond.notify()
async with zk.recipes.SharedLock(path).reader_lock:
try:
for _ in range(WORKERS):
asyncio.create_task(start_worker())
async with cond:
await asyncio.wait_for(
cond.wait_for(lambda: counter == WORKERS), timeout=2)
finally:
await zk.deleteall(path)
assert counter == WORKERS
def get_status(self):
if self.status is not None:
return self.status
result = None
condition = asyncio.Condition(loop=self.loop)
@asyncio.coroutine
def callback(status):
nonlocal result
result = status
with (yield from condition):
condition.notify()
self.add_status_change_callback(callback)
try:
yield from self._get_connection()
with (yield from condition):
yield from condition.wait_for(lambda: result is not None)
return result
except (ConnectionError, ConnectionRefusedError):
return 'offline'
finally:
self.remove_status_change_callback(callback)
async def test_long_func(loop):
counter = 0
condition = asyncio.Condition()
async def task():
nonlocal counter
async with condition:
counter += 1
await asyncio.sleep(0.5)
condition.notify_all()
periodic = aiomisc.PeriodicCallback(task)
periodic.start(0.1, loop)
await asyncio.sleep(1)
with pytest.raises(asyncio.CancelledError):
await periodic.stop()
async with condition:
await asyncio.wait_for(
condition.wait_for(lambda: counter == 2),
timeout=2,
)
async def test_lock_by_with_statement(self):
primitives = [
asyncio.Lock(),
asyncio.Condition(),
asyncio.Semaphore(),
asyncio.BoundedSemaphore(),
]
with self.assertWarns(DeprecationWarning):
@asyncio.coroutine
def test(lock):
yield from asyncio.sleep(0.01)
self.assertFalse(lock.locked())
with self.assertRaisesRegex(TypeError,
"object is not iterable"):
with (yield from lock):
pass
self.assertFalse(lock.locked())
for lock in primitives:
await test(lock)
self.assertFalse(lock.locked())
def __init__(self, service, address,
*, minsize, maxsize, loop=None, timeout=None):
assert isinstance(minsize, int) and minsize >= 0, (
"minsize must be int >= 0", minsize, type(minsize))
assert maxsize is not None, "Arbitrary pool size is disallowed."
assert isinstance(maxsize, int) and maxsize > 0, (
"maxsize must be int > 0", maxsize, type(maxsize))
assert minsize <= maxsize, (
"Invalid pool min/max sizes", minsize, maxsize)
if loop is None:
loop = asyncio.get_event_loop()
self._address = address
self.minsize = minsize
self.maxsize = maxsize
self._loop = loop
self._pool = collections.deque(maxlen=maxsize)
self._used = set()
self._acquiring = 0
self._cond = asyncio.Condition(loop=loop)
self._service = service
self._timeout = timeout
self.closed = False
self._release_tasks = set()
