async def _run_draw_loop(self):
period = 1 / self._frame_rate
while True:
t_start = anyio.current_time()
await self._hasPeers.wait()
peers = self._peers.copy()
self.inputEvents.clear()
for msg in self._receiveQueue:
self._handleDeferredMessage(msg)
self._receiveQueue.clear()
self._is_draw_context = True
with self.pushContext():
self._draw_fn(self)
self._swapBuffer()
self._is_draw_context = False
# batch messages to minimize send count
# honor explicit batch boundaries (empty string)
# force a break mid-queue to provide some pipelining with client
if len(self._sendQueue) > 1:
mid_index = len(self._sendQueue) // 2
if '' not in self._sendQueue[mid_index - 1:mid_index + 1]:
self._sendQueue.insert(mid_index, '')
strings_iter = iter(self._sendQueue)
while True:
msg = ''.join(takewhile(lambda s: s, strings_iter))
if msg:
await self._broadcast(peers, msg)
else:
break
self._sendQueue.clear()
self.frameCount += 1
user_elapsed = anyio.current_time() - t_start
await anyio.sleep(max(0, period - user_elapsed))
def _get_resource(self, url: str, error_message: str = '') -> Tuple[bool, float]:
fetch_time = 0
unreachable = False
if not error_message:
error_message = f'unable to get resource at {url}'
try:
before = anyio.current_time()
self._driver.get(url)
fetch_time = anyio.current_time() - before
except WebDriverException:
logger.exception(error_message)
unreachable = True
return unreachable, fetch_time
async def _handle_url(self, url: str) -> None:
if self._is_url_already_processed(url):
return
static_url = text = ''
response: Optional[httpx.Response] = None
ur = uri_reference(url)
if ur.scheme == 'file':
static_url = url
logger.debug(
'url %s is a file url so we attempt to read its content')
file_path = ur.path[1:] if platform.system(
) == 'Windows' else ur.path
try:
before = anyio.current_time()
async with await anyio.open_file(file_path) as f:
text = await f.read()
fetch_time = anyio.current_time() - before
except OSError:
logger.exception('unable to open file %s', url)
self.unreachable_urls.add(url)
return
else:
response: httpx.Response = await self._fetch(url)
if response.is_error:
logger.info(
'fetching url %s returns an error with status code %s',
url, response.status_code)
self.unreachable_urls.add(url)
return
fetch_time = response.elapsed.total_seconds()
# we update some variables for statistics
self.request_counter += 1
self.reachable_urls.add(url)
self._total_fetch_time += fetch_time
try:
await self.parse(
self, self._get_static_response(static_url, text, response))
except Exception:
logger.exception(
'something unexpected happened while parsing the content at url %s',
url)
if not self._ignore_errors:
self._queue.task_done()
raise
self._queue.task_done()
logger.info('content at url %s has been processed', url)
async def run(self) -> None:
"""Runs the spider."""
async with anyio.create_task_group() as tg:
tg.start_soon(self.worker, tg)
await self._queue.join()
# at this point, all the urls were handled, so the only remaining task is the worker
tg.cancel_scope.cancel()
await self._cleanup()
self._duration = anyio.current_time() - self._start_time
async def wait_task_can_start(self) -> None:
while True:
# NOTE: this is an implementation of the "virtual scheduling" variant
# of the GCRA algorithm.
# `next_start_time` represents the TAT (theoretical time of arrival).
# See: https://en.wikipedia.org/wiki/Generic_cell_rate_algorithm
now = anyio.current_time()
next_start_time = max(self.next_start_time, now)
time_until_start = next_start_time - now
threshold = self.period - self.task_delta
if time_until_start <= threshold:
break
await anyio.sleep(max(0, time_until_start - threshold))
async def main() -> None:
async with aioros.init_node("clock", initialize_time=False):
await aioros.set_param("/use_sim_time", True)
async with aioros.create_publication("/clock", Clock) as publisher:
clock = Clock()
ts = anyio.current_time()
while aioros.is_running():
for _ in range(10):
print(clock)
await publisher.publish(clock)
clock.clock.nsecs += 100000000
ts += 0.1
await anyio.sleep_until(ts)
clock.clock.secs += 1
clock.clock.nsecs = 0
async def task(self, *args: Any) -> None:
time = float(anyio.current_time())
self.start_times.append(time)
async def test_current_time(self) -> None:
value = await maybe_async(current_time())
assert type(value) is float
async def test_shielded_deadline() -> None:
with move_on_after(10):
with CancelScope(shield=True):
with move_on_after(1000):
assert current_effective_deadline() - current_time() > 900
async def notify_task_started(self) -> None:
now = anyio.current_time()
self.next_start_time = max(self.next_start_time,
now) + self.task_delta
