class Worker(BaseWorker[BaseClient, None]):
def __init__(self, supervisor: Supervisor, options: BaseClient,
misc: None) -> None:
self._lock = Lock()
self._bin: Optional[PurePath] = None
self._proc: Optional[Process] = None
self._cwd: Optional[PurePath] = None
super().__init__(supervisor, options=options, misc=misc)
go(supervisor.nvim, aw=self._install())
go(supervisor.nvim, aw=self._poll())
async def _poll(self) -> None:
try:
while True:
await sleep(9)
finally:
proc = self._proc
if proc:
with suppress(ProcessLookupError):
proc.kill()
await proc.wait()
async def _install(self) -> None:
vars_dir = self._supervisor.vars_dir / "clients" / "t9"
bin_path = t9_bin(vars_dir)
if access(bin_path, X_OK):
self._bin = bin_path
else:
for _ in range(9):
await sleep(0)
await awrite(self._supervisor.nvim, LANG("begin T9 download"))
self._bin = await ensure_updated(
vars_dir,
retries=self._supervisor.limits.download_retries,
timeout=self._supervisor.limits.download_timeout,
)
if not self._bin:
await awrite(self._supervisor.nvim, LANG("failed T9 download"))
else:
await awrite(self._supervisor.nvim, LANG("end T9 download"))
async def _clean(self) -> None:
proc = self._proc
if proc:
self._proc = None
with suppress(ProcessLookupError):
proc.kill()
await proc.wait()
async def _comm(self, cwd: PurePath, json: str) -> Optional[str]:
async def cont() -> Optional[str]:
async with self._lock:
if self._bin and not self._proc:
self._proc = await _proc(self._bin, cwd=cwd)
if self._proc:
self._cwd = cwd
if not self._proc:
return None
else:
assert self._proc.stdin and self._proc.stdout
try:
self._proc.stdin.write(encode(json))
self._proc.stdin.write(b"\n")
await self._proc.stdin.drain()
out = await self._proc.stdout.readline()
except (ConnectionError, LimitOverrunError, ValueError):
return await self._clean()
else:
return decode(out)
if self._lock.locked():
return None
else:
return await shield(cont())
async def work(self, context: Context) -> AsyncIterator[Completion]:
if self._cwd != context.cwd:
await self._clean()
if self._bin:
req = _encode(
self._supervisor.match,
context=context,
limit=self._supervisor.match.max_results,
)
json = dumps(req, check_circular=False, ensure_ascii=False)
reply = await self._comm(context.cwd, json=json)
if reply:
try:
resp = loads(reply)
except JSONDecodeError as e:
log.warn("%s", e)
else:
for comp in _decode(self._options, reply=resp):
yield comp
class SmartQueue(Generic[Item]):
def __init__(self, items: AsyncIterator[Item]):
"""
:param items: the items to be iterated over
"""
self._buffer: "asyncio.Queue[Item]" = asyncio.Queue(maxsize=1)
self._incoming_finished = False
self._buffer_task: Optional[
asyncio.Task] = asyncio.get_event_loop().create_task(
self._fill_buffer(items))
"""The items scheduled for reassignment to another consumer"""
self._rescheduled_items: Set[Handle[Item]] = set()
"""The items currently assigned to consumers"""
self._in_progress: Set[Handle[Item]] = set()
# Synchronization primitives
self._lock = Lock()
self._new_items = Condition(lock=self._lock)
self._eof = Condition(lock=self._lock)
async def _fill_buffer(self, incoming: AsyncIterator[Item]):
try:
async for item in incoming:
await self._buffer.put(item)
async with self._lock:
self._new_items.notify_all()
self._incoming_finished = True
async with self._lock:
self._eof.notify_all()
self._new_items.notify_all()
except asyncio.CancelledError:
pass
async def close(self):
if self._buffer_task:
self._buffer_task.cancel()
await self._buffer_task
self._buffer_task = None
def finished(self):
return (not self.has_unassigned_items() and not (self._in_progress)
and self._incoming_finished)
def has_unassigned_items(self) -> bool:
"""Check if this queue has a new or rescheduled item immediately available."""
return bool(self._rescheduled_items) or bool(self._buffer.qsize())
def new_consumer(self) -> "Consumer[Item]":
return Consumer(self)
def __find_rescheduled_item(
self, consumer: "Consumer[Item]") -> Optional[Handle[Item]]:
return next(
(handle for handle in self._rescheduled_items
if consumer not in handle._prev_consumers),
None,
)
async def get(self, consumer: "Consumer[Item]") -> Handle[Item]:
"""Get a handle to the next item to be processed (either a new one or rescheduled)."""
async with self._lock:
while not self.finished():
handle = self.__find_rescheduled_item(consumer)
if handle:
self._rescheduled_items.remove(handle)
self._in_progress.add(handle)
handle.assign_consumer(consumer)
return handle
if self._buffer.qsize():
next_elem = await self._buffer.get()
handle = Handle(next_elem, consumer=consumer)
self._in_progress.add(handle)
return handle
await self._new_items.wait()
self._new_items.notify_all()
raise StopAsyncIteration
async def mark_done(self, handle: Handle[Item]) -> None:
"""Mark an item, referred to by `handle`, as done."""
assert handle in self._in_progress, "handle is not in progress"
async with self._lock:
self._in_progress.remove(handle)
self._eof.notify_all()
self._new_items.notify_all()
if _logger.isEnabledFor(logging.DEBUG):
stats = self.stats()
_logger.debug("status: " +
", ".join(f"{key}: {val}"
for key, val in stats.items()))
async def reschedule(self, handle: Handle[Item]) -> None:
"""Free the item for reassignment to another consumer."""
assert handle in self._in_progress, "handle is not in progress"
async with self._lock:
self._in_progress.remove(handle)
self._rescheduled_items.add(handle)
self._new_items.notify_all()
async def reschedule_all(self, consumer: "Consumer[Item]"):
"""Make all items currently assigned to the consumer available for reassignment."""
async with self._lock:
handles = [
handle for handle in self._in_progress
if handle.consumer == consumer
]
for handle in handles:
self._in_progress.remove(handle)
self._rescheduled_items.add(handle)
self._new_items.notify_all()
def stats(self) -> Dict:
return {
"locked": self._lock.locked(),
"in progress": len(self._in_progress),
"rescheduled": len(self._rescheduled_items),
"in buffer": self._buffer.qsize(),
"incoming finished": self._incoming_finished,
}
async def wait_until_done(self) -> None:
"""Wait until all items in the queue are processed."""
async with self._lock:
while not self.finished():
await self._eof.wait()
class SmartQueue(Generic[Item], object):
def __init__(self, items: Iterable[Item], *, retry_cnt: int = 2):
self._items: Iterator[Item] = peekable(items)
self._rescheduled_items: Set[Handle[Item]] = set()
self._in_progress: Set[Handle[Item]] = set()
# Synchronization primitives
self._lock = Lock()
self._new_items = Condition(lock=self._lock)
self._eof = Condition(lock=self._lock)
def has_new_items(self) -> bool:
"""Check whether this queue has any items that were not retrieved by any consumer yet."""
return bool(self._items)
def has_unassigned_items(self) -> bool:
"""Check whether this queue has any unassigned items.
An item is _unassigned_ if it's new (hasn't been retrieved yet by any consumer)
or it has been rescheduled and is not in progress.
A queue has unassigned items iff `get()` will immediately return some item,
without waiting for an item that is currently "in progress" to be rescheduled.
"""
return self.has_new_items() or bool(self._rescheduled_items)
def new_consumer(self) -> "Consumer[Item]":
return Consumer(self)
def __has_data(self):
return self.has_unassigned_items() or bool(self._in_progress)
def __find_rescheduled_item(
self, consumer: "Consumer[Item]") -> Optional[Handle[Item]]:
return next(
(handle for handle in self._rescheduled_items
if consumer not in handle._prev_consumers),
None,
)
async def get(self, consumer: "Consumer[Item]") -> Handle[Item]:
async with self._lock:
while self.__has_data():
handle = self.__find_rescheduled_item(consumer)
if handle:
self._rescheduled_items.remove(handle)
self._in_progress.add(handle)
handle.assign_consumer(consumer)
return handle
if self.has_new_items():
next_elem = next(self._items)
handle = Handle(next_elem, consumer=consumer)
self._in_progress.add(handle)
return handle
await self._new_items.wait()
self._new_items.notify_all()
raise StopAsyncIteration
async def mark_done(self, handle: Handle[Item]) -> None:
assert handle in self._in_progress, "handle is not in progress"
async with self._lock:
self._in_progress.remove(handle)
self._eof.notify_all()
self._new_items.notify_all()
if _logger.isEnabledFor(logging.DEBUG):
_logger.debug(
f"status in-progress={len(self._in_progress)}, have_item={bool(self._items)}"
)
async def reschedule(self, handle: Handle[Item]) -> None:
assert handle in self._in_progress, "handle is not in progress"
async with self._lock:
self._in_progress.remove(handle)
self._rescheduled_items.add(handle)
self._new_items.notify_all()
async def reschedule_all(self, consumer: "Consumer[Item]"):
async with self._lock:
handles = [
handle for handle in self._in_progress
if handle.consumer == consumer
]
for handle in handles:
self._in_progress.remove(handle)
self._rescheduled_items.add(handle)
self._new_items.notify_all()
def stats(self) -> Dict:
return {
"locked": self._lock.locked(),
"items": bool(self._items),
"in-progress": len(self._in_progress),
"rescheduled-items": len(self._rescheduled_items),
}
async def wait_until_done(self) -> None:
async with self._lock:
while self.__has_data():
await self._eof.wait()