async def _write_items_from_queue(self, f):
self.logger.info('Retrieving item from queue for file %s. Timeout: %s',
self.path, p.no('second', self.timeout))
try:
data, fut = self._queue.get_nowait()
except asyncio.QueueEmpty:
data, fut = await asyncio.wait_for(self._queue.get(),
timeout=self.timeout)
futs = [fut]
try:
while not self._queue.empty():
try:
item, fut = self._queue.get_nowait()
data += item
futs.append(fut)
except asyncio.QueueEmpty:
self.logger.warning(
'QueueEmpty error was unexpectedly caught for file %s',
self.path)
self.logger.info('Retrieved %s from queue. Writing to file %s.',
p.no('item', len(futs)), self.path)
await f.write(data)
self.logger.info('%s written to file %s', p.no('byte', len(data)),
self.path)
for fut in futs:
fut.set_result(True)
self.logger.debug('Result set on future %s', id(fut))
except Exception as e:
for fut in futs:
fut.set_exception(e)
finally:
asyncio.get_event_loop().call_soon(self._task_done, len(futs))
async def close(self, exc: Optional[BaseException] = None) -> None:
task_count = self._scheduler.task_count
if task_count:
self.logger.info('Connection waiting on %s to complete',
p.no('task', task_count))
await self._scheduler.close()
self.logger.connection_finished(exc)
async def close(self) -> None:
num_connections = self.num_connections
if num_connections:
self.logger.warning('Waiting to complete tasks for %s', p.no('connection', num_connections))
await asyncio.wait([self._scheduler.close(), self.wait_num_connected(0)], timeout=self.timeout)
self.logger.info('Protocol factory tasks finished, and all connections have been closed')
await asyncio.wait_for(self.close_actions(), self.timeout)
self.logger.info('Actions complete')
connections_manager.clear_server(self.full_name)
async def manage(self) -> None:
self._status.set_starting()
await makedirs(self.path.parent, exist_ok=True)
self.logger.info('Opening file %s', self.path)
async with settings.FILE_OPENER(self.path,
mode=self.mode,
buffering=self.buffering) as f:
try:
self.logger.debug('File %s opened', self.path)
self._status.set_started()
while True:
try:
await self._write_items_from_queue(f)
except asyncio.TimeoutError:
qsize = self._queue.qsize()
if qsize:
self.logger.warning(
'Get item for file %s timed out out even though there %s %s in the queue id %s',
self.path, p.plural_verb('is', qsize),
p.no('item', qsize), id(self._queue))
await self._write_items_from_queue(f)
else:
self.logger.info(
'File %s closing due to timeout on new items to write',
self.path)
break
except asyncio.CancelledError as e:
self.logger.info('File %s closing due to task being cancelled',
self.path)
raise e
finally:
self._cleanup()
qsize = self._queue.qsize()
if qsize:
self.logger.warning(
'There %s %s in queue id %s for path %s even after cleanup',
p.plural_verb('is', qsize), p.no('item', qsize),
id(self._queue), self.path)
await self._write_items_from_queue(f)
self.logger.info('File %s closed', self.path)
async def write(self, data: AnyStr) -> None:
fut = asyncio.Future()
if self._exception:
fut.set_exception(self._exception)
else:
self._queue.put_nowait((data, fut))
if self.logger.isEnabledFor(logging.DEBUG):
qsize = self._queue.qsize()
self.logger.debug('Message added to queue with future %s',
id(fut))
self.logger.debug(
'There %s now %s in the write queue %s for %s',
p.plural_verb('is', qsize), p.no('item', qsize),
id(self._queue), self.path)
await fut
async def start(self, context: BaseContext = None, logger: LoggerType = None) -> None:
self.context = context or self.context
self.logger = logger or self.logger
coros = []
if self.action:
coros.append(self.action.start(logger=logger))
if self.preaction:
coros.append(self.preaction.start(logger=logger))
if self.requester:
coros.append(self.requester.start(logger=logger))
await asyncio.gather(*coros)
if self.expire_connections_after_inactive_minutes:
self.logger.info('Connections will expire after %s',
p.no('minute', self.expire_connections_check_interval_minutes))
self._scheduler.call_cb_periodic(self.expire_connections_check_interval_minutes * 60,
self.check_expired_connections,
task_name=f'Check expired connections for {self.full_name}')
def _task_done(self, num: int) -> None:
for _ in range(0, num):
self._queue.task_done()
self.logger.info('Task done set for %s on file %s', p.no('item', num),
self.path)
def on_buffer_decoded(self,
data: bytes,
num: int,
source: str = 'buffer') -> None:
self._raw_received(data, logging.DEBUG)
self.info("Decoded %s in %s", p.no('message', num), source)
def warn_on_cert_expiry(self, cert_name: str, num_days: int,
expiry_time: datetime.datetime):
self.logger.warning('%s ssl cert will expire in less than %s, on %s',
cert_name, p.no('day', num_days + 1), expiry_time)
def log_num_connections(self, action: str, num_connections: int):
if self.isEnabledFor(logging.DEBUG):
self.log(logging.DEBUG, 'Connection %s. There %s now %s.', action,
p.plural_verb('is', p.num(num_connections)),
p.no('active connection'))