async def test_rollover_happens_before_a_logline_is_emitted(self):
handler = AsyncTimedRotatingFileHandler(
filename=self.temp_file.name,
when=RolloverInterval.SECONDS,
backup_count=1,
)
formatter = Formatter("%(asctime)s %(message)s")
handler.formatter = formatter
self.assertTrue(os.path.exists(self.temp_file.name))
await asyncio.sleep(1.1)
record = make_log_record(msg="testing - initial")
await handler.emit(record)
await handler.close()
def __init__(self,
stream=None,
level: Union[str, int, LogLevel] = LogLevel.NOTSET,
formatter: Formatter = None,
filter: Filter = None,
*,
loop: Optional[AbstractEventLoop] = None) -> None:
super().__init__(loop=loop)
if stream is None:
stream = sys.stderr
self.stream = stream
self.level = level
if formatter is None:
formatter = Formatter()
self.formatter = formatter
if filter:
self.add_filter(filter)
self.protocol_class = AiologgerProtocol
self._initialization_lock = asyncio.Lock()
self.writer = None
def __init__(
self,
url: StrOrURL,
method: str = "GET",
level: Union[str, int, LogLevel] = LogLevel.NOTSET,
formatter: Formatter = None,
filter: Filter = None,
get_msg_dict: Callable[[], dict] = _get_msg_dict,
*,
loop: Optional[AbstractEventLoop] = None,
):
super().__init__(loop=loop)
self._url = url
self._method = method.upper()
self.level = level
self.formatter: Formatter = formatter if formatter else Formatter()
self.formatter.__class__.get_msg_dict = get_msg_dict
# self.formatter.get_msg_dict = _get_msg_dict # add get_msg_dict for formatter
if filter:
self.add_filter(filter)
async def test_rollover(self):
handler = AsyncTimedRotatingFileHandler(
filename=self.temp_file.name,
when=RolloverInterval.SECONDS,
backup_count=1,
)
formatter = Formatter("%(asctime)s %(message)s")
handler.formatter = formatter
r1 = make_log_record(msg="testing - initial")
await handler.emit(r1)
self.assertTrue(os.path.exists(self.temp_file.name))
await asyncio.sleep(1.1)
r2 = make_log_record(msg="testing - after delay")
await handler.emit(r2)
await handler.close()
# At this point, we should have a recent rotated file which we
# can test for the existence of. However, in practice, on some
# machines which run really slowly, we don't know how far back
# in time to go to look for the log file. So, we go back a fair
# bit, and stop as soon as we see a rotated file. In theory this
# could of course still fail, but the chances are lower.
found = False
now = datetime.datetime.now()
GO_BACK = 5 * 60 # seconds
for secs in range(GO_BACK):
prev = now - datetime.timedelta(seconds=secs)
fn = self.temp_file.name + prev.strftime(".%Y-%m-%d_%H-%M-%S")
found = os.path.exists(fn)
if found:
self.files_to_remove.append(fn)
break
self.assertTrue(
found, msg=f"No rotated files found, went back {GO_BACK} seconds")
def get_logger(level):
return Logger.with_default_handlers(
name=__name__,
level=level,
formatter=Formatter(fmt=("%(levelname)05s [%(asctime)s.%(msecs)03d]"
"[%(module)s:%(lineno)d]: %(message)s")))
import aiofiles
import argparse
import logging
import random
import tqdm
from aiohttp import ClientSession
from aiologger import Logger
from aiologger.formatters.base import Formatter
from pathlib import Path
from tools.aiohttp_ignore_ssl_error import ignore_aiohttp_ssl_error
aio_logger = Logger.with_default_handlers(
name='aio_image_downloader',
# formatter=Formatter(fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
formatter=Formatter(fmt='%(message)s'),
level=logging.INFO,
)
glob_configs = {"export_dir": Path("cats")}
def build_img_export_name(img_url) -> Path:
try:
image_name = str(img_url[(img_url.rfind('/')) + 1:])
if '?' in image_name:
image_name = image_name[:image_name.find('?')]
except:
image_name = str(random.randint(11111, 99999)) + '.jpg'
return glob_configs["export_dir"] / image_name
def _get_formatter(handler_configuration: dict) -> Optional[Formatter]:
formatter = None
if 'formatter' in handler_configuration:
formatter = Formatter(**handler_configuration['formatter'])
return formatter
import json
import sys
from asyncio import AbstractEventLoop
from typing import Optional, Union
from aiologger import settings
from aiologger.utils import loop_compat
from aiologger.filters import Filterer
from aiologger.formatters.base import Formatter
from aiologger.formatters.json import JsonFormatter
from aiologger.levels import LogLevel, get_level_name, check_level
from aiologger.records import LogRecord
# Handler relies on any formatter
_default_formatter = Formatter()
@loop_compat
class Handler(Filterer):
"""
Handler instances dispatch logging events to specific destinations.
The base handler class. Acts as a placeholder which defines the Handler
interface. Handlers can optionally use Formatter instances to format
records as desired. By default, no formatter is specified; in this case,
the 'raw' message as determined by record.message is logged.
"""
def __init__(self, level: LogLevel = LogLevel.NOTSET) -> None:
"""
import os
from aiologger import Logger
from aiologger.handlers.files import AsyncFileHandler
from aiologger.formatters.base import Formatter
# In version 0.4.0 AsyncFileHandler don't have formatter in __init__().
class AsyncFileHandlerFormatter(AsyncFileHandler):
def __init__(self, formatter: Formatter = None, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.formatter = formatter
formatter = Formatter(
fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = Logger.with_default_handlers(name='converter', formatter=formatter)
logger.add_handler(
AsyncFileHandlerFormatter(filename=os.path.join(os.path.dirname(__file__),
f'logs/{logger.name}.txt'),
formatter=formatter))