async def test_checkpoint_same_timestamp(raw_log_entries, raw_log_entries2, log_config, context): handler = LogFileHandler(log_config, context) os.makedirs(log_config.logs_path, exist_ok=True) observer = start_observer(handler, log_config.logs_path) path = Path(f"{log_config.logs_path}{log_config.prefix}") special_cases = [ raw_log_entries.data[-1][0:24] + raw_log_entries2.data[0][24:], raw_log_entries.data[-1][0:24] + raw_log_entries2.data[1][24:], raw_log_entries.data[-1][0:24] + raw_log_entries2.data[-1][24:] ] with open(path, 'w') as f: for line in raw_log_entries.data: f.write(line + '\n') f.write(special_cases[0] + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 3 assert lines[0] == raw_log_entries.data[0] + '\n' assert lines[1] == special_cases[0] + '\n' assert lines[2] == raw_log_entries.data[-1] + '\n' observer.stop() observer.join() handler = LogFileHandler(log_config, context) observer = start_observer(handler, log_config.logs_path) # with open(path, 'a') as f: for line in special_cases: f.write(line + '\n') for line in raw_log_entries2.data: f.write(line + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 4 assert lines[0] == special_cases[0] + '\n' assert lines[1] == special_cases[-1] + '\n' assert lines[2] == raw_log_entries2.data[0] + '\n' assert lines[3] == raw_log_entries2.data[-1] + '\n' await asyncio.sleep(1) observer.stop() observer.join()
async def __service__(context: EventContext) -> Spawn[LogRawBatch]: config = context.settings(datatype=LogReaderConfig) event_handler = LogFileHandler(config, context) logger.info(context, "Starting LogFileHandler...", extra=extra(logs_path=config.logs_path, checkpoint_path=config.checkpoint_path)) observer = start_observer(event_handler, config.logs_path) logger.info(context, "LogFileHandler started.") try: while True: batch = await event_handler.get_and_reset_batch() if len(batch) == 0: logger.info( context, "LogFileHandler returned empty batch. Sleeping...") await asyncio.sleep(config.batch_wait_interval_secs) else: for i in range(0, len(batch), config.batch_size): yield LogRawBatch(data=batch[i:i + config.batch_size + 1]) await asyncio.sleep(config.batch_wait_interval_secs) event_handler.close_inactive_files() except KeyboardInterrupt: # pragma: no cover pass except Exception as e: # pylint: disable=broad-except # pragma: no cover logger.error(context, e) finally: observer.stop() observer.join()
async def test_checkpoint_expire(raw_log_entries, raw_log_entries2, log_config, context): log_config.file_checkpoint_expire_secs = 1 handler = LogFileHandler(log_config, context) os.makedirs(log_config.logs_path, exist_ok=True) observer = start_observer(handler, log_config.logs_path) path = Path(f"{log_config.logs_path}{log_config.prefix}") with open(path, 'w') as f: for line in raw_log_entries.data: f.write(line + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 2 assert lines[0] == raw_log_entries.data[0] + '\n' assert lines[1] == raw_log_entries.data[-1] + '\n' observer.stop() observer.join() handler = LogFileHandler(log_config, context) observer = start_observer(handler, log_config.logs_path) await asyncio.sleep(2) # with open(path, 'a') as f: for line in raw_log_entries2.data: f.write(line + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 4 assert lines[0] == raw_log_entries.data[0] + '\n' assert lines[1] == raw_log_entries.data[-1] + '\n' assert lines[2] == raw_log_entries2.data[0] + '\n' assert lines[3] == raw_log_entries2.data[-1] + '\n' await asyncio.sleep(1) observer.stop() observer.join()
async def test_read_one_batch(raw_log_entries, log_config, context): handler = LogFileHandler(log_config, context) os.makedirs(log_config.logs_path, exist_ok=True) observer = start_observer(handler, log_config.logs_path) path = Path(f"{log_config.logs_path}{log_config.prefix}") with open(path, 'w') as f: for line in raw_log_entries.data: f.write(line + '\n') await asyncio.sleep(2) lines = await handler.get_and_reset_batch() assert len(lines) == 2 assert lines[0] == raw_log_entries.data[0] + '\n' assert lines[1] == raw_log_entries.data[-1] + '\n' observer.stop() observer.join()
async def test_move_file(raw_log_entries, raw_log_entries2, log_config, context): handler = LogFileHandler(log_config, context) os.makedirs(log_config.logs_path, exist_ok=True) observer = start_observer(handler, log_config.logs_path) path = Path(f"{log_config.logs_path}{log_config.prefix}") move_path = Path(f"{log_config.logs_path}{str(uuid.uuid4())}") with open(path, 'w') as f: for line in raw_log_entries.data: f.write(line + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 2 assert lines[0] == raw_log_entries.data[0] + '\n' assert lines[1] == raw_log_entries.data[-1] + '\n' os.rename(path, move_path) await asyncio.sleep(1) with open(path, 'w') as f: for line in raw_log_entries2.data: f.write(line + '\n') f.flush() await asyncio.sleep(1) lines = await handler.get_and_reset_batch() assert len(lines) == 2 assert lines[0] == raw_log_entries2.data[0] + '\n' assert lines[1] == raw_log_entries2.data[-1] + '\n' await asyncio.sleep(1) observer.stop() observer.join()