async def test_checkpoint_same_timestamp(raw_log_entries, raw_log_entries2,
log_config, context):
handler = LogFileHandler(log_config, context)
os.makedirs(log_config.logs_path, exist_ok=True)
observer = start_observer(handler, log_config.logs_path)
path = Path(f"{log_config.logs_path}{log_config.prefix}")
special_cases = [
raw_log_entries.data[-1][0:24] + raw_log_entries2.data[0][24:],
raw_log_entries.data[-1][0:24] + raw_log_entries2.data[1][24:],
raw_log_entries.data[-1][0:24] + raw_log_entries2.data[-1][24:]
]
with open(path, 'w') as f:
for line in raw_log_entries.data:
f.write(line + '\n')
f.write(special_cases[0] + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 3
assert lines[0] == raw_log_entries.data[0] + '\n'
assert lines[1] == special_cases[0] + '\n'
assert lines[2] == raw_log_entries.data[-1] + '\n'
observer.stop()
observer.join()
handler = LogFileHandler(log_config, context)
observer = start_observer(handler, log_config.logs_path)
# with open(path, 'a') as f:
for line in special_cases:
f.write(line + '\n')
for line in raw_log_entries2.data:
f.write(line + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 4
assert lines[0] == special_cases[0] + '\n'
assert lines[1] == special_cases[-1] + '\n'
assert lines[2] == raw_log_entries2.data[0] + '\n'
assert lines[3] == raw_log_entries2.data[-1] + '\n'
await asyncio.sleep(1)
observer.stop()
observer.join()
async def __service__(context: EventContext) -> Spawn[LogRawBatch]:
config = context.settings(datatype=LogReaderConfig)
event_handler = LogFileHandler(config, context)
logger.info(context,
"Starting LogFileHandler...",
extra=extra(logs_path=config.logs_path,
checkpoint_path=config.checkpoint_path))
observer = start_observer(event_handler, config.logs_path)
logger.info(context, "LogFileHandler started.")
try:
while True:
batch = await event_handler.get_and_reset_batch()
if len(batch) == 0:
logger.info(
context,
"LogFileHandler returned empty batch. Sleeping...")
await asyncio.sleep(config.batch_wait_interval_secs)
else:
for i in range(0, len(batch), config.batch_size):
yield LogRawBatch(data=batch[i:i + config.batch_size + 1])
await asyncio.sleep(config.batch_wait_interval_secs)
event_handler.close_inactive_files()
except KeyboardInterrupt: # pragma: no cover
pass
except Exception as e: # pylint: disable=broad-except # pragma: no cover
logger.error(context, e)
finally:
observer.stop()
observer.join()
async def test_checkpoint_expire(raw_log_entries, raw_log_entries2, log_config,
context):
log_config.file_checkpoint_expire_secs = 1
handler = LogFileHandler(log_config, context)
os.makedirs(log_config.logs_path, exist_ok=True)
observer = start_observer(handler, log_config.logs_path)
path = Path(f"{log_config.logs_path}{log_config.prefix}")
with open(path, 'w') as f:
for line in raw_log_entries.data:
f.write(line + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 2
assert lines[0] == raw_log_entries.data[0] + '\n'
assert lines[1] == raw_log_entries.data[-1] + '\n'
observer.stop()
observer.join()
handler = LogFileHandler(log_config, context)
observer = start_observer(handler, log_config.logs_path)
await asyncio.sleep(2)
# with open(path, 'a') as f:
for line in raw_log_entries2.data:
f.write(line + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 4
assert lines[0] == raw_log_entries.data[0] + '\n'
assert lines[1] == raw_log_entries.data[-1] + '\n'
assert lines[2] == raw_log_entries2.data[0] + '\n'
assert lines[3] == raw_log_entries2.data[-1] + '\n'
await asyncio.sleep(1)
observer.stop()
observer.join()
async def test_read_one_batch(raw_log_entries, log_config, context):
handler = LogFileHandler(log_config, context)
os.makedirs(log_config.logs_path, exist_ok=True)
observer = start_observer(handler, log_config.logs_path)
path = Path(f"{log_config.logs_path}{log_config.prefix}")
with open(path, 'w') as f:
for line in raw_log_entries.data:
f.write(line + '\n')
await asyncio.sleep(2)
lines = await handler.get_and_reset_batch()
assert len(lines) == 2
assert lines[0] == raw_log_entries.data[0] + '\n'
assert lines[1] == raw_log_entries.data[-1] + '\n'
observer.stop()
observer.join()
async def test_move_file(raw_log_entries, raw_log_entries2, log_config,
context):
handler = LogFileHandler(log_config, context)
os.makedirs(log_config.logs_path, exist_ok=True)
observer = start_observer(handler, log_config.logs_path)
path = Path(f"{log_config.logs_path}{log_config.prefix}")
move_path = Path(f"{log_config.logs_path}{str(uuid.uuid4())}")
with open(path, 'w') as f:
for line in raw_log_entries.data:
f.write(line + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 2
assert lines[0] == raw_log_entries.data[0] + '\n'
assert lines[1] == raw_log_entries.data[-1] + '\n'
os.rename(path, move_path)
await asyncio.sleep(1)
with open(path, 'w') as f:
for line in raw_log_entries2.data:
f.write(line + '\n')
f.flush()
await asyncio.sleep(1)
lines = await handler.get_and_reset_batch()
assert len(lines) == 2
assert lines[0] == raw_log_entries2.data[0] + '\n'
assert lines[1] == raw_log_entries2.data[-1] + '\n'
await asyncio.sleep(1)
observer.stop()
observer.join()
