def _collect_log_lines(self, log_file):
"""
Collects logs from logfiles, formats and puts in queue
"""
L = log_file
fpath = L["fpath"]
fmtfn = L["formatter_fn"]
formatter = L["formatter"]
freader = Pygtail(fpath)
for line, line_info in self._iter_logs(freader, fmtfn):
log = self._assign_default_log_values(fpath, line, formatter)
try:
_log = fmtfn(line)
# Identify logs inside a log
# Like process logs inside docker logs
if isinstance(_log, RawLog):
formatter, raw_log = _log["formatter"], _log["raw"]
log.update(_log)
# Give them to actual formatters
_log = load_formatter_fn(formatter)(raw_log)
log.update(_log)
except (SystemExit, KeyboardInterrupt) as e:
raise
except:
log["error"] = True
log["error_tb"] = traceback.format_exc()
self.log.exception("error_during_handling_log_line",
log=log["raw"])
if log["id"] == None:
log["id"] = uuid.uuid1().hex
log = self._remove_redundancy(log)
if self._validate_log_format(log) == "failed":
continue
self.queue.put(
dict(log=json.dumps(log), freader=freader,
line_info=line_info))
self.log.debug("tally:put_into_self.queue",
size=self.queue.qsize())
while not freader.is_fully_acknowledged():
t = self.PYGTAIL_ACK_WAIT_TIME
self.log.debug("waiting_for_pygtail_to_fully_ack", wait_time=t)
time.sleep(t)
def collect_log_lines(self, log_file):
L = log_file
fpath = L['fpath']
fmtfn = L['formatter_fn']
formatter = L['formatter']
freader = Pygtail(fpath)
for line, line_info in self._iter_logs(freader, fmtfn):
log = self.assign_default_log_values(fpath, line, formatter)
try:
_log = fmtfn(line)
if isinstance(_log, RawLog):
formatter, raw_log = _log['formatter'], _log['raw']
log.update(_log)
_log = load_formatter_fn(formatter)(raw_log)
log.update(_log)
except (SystemExit, KeyboardInterrupt) as e:
raise
except:
log['error'] = True
log['error_tb'] = traceback.format_exc()
self.log.exception('error_during_handling_log_line',
log=log['raw'])
if log['id'] == None:
log['id'] = uuid.uuid1().hex
log = self._remove_redundancy(log)
if self.validate_log_format(log) == 'failed': continue
self.queue.put(
dict(log=json.dumps(log), freader=freader,
line_info=line_info))
self.log.debug('tally:put_into_self.queue',
size=self.queue.qsize())
while not freader.is_fully_acknowledged():
t = self.PYGTAIL_ACK_WAIT_TIME
self.log.debug('waiting_for_pygtail_to_fully_ack', wait_time=t)
time.sleep(t)
time.sleep(self.LOG_FILE_POLL_INTERVAL)
