def process_log(name):
"""Let logstapo loose on a specifig log.
:param name: The name of the log to process
:return: A ``(lines, failed)`` tuple. `lines` is a list of
``(line, data)`` tuples and `failed` is a list of raw
lines that could not be parsed.
"""
config = current_config.data # avoid context lookup all the time
verbosity = config['verbosity']
data = config['logs'][name]
garbage = data['garbage']
ignore = data['ignore']
regexps = [config['regexps'][regex_name] for regex_name in config['logs'][name]['regexps']]
if verbosity >= 1: # pragma: no cover
verbose_echo(1, "*** Processing log '{}' ({})".format(name, ', '.join(data['files'])))
if garbage:
verbose_echo(1, ' Garbage patterns:')
for pattern in garbage:
verbose_echo(1, ' - {}'.format(pattern.pattern))
if ignore:
verbose_echo(1, ' Ignore patterns:')
for source, patterns in ignore.items():
if source.pattern is None:
verbose_echo(1, ' - Any source'.format(source.pattern))
else:
verbose_echo(1, ' - Source: {}'.format(source.pattern))
for pattern in patterns:
verbose_echo(1, ' - {}'.format(pattern.pattern))
lines = itertools.chain.from_iterable(_iter_log_lines(f, config['dry_run']) for f in data['files'])
invalid = []
other = []
garbage_count = 0
ignored_count = 0
for line in lines:
if garbage and any(x.test(line) for x in garbage):
garbage_count += 1
debug_echo('garbage: ' + line)
continue
parsed = _parse_line(line, regexps)
if parsed is None:
warning_echo('[{}] Could not parse: {}'.format(name, line))
invalid.append(line)
continue
if _check_ignored(parsed, ignore):
ignored_count += 1
debug_echo('ignored: ' + line)
continue
verbose_echo(2, line)
other.append((line, parsed))
verbose_echo(1, 'Stats: {} garbage / {} invalid / {} ignored / {} other'.format(garbage_count, len(invalid),
ignored_count, len(other)))
return other, invalid
def _check_rotated_file(path, inode):
for func in (_check_rotated_numext, _check_rotated_dateext):
rotated_path = func(path)
if rotated_path is None:
debug_echo('no rotated file found using ' + func.__name__)
continue
debug_echo('found rotated file candidate using {}: {}'.format(func.__name__, rotated_path))
if os.stat(rotated_path).st_ino == inode:
debug_echo('inodes match, using candidate')
return rotated_path
else:
debug_echo('inodes do not match, discarding candidate')
def _check_rotated_file(path, inode):
for func in (_check_rotated_numext, _check_rotated_dateext):
rotated_path = func(path)
if rotated_path is None:
debug_echo('no rotated file found using ' + func.__name__)
continue
debug_echo('found rotated file candidate using {}: {}'.format(
func.__name__, rotated_path))
if os.stat(rotated_path).st_ino == inode:
debug_echo('inodes match, using candidate')
return rotated_path
else:
debug_echo('inodes do not match, discarding candidate')
def _parse_offset_file(path):
debug_echo('checking offset file ' + path)
try:
with open(path) as f:
inode = int(f.readline())
offset = int(f.readline()) # pragma: no branch
except FileNotFoundError as exc:
debug_echo('open() failed: {}'.format(exc))
return None, 0
except ValueError as exc:
debug_echo('could not parse: {}'.format(exc))
return None, 0
else:
debug_echo('inode={}, offset={}'.format(inode, offset))
return inode, offset
def _parse_offset_file(path):
debug_echo('checking offset file ' + path)
try:
with open(path) as f:
inode = int(f.readline())
offset = int(f.readline()) # pragma: no branch
except FileNotFoundError as exc:
debug_echo('open() failed: {}'.format(exc))
return None, 0
except ValueError as exc:
debug_echo('could not parse: {}'.format(exc))
return None, 0
else:
debug_echo('inode={}, offset={}'.format(inode, offset))
return inode, offset
def run(self, data):
msg = MIMEText(self._build_msg(data))
msg['Subject'] = self.subject
msg['From'] = self.sender
msg['To'] = ', '.join(sorted(self.recipients))
cls = smtplib.SMTP_SSL if self.ssl else smtplib.SMTP
debug_echo('using {} client'.format(cls.__name__))
if current_config['dry_run']:
debug_echo('not sending email due to dry-run')
return
smtp = cls(self.host, self.port)
smtp.set_debuglevel(current_config['debug'])
with smtp:
smtp.ehlo()
if self.starttls:
debug_echo('issuing STARTTLS')
smtp.starttls()
smtp.ehlo()
if self.username and self.password:
debug_echo('logging in as ' + self.username)
smtp.login(self.username, self.password)
debug_echo('sending email')
smtp.send_message(msg)
def run(self, data):
msg = MIMEText(self._build_msg(data))
msg['Subject'] = self.subject
msg['From'] = self.sender
msg['To'] = ', '.join(sorted(self.recipients))
cls = smtplib.SMTP_SSL if self.ssl else smtplib.SMTP
debug_echo('using {} client'.format(cls.__name__))
if current_config['dry_run']:
debug_echo('not sending email due to dry-run')
return
smtp = cls(self.host, self.port)
smtp.set_debuglevel(current_config['debug'])
with smtp:
smtp.ehlo()
if self.starttls:
debug_echo('issuing STARTTLS')
smtp.starttls()
smtp.ehlo()
if self.username and self.password:
debug_echo('logging in as ' + self.username)
smtp.login(self.username, self.password)
debug_echo('sending email')
smtp.send_message(msg)
def logtail(path, offset_path=None, *, dry_run=False):
"""Yield new lines from a logfile.
:param path: The path to the file to read from
:param offset_path: The path to the file where offset/inode
information will be stored. If not set,
``<file>.offset`` will be used.
:param dry_run: If ``True``, the offset file will not be modified
or created.
"""
if offset_path is None:
offset_path = path + '.offset'
try:
logfile = open(path, encoding='utf-8', errors='replace')
except OSError as exc:
warning_echo('Could not read: {} ({})'.format(path, exc))
return
closer = ExitStack()
closer.enter_context(logfile)
with closer:
line_iter = iter([])
stat = os.stat(logfile.fileno())
debug_echo('logfile inode={}, size={}'.format(stat.st_ino, stat.st_size))
inode, offset = _parse_offset_file(offset_path)
if inode is not None:
if stat.st_ino == inode:
debug_echo('inodes are the same')
if offset == stat.st_size:
debug_echo('offset points to eof')
return
elif offset > stat.st_size:
warning_echo('File shrunk since last read: {} ({} < {})'.format(path, stat.st_size, offset))
offset = 0
else:
debug_echo('inode changed, checking for rotated file')
rotated_path = _check_rotated_file(path, inode)
if rotated_path is not None:
try:
rotated_file = open(rotated_path, encoding='utf-8', errors='replace')
except OSError as exc:
warning_echo('Could not read rotated file: {} ({})'.format(rotated_path, exc))
else:
closer.enter_context(rotated_file)
rotated_file.seek(offset)
line_iter = itertools.chain(line_iter, iter(rotated_file))
offset = 0
logfile.seek(offset)
line_iter = itertools.chain(line_iter, iter(logfile))
for line in line_iter:
line = line.strip()
yield line
pos = logfile.tell()
debug_echo('reached end of logfile at {}'.format(pos))
if not dry_run:
debug_echo('writing offset file: ' + offset_path)
_write_offset_file(offset_path, stat.st_ino, pos)
else:
debug_echo('dry run - not writing offset file')
def logtail(path, offset_path=None, *, dry_run=False):
"""Yield new lines from a logfile.
:param path: The path to the file to read from
:param offset_path: The path to the file where offset/inode
information will be stored. If not set,
``<file>.offset`` will be used.
:param dry_run: If ``True``, the offset file will not be modified
or created.
"""
if offset_path is None:
offset_path = path + '.offset'
try:
logfile = open(path, encoding='utf-8', errors='replace')
except OSError as exc:
warning_echo('Could not read: {} ({})'.format(path, exc))
return
closer = ExitStack()
closer.enter_context(logfile)
with closer:
line_iter = iter([])
stat = os.stat(logfile.fileno())
debug_echo('logfile inode={}, size={}'.format(stat.st_ino,
stat.st_size))
inode, offset = _parse_offset_file(offset_path)
if inode is not None:
if stat.st_ino == inode:
debug_echo('inodes are the same')
if offset == stat.st_size:
debug_echo('offset points to eof')
return
elif offset > stat.st_size:
warning_echo(
'File shrunk since last read: {} ({} < {})'.format(
path, stat.st_size, offset))
offset = 0
else:
debug_echo('inode changed, checking for rotated file')
rotated_path = _check_rotated_file(path, inode)
if rotated_path is not None:
try:
rotated_file = open(rotated_path,
encoding='utf-8',
errors='replace')
except OSError as exc:
warning_echo(
'Could not read rotated file: {} ({})'.format(
rotated_path, exc))
else:
closer.enter_context(rotated_file)
rotated_file.seek(offset)
line_iter = itertools.chain(line_iter,
iter(rotated_file))
offset = 0
logfile.seek(offset)
line_iter = itertools.chain(line_iter, iter(logfile))
for line in line_iter:
line = line.strip()
yield line
pos = logfile.tell()
debug_echo('reached end of logfile at {}'.format(pos))
if not dry_run:
debug_echo('writing offset file: ' + offset_path)
_write_offset_file(offset_path, stat.st_ino, pos)
else:
debug_echo('dry run - not writing offset file')
def test_debug_echo(mocker, mock_config, debug):
mock_config({'debug': debug})
secho = mocker.patch('logstapo.util.click.secho')
util.debug_echo('test')
assert secho.called == debug
