def process_log(name): """Let logstapo loose on a specifig log. :param name: The name of the log to process :return: A ``(lines, failed)`` tuple. `lines` is a list of ``(line, data)`` tuples and `failed` is a list of raw lines that could not be parsed. """ config = current_config.data # avoid context lookup all the time verbosity = config['verbosity'] data = config['logs'][name] garbage = data['garbage'] ignore = data['ignore'] regexps = [config['regexps'][regex_name] for regex_name in config['logs'][name]['regexps']] if verbosity >= 1: # pragma: no cover verbose_echo(1, "*** Processing log '{}' ({})".format(name, ', '.join(data['files']))) if garbage: verbose_echo(1, ' Garbage patterns:') for pattern in garbage: verbose_echo(1, ' - {}'.format(pattern.pattern)) if ignore: verbose_echo(1, ' Ignore patterns:') for source, patterns in ignore.items(): if source.pattern is None: verbose_echo(1, ' - Any source'.format(source.pattern)) else: verbose_echo(1, ' - Source: {}'.format(source.pattern)) for pattern in patterns: verbose_echo(1, ' - {}'.format(pattern.pattern)) lines = itertools.chain.from_iterable(_iter_log_lines(f, config['dry_run']) for f in data['files']) invalid = [] other = [] garbage_count = 0 ignored_count = 0 for line in lines: if garbage and any(x.test(line) for x in garbage): garbage_count += 1 debug_echo('garbage: ' + line) continue parsed = _parse_line(line, regexps) if parsed is None: warning_echo('[{}] Could not parse: {}'.format(name, line)) invalid.append(line) continue if _check_ignored(parsed, ignore): ignored_count += 1 debug_echo('ignored: ' + line) continue verbose_echo(2, line) other.append((line, parsed)) verbose_echo(1, 'Stats: {} garbage / {} invalid / {} ignored / {} other'.format(garbage_count, len(invalid), ignored_count, len(other))) return other, invalid
def _check_rotated_file(path, inode): for func in (_check_rotated_numext, _check_rotated_dateext): rotated_path = func(path) if rotated_path is None: debug_echo('no rotated file found using ' + func.__name__) continue debug_echo('found rotated file candidate using {}: {}'.format(func.__name__, rotated_path)) if os.stat(rotated_path).st_ino == inode: debug_echo('inodes match, using candidate') return rotated_path else: debug_echo('inodes do not match, discarding candidate')
def _check_rotated_file(path, inode): for func in (_check_rotated_numext, _check_rotated_dateext): rotated_path = func(path) if rotated_path is None: debug_echo('no rotated file found using ' + func.__name__) continue debug_echo('found rotated file candidate using {}: {}'.format( func.__name__, rotated_path)) if os.stat(rotated_path).st_ino == inode: debug_echo('inodes match, using candidate') return rotated_path else: debug_echo('inodes do not match, discarding candidate')
def _parse_offset_file(path): debug_echo('checking offset file ' + path) try: with open(path) as f: inode = int(f.readline()) offset = int(f.readline()) # pragma: no branch except FileNotFoundError as exc: debug_echo('open() failed: {}'.format(exc)) return None, 0 except ValueError as exc: debug_echo('could not parse: {}'.format(exc)) return None, 0 else: debug_echo('inode={}, offset={}'.format(inode, offset)) return inode, offset
def run(self, data): msg = MIMEText(self._build_msg(data)) msg['Subject'] = self.subject msg['From'] = self.sender msg['To'] = ', '.join(sorted(self.recipients)) cls = smtplib.SMTP_SSL if self.ssl else smtplib.SMTP debug_echo('using {} client'.format(cls.__name__)) if current_config['dry_run']: debug_echo('not sending email due to dry-run') return smtp = cls(self.host, self.port) smtp.set_debuglevel(current_config['debug']) with smtp: smtp.ehlo() if self.starttls: debug_echo('issuing STARTTLS') smtp.starttls() smtp.ehlo() if self.username and self.password: debug_echo('logging in as ' + self.username) smtp.login(self.username, self.password) debug_echo('sending email') smtp.send_message(msg)
def logtail(path, offset_path=None, *, dry_run=False): """Yield new lines from a logfile. :param path: The path to the file to read from :param offset_path: The path to the file where offset/inode information will be stored. If not set, ``<file>.offset`` will be used. :param dry_run: If ``True``, the offset file will not be modified or created. """ if offset_path is None: offset_path = path + '.offset' try: logfile = open(path, encoding='utf-8', errors='replace') except OSError as exc: warning_echo('Could not read: {} ({})'.format(path, exc)) return closer = ExitStack() closer.enter_context(logfile) with closer: line_iter = iter([]) stat = os.stat(logfile.fileno()) debug_echo('logfile inode={}, size={}'.format(stat.st_ino, stat.st_size)) inode, offset = _parse_offset_file(offset_path) if inode is not None: if stat.st_ino == inode: debug_echo('inodes are the same') if offset == stat.st_size: debug_echo('offset points to eof') return elif offset > stat.st_size: warning_echo('File shrunk since last read: {} ({} < {})'.format(path, stat.st_size, offset)) offset = 0 else: debug_echo('inode changed, checking for rotated file') rotated_path = _check_rotated_file(path, inode) if rotated_path is not None: try: rotated_file = open(rotated_path, encoding='utf-8', errors='replace') except OSError as exc: warning_echo('Could not read rotated file: {} ({})'.format(rotated_path, exc)) else: closer.enter_context(rotated_file) rotated_file.seek(offset) line_iter = itertools.chain(line_iter, iter(rotated_file)) offset = 0 logfile.seek(offset) line_iter = itertools.chain(line_iter, iter(logfile)) for line in line_iter: line = line.strip() yield line pos = logfile.tell() debug_echo('reached end of logfile at {}'.format(pos)) if not dry_run: debug_echo('writing offset file: ' + offset_path) _write_offset_file(offset_path, stat.st_ino, pos) else: debug_echo('dry run - not writing offset file')
def logtail(path, offset_path=None, *, dry_run=False): """Yield new lines from a logfile. :param path: The path to the file to read from :param offset_path: The path to the file where offset/inode information will be stored. If not set, ``<file>.offset`` will be used. :param dry_run: If ``True``, the offset file will not be modified or created. """ if offset_path is None: offset_path = path + '.offset' try: logfile = open(path, encoding='utf-8', errors='replace') except OSError as exc: warning_echo('Could not read: {} ({})'.format(path, exc)) return closer = ExitStack() closer.enter_context(logfile) with closer: line_iter = iter([]) stat = os.stat(logfile.fileno()) debug_echo('logfile inode={}, size={}'.format(stat.st_ino, stat.st_size)) inode, offset = _parse_offset_file(offset_path) if inode is not None: if stat.st_ino == inode: debug_echo('inodes are the same') if offset == stat.st_size: debug_echo('offset points to eof') return elif offset > stat.st_size: warning_echo( 'File shrunk since last read: {} ({} < {})'.format( path, stat.st_size, offset)) offset = 0 else: debug_echo('inode changed, checking for rotated file') rotated_path = _check_rotated_file(path, inode) if rotated_path is not None: try: rotated_file = open(rotated_path, encoding='utf-8', errors='replace') except OSError as exc: warning_echo( 'Could not read rotated file: {} ({})'.format( rotated_path, exc)) else: closer.enter_context(rotated_file) rotated_file.seek(offset) line_iter = itertools.chain(line_iter, iter(rotated_file)) offset = 0 logfile.seek(offset) line_iter = itertools.chain(line_iter, iter(logfile)) for line in line_iter: line = line.strip() yield line pos = logfile.tell() debug_echo('reached end of logfile at {}'.format(pos)) if not dry_run: debug_echo('writing offset file: ' + offset_path) _write_offset_file(offset_path, stat.st_ino, pos) else: debug_echo('dry run - not writing offset file')
def test_debug_echo(mocker, mock_config, debug): mock_config({'debug': debug}) secho = mocker.patch('logstapo.util.click.secho') util.debug_echo('test') assert secho.called == debug