def test_nginx_parser(self): nginx_parser = NginxParser(self.log_dir, "access.nginx.*") control_re_pattern = r'^(?P<remote_addr>(?:\d{1,3}\.){3}\d{1,3}).*'\ r'\[(?P<time_local>.+)\].*'\ r'"(?P<http_method>POST|GET)\s'\ r'(?P<url>.*)\s'\ r'(?P<http_protocol>.*)"\s'\ r'(?P<status>\d{3})\s(?P<size>[0-9]*)\s'\ r'"(?P<http_referer>.*)"\s'\ r'"(?P<http_user_agent>.*)".*$' self.assertEquals(nginx_parser.compiled_log_format, control_re_pattern) for x in nginx_parser.apply_func(nginx_parser.parse_re.match): self.assertEquals(len(x.groupdict()), 9) for x in nginx_parser.get_fields(): self.assertEquals(len(x), 9) ## with self.assertRaises(RuntimeError): ## for x in nginx_parser.field_map('not_valid_field', str): ## print x log = nginx_parser.field_map('status', lambda x: "bbb") for x in log: self.assertEquals(x['status'], "bbb") for x in nginx_parser.map_fn('status', lambda x: (x, 1)): self.assertEquals(x[1], 1) def sort_fn(i_list): new = defaultdict(list) map(lambda x: new[x[0]].append(x[1]), i_list) return new.items() for x in sort_fn(nginx_parser.map_fn('status', lambda x: (x, 1))): pass
def main(): from alstat import get_version from alstat.parsers import NginxParser t = time.time() from optparse import OptionParser usage = "%prog [options] field1, field2, field3 ... " parser = OptionParser(usage) parser.add_option("-d", "--logs-dir", dest="dirname", help="Logs dir", metavar="DIR") parser.add_option("-p", "--names-pattern", help="Log files name pattern", action="store", type="string", dest="pattern", metavar="PATTERN", default="*") parser.add_option("-f", "--log_format", action="store", type="string", dest="log_format", metavar="LOG FORMAT", help="Can be nginx|base", default="nginx") parser.add_option("-l", action="store_true", dest="fields_list", help="Show list of available fields") parser.add_option("-v", action="store_true", dest="verbose", help="Verbose mode") parser.add_option("-V", "--version", action="store_true", dest="version", help="Print version") (options, args) = parser.parse_args() if options.version: print("Alstat version {version}".format(version=get_version())) sys.exit(1) if options.verbose: print("Alstat v{version} start at {time}".format(version=get_version(), time=time.ctime(t))) if options.fields_list: print("You can use fieldnames: " + ", ".join(NginxParser.PATTERNS_MAP.keys())) sys.exit(1) if not args: parser.error("You need specify field name") if options.log_format not in ('nginx', 'base'): parser.error("Log format can be nginx | base") if not options.dirname: parser.error("You need to specify logs dir") log_parser = NginxParser(options.dirname, options.pattern) for x in log_parser.get_fields(): print(" ".join([v for k, v in x.iteritems() if k in args])) if options.verbose: print("Analyze completed {time_duration} sec at {time} ".format(time_duration=(time.time()-t) * 10, time=time.ctime())) sys.exit(1)