/
ngx_tail.py
227 lines (187 loc) · 7.35 KB
/
ngx_tail.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
#!/bin/env python
# -*- coding: utf-8 -*-
"""nginx_log_stat - realtime parse nginx log, send stat metric to statsd
modify from ngx_top https://github.com/lebinh/ngxtop
Usage:
nginx_log_stat [options] config --access-log=<access-log> --vhost-prefix=<vhost-prefix> --statsd-port=<statsd-port>
nginx_log_stat info
Options:
-f <format>, --log-format <format> log format as specify in log_format directive. [default: combined]
--no-follow ngxtop default behavior is to ignore current lines in log
and only watch for new lines as they are written to the access log.
Use this flag to tell ngxtop to process the current content of the access log instead.
-v, --verbose more verbose output
-d, --debug print every line and parsed record
-h, --help print this help message.
--version print version information.
Advanced / experimental options:
-c <file>, --config <file> allow ngxtop to parse nginx config file for log format and location.
-i <filter-expression>, --filter <filter-expression> filter in, records satisfied given expression are processed.
-p <filter-expression>, --pre-filter <filter-expression> in-filter expression to check in pre-parsing phase.
"""
from __future__ import print_function
import atexit
from contextlib import closing
import curses
import logging
import os
import sqlite3
import time
import sys
import signal
import socket
import statsd_cli
try:
import urlparse
except ImportError:
import urllib.parse as urlparse
import sys
sys.path.append('.')
from config_parser import detect_log_config, detect_config_path, extract_variables, build_pattern
from utils import error_exit
# ======================
# generator utilities
# ======================
def follow(the_file):
"""
Follow a given file and yield new lines when they are available, like `tail -f`.
"""
with open(the_file) as f:
f.seek(0, 2) # seek to eof
while True:
line = f.readline()
if not line:
time.sleep(0.1) # sleep briefly before trying again
continue
yield line
def map_field(field, func, dict_sequence):
"""
Apply given function to value of given key in every dictionary in sequence and
set the result as new value for that key.
"""
for item in dict_sequence:
try:
item[field] = func(item.get(field, None))
yield item
except ValueError:
pass
def add_field(field, func, dict_sequence):
"""
Apply given function to the record and store result in given field of current record.
Do nothing if record already contains given field.
"""
for item in dict_sequence:
if field not in item:
item[field] = func(item)
yield item
def trace(sequence, phase=''):
for item in sequence:
logging.debug('%s:\n%s', phase, item)
yield item
# ======================
# Access log parsing
# ======================
def parse_request_path(record):
if 'request_uri' in record:
uri = record['request_uri']
elif 'request' in record:
uri = ' '.join(record['request'].split(' ')[1:-1])
else:
uri = None
return urlparse.urlparse(uri).path if uri else None
def parse_status_type(record):
return record['status'] // 100 if 'status' in record else None
def to_int(value):
return int(value) if value and value != '-' else 0
def to_float(value):
return float(value) if value and value != '-' else 0.0
def parse_log(lines, pattern):
matches = (pattern.match(l) for l in lines)
records = (m.groupdict() for m in matches if m is not None)
records = map_field('status', to_int, records)
records = add_field('status_type', parse_status_type, records)
records = add_field('bytes_sent', lambda r: r['body_bytes_sent'], records)
records = map_field('bytes_sent', to_int, records)
records = map_field('request_time', to_float, records)
records = add_field('request_path', parse_request_path, records)
return records
def process_log(lines, pattern, processor, arguments):
print(lines)
pre_filer_exp = arguments['--pre-filter']
#下面的for循环会导致lines这个生成器的yield返回数据
if pre_filer_exp:
lines = (line for line in lines if eval(pre_filer_exp, {}, dict(line=line)))
#下面的for循环会导致records这个生成器的yield返回数据
records = parse_log(lines, pattern)
print(records)
filter_exp = arguments['--filter']
if filter_exp:
records = (r for r in records if eval(filter_exp, {}, r))
vhost_prefix = arguments['--vhost-prefix']
statsd_port = arguments['--statsd-port']
statsd = statsd_cli.StatsdClient(port=int(statsd_port))
#send to statsd
for record in records:
#这里兼容一下微官网的，如果是wechat则特殊处理一下
if vhost_prefix == 'wechat':
metric_qps = socket.gethostname() + '.nginx.qps'
else:
metric_qps = socket.gethostname() + '.' + vhost_prefix + '_nginx.qps'
print(metric_qps)
statsd.increment(metric_qps)
if record.has_key('status'):
if vhost_prefix == 'wechat':
metric_status = socket.gethostname() + '.' + 'nginx.' + 'status_code.' + str(record['status'])
else:
metric_status = socket.gethostname() + '.' + vhost_prefix + '_nginx.status_code.' + str(record['status'])
statsd.increment(metric_status)
def build_source(access_log, arguments):
# constructing log source
if access_log == 'stdin':
lines = sys.stdin
elif arguments['--no-follow']:
lines = open(access_log)
else:
lines = follow(access_log)
return lines
def process(arguments):
access_log = arguments['--access-log']
log_format = arguments['--log-format']
if access_log is None and not sys.stdin.isatty():
# assume logs can be fetched directly from stdin when piped
access_log = 'stdin'
if access_log is None:
access_log, log_format = detect_log_config(arguments)
logging.info('access_log: %s', access_log)
logging.info('log_format: %s', log_format)
if access_log != 'stdin' and not os.path.exists(access_log):
error_exit('access log file "%s" does not exist' % access_log)
if arguments['info']:
print('nginx configuration file:\n ', detect_config_path())
print('access log file:\n ', access_log)
print('access log format:\n ', log_format)
print('available variables:\n ', ', '.join(sorted(extract_variables(log_format))))
return
source = build_source(access_log, arguments)
pattern = build_pattern(log_format)
#processor = build_processor(arguments)
processor = None
process_log(source, pattern, processor, arguments)
def main():
from docopt import docopt
args = docopt(__doc__, version='nginx_log_stat 0.02')
print(args)
log_level = logging.WARNING
if args['--verbose']:
log_level = logging.INFO
if args['--debug']:
log_level = logging.DEBUG
logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s')
logging.debug('arguments:\n%s', args)
try:
process(args)
except KeyboardInterrupt:
sys.exit(0)
if __name__ == '__main__':
logging.basicConfig(filename='nginx_log_stat.log', level=logging.DEBUG)
main()