def test_save_stats_timeout(self):
q3 = OpenTSDBQuery(self.payload3)
interval = int(
(q3.get_end_timestamp() - q3.get_start_timestamp()) / 60)
p.save_stats(q3, None, 20, True)
j = stats["{}_{}".format(q3.get_id(), 'stats')]
oj = json.loads(j)
# _stats
self.assertTrue(oj['timeout'])
self.assertEqual(oj['duration'], 20)
self.assertDictEqual(oj['summary'], {})
# _query
qx = q["{}_{}".format(q3.get_id(), 'query')]
qs = json.dumps(json.loads(qx), sort_keys=True)
self.assertEqual(qs, q3.to_json(True))
# _interval
ki = meta["{}_{}".format(q3.get_id(), interval)]
t = ki['timestamp']
self.assertEqual(ki['timeout_counter'], 1)
self.assertEqual(ki['total_counter'], 1)
self.assertEqual(t, ki['timeout_last'])
self.assertEqual(t, ki['first_occurrence'])
class ProxyRequestHandler(BaseHTTPRequestHandler):
protector = None
backend_address = None
timeout = None
def __init__(self, *args, **kwargs):
self.http_request = HTTPRequest()
self.tsdb_query = None
# Address to time series backend
backend_host, backend_port = self.backend_address
self.backend_netloc = "{}:{}".format(backend_host, backend_port)
self.scheme = "http"
self.path = None
self.connection = None
self.rfile = None
self.wfile = None
self.close_connection = 0
BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
def log_error(self, log_format, *args):
# Suppress "Request timed out: timeout('timed out',)"
# if isinstance(args[0], socket.timeout):
# logging.error("{}".format(traceback.format_exc()))
# logging.error(pprint.pprint(args))
self.log_message(log_format, *args)
def log_message(self, format, *args):
"""Log an arbitrary message.
This is used by all other logging functions. Override
it if you have specific logging wishes.
The first argument, FORMAT, is a format string for the
message to be logged. If the format string contains
any % escapes requiring parameters, they should be
specified as subsequent arguments (it's just like
printf!).
The client ip address and current date/time are prefixed to every
message.
"""
xff = '-'
xgo = '-'
ua = '-'
try:
xff = self.headers.getheader('X-Forwarded-For', '-')
xgo = self.headers.getheader('X-Grafana-Org-Id', '-')
ua = self.headers.getheader('User-Agent', '-')
except AttributeError as e:
requestline = getattr(self, 'raw_requestline', '-')
logging.warning(
"Malformed/missing request header. Requestline: %s" %
requestline)
logging.info(
"%s - - [%s] %s [X-Forwarded-For: %s, X-Grafana-Org-Id: %s, User-Agent: %s]"
% (self.client_address[0], self.log_date_time_string(),
format % args, xff, xgo, ua))
def do_GET(self):
top = re.match("^/top/(duration|dps)$", self.path)
if self.path == "/metrics":
data = generate_latest()
self.send_response(httplib.OK)
self.send_header("Content-Type", CONTENT_TYPE_LATEST)
self.send_header("Content-Length", str(len(data)))
self.send_header('Connection', 'close')
self.end_headers()
self.wfile.write(data)
elif top:
data = self.protector.get_top(top.group(1))
self.send_response(httplib.OK)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(data)))
self.send_header('Connection', 'close')
self.end_headers()
self.wfile.write(data)
else:
self.headers['Host'] = self.backend_netloc
self.filter_headers(self.headers)
self._handle_request(self.scheme, self.backend_netloc, self.path,
self.headers)
self.finish()
self.connection.close()
def do_POST(self):
length = int(self.headers['Content-Length'])
post_data = self.rfile.read(length)
self.headers['Host'] = self.backend_netloc
self.filter_headers(self.headers)
# Deny put requests
if self.path == "/api/put":
logging.warning("OpenTSDBQuery blocked. Reason: %s",
"/api/put not allowed")
self.send_error(httplib.FORBIDDEN, "/api/put not allowed")
return
# Process query requests
if self.path == "/api/query":
self.tsdb_query = OpenTSDBQuery(post_data)
self.headers['X-Protector'] = self.tsdb_query.get_id()
# Check the payload against the Protector rule set
result = self.protector.check(self.tsdb_query)
if not result.is_ok():
self.protector.REQUESTS_BLOCKED.labels(
self.protector.safe_mode, result.value["rule"]).inc()
if not self.protector.safe_mode:
logging.warning("OpenTSDBQuery blocked: %s. Reason: %s",
self.tsdb_query.get_id(),
result.value["msg"])
self.send_error(httplib.FORBIDDEN, result.value["msg"])
return
post_data = self.tsdb_query.to_json()
self.headers['Content-Length'] = str(len(post_data))
status = self._handle_request(self.scheme,
self.backend_netloc,
self.path,
self.headers,
body=post_data,
method="POST")
#['method', 'path', 'return_code']
self.protector.REQUESTS_COUNT.labels('POST', self.path, status).inc()
self.finish()
self.connection.close()
def send_error(self, code, message=None):
"""
Send and log plain text error reply.
:param code:
:param message:
"""
message = message.strip()
self.log_error("code %d, message: %s", code, message)
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header('Connection', 'close')
self.end_headers()
if message:
# Grafana style
j = {'message': message, 'error': message}
self.wfile.write(json.dumps(j))
def _handle_request(self,
scheme,
netloc,
path,
headers,
body=None,
method="GET"):
"""
Run the actual request
"""
backend_url = "{}://{}{}".format(scheme, netloc, path)
startTime = time.time()
try:
response = self.http_request.request(backend_url,
self.timeout,
method=method,
body=body,
headers=dict(headers))
respTime = time.time()
duration = respTime - startTime
self.protector.TSDB_REQUEST_LATENCY.labels(
response.status, path, method).observe(duration)
self._return_response(response, method, duration)
return response.status
except socket.timeout, e:
respTime = time.time()
duration = respTime - startTime
if method == "POST":
self.protector.save_stats(self.tsdb_query, None, duration,
True)
self.protector.TSDB_REQUEST_LATENCY.labels(
httplib.GATEWAY_TIMEOUT, path, method).observe(duration)
self.send_error(
httplib.GATEWAY_TIMEOUT,
"Query timed out. Configured timeout: {}s".format(
self.timeout))
return httplib.GATEWAY_TIMEOUT
except Exception as e:
respTime = time.time()
duration = respTime - startTime
err = "Invalid response from backend: '{}'".format(e)
logging.debug(err)
self.protector.TSDB_REQUEST_LATENCY.labels(
httplib.BAD_GATEWAY, path, method).observe(duration)
self.send_error(httplib.BAD_GATEWAY, err)
return httplib.BAD_GATEWAY