def test_both(self): q = OpenTSDBQuery(self.payload3) self.assertFalse(self.query_no_tags_filters.check(q).is_ok()) q = OpenTSDBQuery(self.payload4) self.assertFalse(self.query_no_tags_filters.check(q).is_ok())
def test_above(self): current_time = int(round(time.time())) q = OpenTSDBQuery(self.payload2) q.set_stats({'timestamp': current_time - 31}) self.assertTrue(self.exceed_frequency.check(q).is_ok())
def test_too_soon(self): current_time = int(round(time.time())) q = OpenTSDBQuery(self.payload1) q.set_stats({'timestamp': current_time - 30}) self.assertFalse(self.exceed_frequency.check(q).is_ok())
def test_below(self): current_time = int(round(time.time())) q = OpenTSDBQuery(self.payload1) q.set_stats({'duration': 1.5, 'timestamp': current_time - 1}) self.assertTrue(self.exceed_time_limit.check(q).is_ok())
def test_invalid_queries(self): p.safe_mode = False with self.assertRaisesRegexp(Exception, 'Invalid OpenTSDB query'): p.check(OpenTSDBQuery('{}')) with self.assertRaisesRegexp(Exception, 'Invalid OpenTSDB query'): p.check(OpenTSDBQuery('{"start": ""}'))
def test_old_relative(self): self.assertFalse( self.query_old_data.check(OpenTSDBQuery(self.payload2)).is_ok()) self.assertFalse( self.query_old_data.check(OpenTSDBQuery(self.payload3)).is_ok()) self.assertTrue( self.query_old_data.check(OpenTSDBQuery(self.payload4)).is_ok())
def test_safe_mode(self): p.blockedlist = ["^releases$", "^mymetric"] p.safe_mode = True self.assertFalse(p.check(OpenTSDBQuery(self.payload4)).is_ok()) p.safe_mode = False self.assertFalse(p.check(OpenTSDBQuery(self.payload4)).is_ok())
def test_blockedlist(self): p.blockedlist = ["^releases$", "^mymetric\.", ".*java.*boot.*version.*"] self.assertFalse(p.check(OpenTSDBQuery(self.payload1)).is_ok()) self.assertTrue(p.check(OpenTSDBQuery(self.payload2)).is_ok()) self.assertTrue(p.check(OpenTSDBQuery(self.payload3)).is_ok()) p.blockedlist = [] self.assertTrue(p.check(OpenTSDBQuery(self.payload1)).is_ok())
def do_POST(self): length = int(self.headers['Content-Length']) post_data = self.rfile.read(length) self.headers['Host'] = self.backend_netloc self.filter_headers(self.headers) # Deny put requests if self.path == "/api/put": logging.warning("OpenTSDBQuery blocked. Reason: %s", "/api/put not allowed") self.send_error(httplib.FORBIDDEN, "/api/put not allowed") return # Process query requests if self.path == "/api/query": self.tsdb_query = OpenTSDBQuery(post_data) self.headers['X-Protector'] = self.tsdb_query.get_id() # Check the payload against the Protector rule set result = self.protector.check(self.tsdb_query) if not result.is_ok(): self.protector.REQUESTS_BLOCKED.labels( self.protector.safe_mode, result.value["rule"]).inc() if not self.protector.safe_mode: logging.warning("OpenTSDBQuery blocked: %s. Reason: %s", self.tsdb_query.get_id(), result.value["msg"]) self.send_error(httplib.FORBIDDEN, result.value["msg"]) return post_data = self.tsdb_query.to_json() self.headers['Content-Length'] = str(len(post_data)) status = self._handle_request(self.scheme, self.backend_netloc, self.path, self.headers, body=post_data, method="POST") #['method', 'path', 'return_code'] self.protector.REQUESTS_COUNT.labels('POST', self.path, status).inc() self.finish() self.connection.close()
def test_save_stats_timeout(self): q3 = OpenTSDBQuery(self.payload3) interval = int( (q3.get_end_timestamp() - q3.get_start_timestamp()) / 60) p.save_stats(q3, None, 20, True) j = stats["{}_{}".format(q3.get_id(), 'stats')] oj = json.loads(j) # _stats self.assertTrue(oj['timeout']) self.assertEqual(oj['duration'], 20) self.assertDictEqual(oj['summary'], {}) # _query qx = q["{}_{}".format(q3.get_id(), 'query')] qs = json.dumps(json.loads(qx), sort_keys=True) self.assertEqual(qs, q3.to_json(True)) # _interval ki = meta["{}_{}".format(q3.get_id(), interval)] t = ki['timestamp'] self.assertEqual(ki['timeout_counter'], 1) self.assertEqual(ki['total_counter'], 1) self.assertEqual(t, ki['timeout_last']) self.assertEqual(t, ki['first_occurrence'])
def test_adaptive(self): current_time = int(round(time.time())) q = OpenTSDBQuery(self.payload1) q.set_stats({'duration': 10, 'timestamp': current_time - 16}) self.assertTrue(self.adaptive_time_limit.check(q).is_ok()) q.set_stats({'duration': 10, 'timestamp': current_time - 15}) self.assertFalse(self.adaptive_time_limit.check(q).is_ok())
def test_above(self): current_time = int(round(time.time())) q = OpenTSDBQuery(self.payload2) q.set_stats({'duration': 20, 'timestamp': current_time - 310}) self.assertTrue(self.exceed_time_limit.check(q).is_ok()) q.set_stats({'duration': 20, 'timestamp': current_time - 210}) self.assertFalse(self.exceed_time_limit.check(q).is_ok())
def test_too_many(self): q = OpenTSDBQuery(self.payload1) q.set_stats({'emittedDPs': 10001}) self.assertFalse(self.too_many_datapoints.check(q).is_ok())
def test_none(self): q = OpenTSDBQuery(self.payload3) self.assertTrue(self.exceed_time_limit.check(q).is_ok())
def test_above(self): q = OpenTSDBQuery(self.payload2) q.set_stats({'duration': 20}) self.assertFalse(self.exceed_time_limit.check(q).is_ok())
def test_no_filters(self): q = OpenTSDBQuery(self.payload2) self.assertTrue(self.query_no_tags_filters.check(q).is_ok())
def test_old_absolute(self): self.assertFalse( self.query_old_data.check(OpenTSDBQuery(self.payload1)).is_ok())
def test_none(self): q = OpenTSDBQuery(self.payload3) self.assertTrue(self.exceed_frequency.check(q).is_ok())
def test_agg(self): q = OpenTSDBQuery(self.payload2) self.assertTrue(self.query_no_aggregator.check(q).is_ok())
class ProxyRequestHandler(BaseHTTPRequestHandler): protector = None backend_address = None timeout = None def __init__(self, *args, **kwargs): self.http_request = HTTPRequest() self.tsdb_query = None # Address to time series backend backend_host, backend_port = self.backend_address self.backend_netloc = "{}:{}".format(backend_host, backend_port) self.scheme = "http" self.path = None self.connection = None self.rfile = None self.wfile = None self.close_connection = 0 BaseHTTPRequestHandler.__init__(self, *args, **kwargs) def log_error(self, log_format, *args): # Suppress "Request timed out: timeout('timed out',)" # if isinstance(args[0], socket.timeout): # logging.error("{}".format(traceback.format_exc())) # logging.error(pprint.pprint(args)) self.log_message(log_format, *args) def log_message(self, format, *args): """Log an arbitrary message. This is used by all other logging functions. Override it if you have specific logging wishes. The first argument, FORMAT, is a format string for the message to be logged. If the format string contains any % escapes requiring parameters, they should be specified as subsequent arguments (it's just like printf!). The client ip address and current date/time are prefixed to every message. """ xff = '-' xgo = '-' ua = '-' try: xff = self.headers.getheader('X-Forwarded-For', '-') xgo = self.headers.getheader('X-Grafana-Org-Id', '-') ua = self.headers.getheader('User-Agent', '-') except AttributeError as e: requestline = getattr(self, 'raw_requestline', '-') logging.warning( "Malformed/missing request header. Requestline: %s" % requestline) logging.info( "%s - - [%s] %s [X-Forwarded-For: %s, X-Grafana-Org-Id: %s, User-Agent: %s]" % (self.client_address[0], self.log_date_time_string(), format % args, xff, xgo, ua)) def do_GET(self): top = re.match("^/top/(duration|dps)$", self.path) if self.path == "/metrics": data = generate_latest() self.send_response(httplib.OK) self.send_header("Content-Type", CONTENT_TYPE_LATEST) self.send_header("Content-Length", str(len(data))) self.send_header('Connection', 'close') self.end_headers() self.wfile.write(data) elif top: data = self.protector.get_top(top.group(1)) self.send_response(httplib.OK) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(data))) self.send_header('Connection', 'close') self.end_headers() self.wfile.write(data) else: self.headers['Host'] = self.backend_netloc self.filter_headers(self.headers) self._handle_request(self.scheme, self.backend_netloc, self.path, self.headers) self.finish() self.connection.close() def do_POST(self): length = int(self.headers['Content-Length']) post_data = self.rfile.read(length) self.headers['Host'] = self.backend_netloc self.filter_headers(self.headers) # Deny put requests if self.path == "/api/put": logging.warning("OpenTSDBQuery blocked. Reason: %s", "/api/put not allowed") self.send_error(httplib.FORBIDDEN, "/api/put not allowed") return # Process query requests if self.path == "/api/query": self.tsdb_query = OpenTSDBQuery(post_data) self.headers['X-Protector'] = self.tsdb_query.get_id() # Check the payload against the Protector rule set result = self.protector.check(self.tsdb_query) if not result.is_ok(): self.protector.REQUESTS_BLOCKED.labels( self.protector.safe_mode, result.value["rule"]).inc() if not self.protector.safe_mode: logging.warning("OpenTSDBQuery blocked: %s. Reason: %s", self.tsdb_query.get_id(), result.value["msg"]) self.send_error(httplib.FORBIDDEN, result.value["msg"]) return post_data = self.tsdb_query.to_json() self.headers['Content-Length'] = str(len(post_data)) status = self._handle_request(self.scheme, self.backend_netloc, self.path, self.headers, body=post_data, method="POST") #['method', 'path', 'return_code'] self.protector.REQUESTS_COUNT.labels('POST', self.path, status).inc() self.finish() self.connection.close() def send_error(self, code, message=None): """ Send and log plain text error reply. :param code: :param message: """ message = message.strip() self.log_error("code %d, message: %s", code, message) self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header('Connection', 'close') self.end_headers() if message: # Grafana style j = {'message': message, 'error': message} self.wfile.write(json.dumps(j)) def _handle_request(self, scheme, netloc, path, headers, body=None, method="GET"): """ Run the actual request """ backend_url = "{}://{}{}".format(scheme, netloc, path) startTime = time.time() try: response = self.http_request.request(backend_url, self.timeout, method=method, body=body, headers=dict(headers)) respTime = time.time() duration = respTime - startTime self.protector.TSDB_REQUEST_LATENCY.labels( response.status, path, method).observe(duration) self._return_response(response, method, duration) return response.status except socket.timeout, e: respTime = time.time() duration = respTime - startTime if method == "POST": self.protector.save_stats(self.tsdb_query, None, duration, True) self.protector.TSDB_REQUEST_LATENCY.labels( httplib.GATEWAY_TIMEOUT, path, method).observe(duration) self.send_error( httplib.GATEWAY_TIMEOUT, "Query timed out. Configured timeout: {}s".format( self.timeout)) return httplib.GATEWAY_TIMEOUT except Exception as e: respTime = time.time() duration = respTime - startTime err = "Invalid response from backend: '{}'".format(e) logging.debug(err) self.protector.TSDB_REQUEST_LATENCY.labels( httplib.BAD_GATEWAY, path, method).observe(duration) self.send_error(httplib.BAD_GATEWAY, err) return httplib.BAD_GATEWAY
def test_guard(self): # Test rules loading guard = Guard(self.config['rules']) q = OpenTSDBQuery(self.payload) self.assertTrue(guard.is_allowed(q))
def test_none(self): q = OpenTSDBQuery(self.payload3) self.assertTrue(self.too_many_datapoints.check(q).is_ok())
def test_less(self): q = OpenTSDBQuery(self.payload2) q.set_stats({'emittedDPs': 999}) self.assertTrue(self.too_many_datapoints.check(q).is_ok())
def test_below(self): q = OpenTSDBQuery(self.payload1) q.set_stats({'duration': 1.5}) self.assertTrue(self.exceed_time_limit.check(q).is_ok())