def test_init_path_stats(): stats = TestablePerPathAccumulators(aggregation_depth=1) stats.start() cur_stats = stats.cur_stats() assert "writes" in cur_stats assert "/" in cur_stats["writes"] assert "writesBytes" in cur_stats assert "/" in cur_stats["writesBytes"] assert "reads" in cur_stats assert "/" in cur_stats["reads"] assert "readsBytes" in cur_stats assert "/" in cur_stats["readsBytes"] assert "total" in cur_stats assert "/writes" in cur_stats["total"] assert "/writeBytes" in cur_stats["total"] assert "/reads" in cur_stats["total"] assert "/readBytes" in cur_stats["total"] #add some traffic zkt = Sniffer(SnifferConfig()) zkt.add_request_handler(stats.handle_request) cur_stats = wait_for_stats(zkt, stats, "set_data", 1) #writes for / should stay 0 assert cur_stats["writes"]["/"] == 0 assert cur_stats["total"]["/writes"] == 20 stats.stop()
def test_per_path_stats(): stats = TestablePerPathAccumulators(aggregation_depth=1) stats.start() zkt = Sniffer(SnifferConfig()) zkt.add_request_handler(stats.handle_request) cur_stats = wait_for_stats(zkt, stats, "set_data", NUMBER_OF_REQUESTS_SET_DATA) assert cur_stats["writes"]["/load-testing"] == 20 assert cur_stats["SetDataRequest"]["/load-testing"] == 20 stats.stop()
def test_watches(): stats = TestablePerPathAccumulators(aggregation_depth=1) stats.start() zkt = Sniffer(SnifferConfig()) zkt.add_request_handler(stats.handle_request) cur_stats = wait_for_stats(zkt, stats, "getdata_watches", NUMBER_OF_REQUESTS_WATCHES) assert cur_stats["watches"]["/test"] == 2 assert cur_stats["GetDataRequest"]["/test"] == 2 assert cur_stats["GetChildrenRequest"]["/test"] == 2 stats.stop()
class EndpointsServer(HttpServer):
MAX_RESULTS = 10
def __init__(
self, iface, zkport, request_handler, reply_handler=None, event_handler=None):
HttpServer.__init__(self)
config = SnifferConfig(iface=iface)
config.zookeeper_port = zkport
config.update_filter()
self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)
self._sniffer.start()
def test_aggregation_depth():
stats = AccumulatedStats(StatsConfig(aggregation_depth=1))
zkt = Sniffer(SnifferConfig())
zkt.add_request_handler(stats.handle_request)
consume_packets('set_data', zkt)
assert stats.global_stats.ops_written == 20
assert stats.global_stats.by_op_counters[OpCodes.SETDATA] == 20
# Did aggregation work?
assert stats.by_path["/load-testing"].ops_written == 20
assert stats.by_path["/load-testing"].by_op_counters[OpCodes.SETDATA] == 20
def test_aggregation_depth():
datapoints = PerPathDatapoints()
zkt = Sniffer(SnifferConfig())
zkt.add_request_handler(datapoints.handle_request)
consume_packets('set_data', zkt)
d = datapoints.sum_minute()
assert d['global']['ops_written'] == 20
assert d['global']['by_op_counters']['SetDataRequest'] == 20
for i in range(0, 4):
assert d['paths']['/load-testing/%d' % (i)]['ops_written'] == 4
assert d['paths']['/load-testing/%d' % (i)]['by_op_counters']['SetDataRequest'] == 4
def test_per_path_stats_aggregated():
stats = TestablePerPathAccumulators(aggregation_depth=2)
stats.start()
zkt = Sniffer(SnifferConfig())
zkt.add_request_handler(stats.handle_request)
cur_stats = wait_for_stats(zkt, stats, "set_data", NUMBER_OF_REQUESTS_SET_DATA)
for i in range(0, 5):
assert cur_stats["writes"]["/load-testing/%d" % (i)] == 4
assert cur_stats["SetDataRequest"]["/load-testing/%d" % (i)] == 4
assert cur_stats["total"]["/writes"] == 20
assert cur_stats["total"]["/reads"] == 16
stats.stop()
class EndpointsServer(HttpServer):
MAX_RESULTS = 10
def __init__(
self, iface, zkport, request_handler, reply_handler=None, event_handler=None, start_sniffer=True):
config = SnifferConfig(iface=iface)
config.zookeeper_port = zkport
config.update_filter()
self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)
if start_sniffer: # pragma: no cover
self._sniffer.start()
super(EndpointsServer, self).__init__()
@property
def sniffer(self):
return self._sniffer
def get_sniffer(request_handler, reply_handler=None, event_handler=None):
config = SnifferConfig()
config.track_replies = True
sniffer = Sniffer(config)
sniffer.add_request_handler(request_handler)
if reply_handler:
sniffer.add_reply_handler(reply_handler)
if event_handler:
sniffer.add_event_handler(event_handler)
return sniffer
def main(_, options):
config = SnifferConfig(options.iface)
config.track_replies = True
config.zookeeper_port = options.zookeeper_port
config.max_queued_requests = options.max_queued_requests
config.client_port = options.client_port if options.client_port != 0 else config.client_port
if options.excluded_hosts and options.included_hosts:
sys.stderr.write(
"The flags --include-host and --exclude-host can't be mixed")
sys.exit(1)
if options.excluded_hosts:
config.excluded_ips += expand_hosts(options.excluded_hosts)
elif options.included_hosts:
config.included_ips += expand_hosts(options.included_hosts)
config.update_filter()
if options.include_pings:
config.include_pings()
loopback = options.iface in ["lo", "lo0"]
if options.unpaired:
p = UnpairedPrinter(options.colors, loopback)
else:
p = DefaultPrinter(options.colors, loopback)
p.start()
sniffer = Sniffer(config, p.request_handler, p.reply_handler,
p.event_handler)
sniffer.start()
try:
while True:
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
pass
sys.stdout.write("\033[0m")
sys.stdout.flush()
def main(_, options):
config = SnifferConfig(options.iface)
config.track_replies = True
config.zookeeper_port = options.zookeeper_port
config.max_queued_requests = options.max_queued_requests
config.client_port = options.client_port if options.client_port != 0 else config.client_port
if options.excluded_hosts and options.included_hosts:
sys.stderr.write("The flags --include-host and --exclude-host can't be mixed")
sys.exit(1)
if options.excluded_hosts:
config.excluded_ips += expand_hosts(options.excluded_hosts)
elif options.included_hosts:
config.included_ips += expand_hosts(options.included_hosts)
config.update_filter()
if options.include_pings:
config.include_pings()
config.dump_bad_packet = options.dump_bad_packet
loopback = options.iface in ["lo", "lo0"]
if options.unpaired:
p = UnpairedPrinter(options.colors, loopback)
else:
p = DefaultPrinter(options.colors, loopback)
p.start()
sniffer = Sniffer(config, p.request_handler, p.reply_handler, p.event_handler)
sniffer.start()
try:
while True:
time.sleep(60)
except (KeyboardInterrupt, SystemExit):
pass
sys.stdout.write("\033[0m")
sys.stdout.flush()
def get_sniffer(printer, with_pings=True):
config = SnifferConfig()
config.track_replies = True
if with_pings:
config.include_pings()
sniffer = Sniffer(config)
sniffer.add_request_handler(printer.request_handler)
sniffer.add_reply_handler(printer.reply_handler)
sniffer.add_event_handler(printer.event_handler)
return sniffer
def __init__(self,
iface,
zkport,
request_handler,
reply_handler=None,
event_handler=None,
start_sniffer=True,
sampling=1.0):
config = SnifferConfig(iface=iface)
config.zookeeper_port = zkport
config.update_filter()
config.sampling = sampling
self._sniffer = Sniffer(config, request_handler, reply_handler,
event_handler)
if start_sniffer: # pragma: no cover
self._sniffer.start()
super(EndpointsServer, self).__init__()
def test_setwatches():
requests = []
def handler(request):
if isinstance(request, SetWatchesRequest):
requests.append(request)
sniffer = Sniffer(SnifferConfig())
sniffer.add_request_handler(handler)
consume_packets('setwatches', sniffer)
assert len(requests) == 1
req = requests[0]
assert len(req.child) == 5
assert "/foo" in req.child
assert "/in" in req.child
assert "/zookeeper" in req.child
assert "/in/portland" in req.child
assert "/" in req.child
def test_setwatches():
requests = []
def handler(request):
if isinstance(request, SetWatchesRequest):
requests.append(request)
sniffer = Sniffer(SnifferConfig())
sniffer.add_request_handler(handler)
consume_packets('setwatches', sniffer)
assert len(requests) == 1
req = requests[0]
assert len(req.child) == 5
assert "/foo" in req.child
assert "/in" in req.child
assert "/zookeeper" in req.child
assert "/in/portland" in req.child
assert "/" in req.child
def __init__(
self, iface, zkport, request_handler, reply_handler=None, event_handler=None, start_sniffer=True):
config = SnifferConfig(iface=iface)
config.zookeeper_port = zkport
config.update_filter()
self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)
if start_sniffer: # pragma: no cover
self._sniffer.start()
super(EndpointsServer, self).__init__()
def get_sniffer(request_handler, reply_handler=None, event_handler=None):
config = SnifferConfig()
config.track_replies = True
sniffer = Sniffer(config)
sniffer.add_request_handler(request_handler)
if reply_handler:
sniffer.add_reply_handler(reply_handler)
if event_handler:
sniffer.add_event_handler(event_handler)
return sniffer
def get_sniffer(printer, with_pings=True):
config = SnifferConfig()
config.track_replies = True
if with_pings:
config.include_pings()
sniffer = Sniffer(config)
sniffer.add_request_handler(printer.request_handler)
sniffer.add_reply_handler(printer.reply_handler)
sniffer.add_event_handler(printer.event_handler)
return sniffer
def _test_requests_replies(pcap_name, request_cls, reply_cls, nreqs, nreps):
requests = []
replies = []
def handler(msg):
if isinstance(msg, request_cls):
requests.append(msg)
elif isinstance(msg, reply_cls):
replies.append(msg)
config = SnifferConfig()
config.track_replies = True
sniffer = Sniffer(config)
sniffer.add_request_handler(handler)
sniffer.add_reply_handler(handler)
consume_packets(pcap_name, sniffer)
assert len(requests) == nreqs
assert len(replies) == nreps
sniffer.stop()
def _test_requests_replies(pcap_name, request_cls, reply_cls, nreqs, nreps):
requests = []
replies = []
def handler(msg):
if isinstance(msg, request_cls):
requests.append(msg)
elif isinstance(msg, reply_cls):
replies.append(msg)
config = SnifferConfig()
config.track_replies = True
sniffer = Sniffer(config)
sniffer.add_request_handler(handler)
sniffer.add_reply_handler(handler)
consume_packets(pcap_name, sniffer)
assert len(requests) == nreqs
assert len(replies) == nreps
sniffer.stop()
def default_zkt():
stats = AccumulatedStats(StatsConfig())
zkt = Sniffer(SnifferConfig())
zkt.add_request_handler(stats.handle_request)
return (zkt, stats)
def main(_, options):
if options.version:
sys.stdout.write("%s\n" % __version__)
sys.exit(0)
config = SnifferConfig(options.iface)
config.track_replies = True
config.zookeeper_port = options.zookeeper_port
config.max_queued_requests = options.max_queued_requests
config.client_port = options.client_port if options.client_port != 0 else config.client_port
if options.excluded_hosts and options.included_hosts:
sys.stderr.write("The flags --include-host and --exclude-host can't be mixed.\n")
sys.exit(1)
if options.excluded_hosts:
config.excluded_ips += expand_hosts(options.excluded_hosts)
elif options.included_hosts:
config.included_ips += expand_hosts(options.included_hosts)
config.update_filter()
if options.include_pings:
config.include_pings()
config.dump_bad_packet = options.dump_bad_packet
loopback = options.iface in ["lo", "lo0"]
if options.count_requests > 0 and options.measure_latency > 0:
sys.stderr.write("The flags --count-requests and --measure-latency can't be mixed.\n")
sys.exit(1)
if options.count_requests > 0:
validate_group_by(options.group_by)
validate_aggregation_depth(options.aggregation_depth)
p = CountPrinter(options.count_requests, options.group_by, loopback, options.aggregation_depth)
elif options.measure_latency > 0:
validate_group_by(options.group_by)
validate_aggregation_depth(options.aggregation_depth)
validate_sort_by(options.sort_by)
p = LatencyPrinter(
options.measure_latency, options.group_by, loopback, options.aggregation_depth,
options.sort_by)
elif options.unpaired:
p = UnpairedPrinter(options.colors, loopback)
else:
p = DefaultPrinter(options.colors, loopback)
p.start()
sniffer = Sniffer(
config,
p.request_handler,
p.reply_handler,
p.event_handler,
error_to_stderr=True
)
sniffer.start()
try:
while p.isAlive():
time.sleep(0.5)
except (KeyboardInterrupt, SystemExit):
p.cancel()
# shutdown sniffer
sniffer.stop()
while sniffer.isAlive():
time.sleep(0.001)
try:
sys.stdout.write("\033[0m")
sys.stdout.flush()
except IOError: pass
def setUp(self): self.zkt = Sniffer(SnifferConfig())
class TestSniffer(TestCase):
def setUp(self):
self.zkt = Sniffer(SnifferConfig())
@mock.patch('os.kill', spec=os.kill)
@mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
def test_run_socket_error(self, mock_sniff, mock_kill):
mock_sniff.side_effect = socket.error
self.zkt.run()
mock_sniff.assert_called_once_with(
filter=self.zkt.config.filter,
store=0,
prn=self.zkt.handle_packet,
iface=self.zkt.config.iface,
stop_filter=self.zkt.wants_stop
)
mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)
@mock.patch('os.kill', spec=os.kill)
@mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
def test_run(self, mock_sniff, mock_kill):
self.zkt.run()
mock_sniff.assert_called_once_with(
filter=self.zkt.config.filter,
store=0,
prn=self.zkt.handle_packet,
iface=self.zkt.config.iface,
stop_filter=self.zkt.wants_stop
)
mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)
def test_exclude(self):
filter_text = 'port 2181'
excluded_ip = '8.8.8.8'
assert self.zkt.config.filter == filter_text
self.zkt.config.excluded_ips = [excluded_ip]
assert self.zkt.config.filter == filter_text
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and host not %s' % (filter_text, excluded_ip)
excluded_ip_two = '8.8.4.4'
excluded_ip_three = '172.24.6.11'
self.zkt.config.excluded_ips = [excluded_ip, excluded_ip_two, excluded_ip_three]
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and host not %s and host not %s and host not %s' % (
filter_text, excluded_ip, excluded_ip_two, excluded_ip_three)
def test_include(self):
filter_text = 'port 2181'
included_ip = '8.8.8.8'
assert self.zkt.config.filter == filter_text
self.zkt.config.included_ips = [included_ip]
assert self.zkt.config.filter == filter_text
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and (host %s)' % (filter_text, included_ip)
included_ip_two = '8.8.4.4'
included_ip_three = '172.24.6.11'
self.zkt.config.included_ips = [included_ip, included_ip_two, included_ip_three]
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and (host %s or host %s or host %s)' % (
filter_text, included_ip, included_ip_two, included_ip_three)
def default_zkt(): stats = AccumulatedStats(StatsConfig()) zkt = Sniffer(SnifferConfig()) zkt.add_request_handler(stats.handle_request) return (zkt, stats)
def default_sniffer(aggregation_depth=1): stats = PerPathStatsAccumulator(aggregation_depth=aggregation_depth) sniffer = Sniffer(SnifferConfig()) sniffer.add_request_handler(stats.update_request_stats) return (sniffer, stats)
class TestSniffer(TestCase):
def setUp(self):
self.zkt = Sniffer(SnifferConfig())
@mock.patch('os.kill', spec=os.kill)
@mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
def test_run_socket_error(self, mock_sniff, mock_kill):
mock_sniff.side_effect = socket.error
self.zkt.run()
mock_sniff.assert_called_once_with(filter=self.zkt.config.filter,
store=0,
prn=self.zkt.handle_packet,
iface=self.zkt.config.iface)
mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)
@mock.patch('os.kill', spec=os.kill)
@mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
def test_run(self, mock_sniff, mock_kill):
self.zkt.run()
mock_sniff.assert_called_once_with(filter=self.zkt.config.filter,
store=0,
prn=self.zkt.handle_packet,
iface=self.zkt.config.iface)
mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)
def test_exclude(self):
filter_text = 'port 2181'
excluded_ip = '8.8.8.8'
assert self.zkt.config.filter == filter_text
self.zkt.config.excluded_ips = [excluded_ip]
assert self.zkt.config.filter == filter_text
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and host not %s' % (filter_text,
excluded_ip)
excluded_ip_two = '8.8.4.4'
excluded_ip_three = '172.24.6.11'
self.zkt.config.excluded_ips = [
excluded_ip, excluded_ip_two, excluded_ip_three
]
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and host not %s and host not %s and host not %s' % (
filter_text, excluded_ip, excluded_ip_two, excluded_ip_three)
def test_include(self):
filter_text = 'port 2181'
included_ip = '8.8.8.8'
assert self.zkt.config.filter == filter_text
self.zkt.config.included_ips = [included_ip]
assert self.zkt.config.filter == filter_text
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and (host %s)' % (filter_text,
included_ip)
included_ip_two = '8.8.4.4'
included_ip_three = '172.24.6.11'
self.zkt.config.included_ips = [
included_ip, included_ip_two, included_ip_three
]
self.zkt.config.update_filter()
assert self.zkt.config.filter == '%s and (host %s or host %s or host %s)' % (
filter_text, included_ip, included_ip_two, included_ip_three)
def main(_, options):
if options.version:
sys.stdout.write("%s\n" % __version__)
sys.exit(0)
config = SnifferConfig(options.iface)
config.track_replies = True
config.zookeeper_port = options.zookeeper_port
config.max_queued_requests = options.max_queued_requests
config.client_port = options.client_port if options.client_port != 0 else config.client_port
if options.excluded_hosts and options.included_hosts:
sys.stderr.write(
"The flags --include-host and --exclude-host can't be mixed.\n")
sys.exit(1)
if options.excluded_hosts:
config.excluded_ips += expand_hosts(options.excluded_hosts)
elif options.included_hosts:
config.included_ips += expand_hosts(options.included_hosts)
config.update_filter()
if options.include_pings:
config.include_pings()
config.dump_bad_packet = options.dump_bad_packet
loopback = options.iface in ["lo", "lo0"]
if options.count_requests > 0 and options.measure_latency > 0:
sys.stderr.write(
"The flags --count-requests and --measure-latency can't be mixed.\n"
)
sys.exit(1)
if options.count_requests > 0:
validate_group_by(options.group_by)
validate_aggregation_depth(options.aggregation_depth)
p = CountPrinter(options.count_requests, options.group_by, loopback,
options.aggregation_depth)
elif options.measure_latency > 0:
validate_group_by(options.group_by)
validate_aggregation_depth(options.aggregation_depth)
validate_sort_by(options.sort_by)
p = LatencyPrinter(options.measure_latency, options.group_by, loopback,
options.aggregation_depth, options.sort_by)
elif options.unpaired:
p = UnpairedPrinter(options.colors, loopback)
else:
p = DefaultPrinter(options.colors, loopback)
p.start()
sniffer = Sniffer(config,
p.request_handler,
p.reply_handler,
p.event_handler,
error_to_stderr=True)
sniffer.start()
try:
while p.isAlive():
time.sleep(0.5)
except (KeyboardInterrupt, SystemExit):
p.cancel()
# shutdown sniffer
sniffer.stop()
while sniffer.isAlive():
time.sleep(0.001)
try:
sys.stdout.write("\033[0m")
sys.stdout.flush()
except IOError:
pass
def default_sniffer(aggregation_depth=1): stats = PerPathStatsAccumulator(aggregation_depth=aggregation_depth) sniffer = Sniffer(SnifferConfig()) sniffer.add_request_handler(stats.update_request_stats) return (sniffer, stats)
def setUp(self):
self.zkt = Sniffer(SnifferConfig())