def test_init_path_stats():
  stats = TestablePerPathAccumulators(aggregation_depth=1)
  stats.start()
  cur_stats = stats.cur_stats()
  assert "writes" in cur_stats
  assert "/" in cur_stats["writes"]
  assert "writesBytes" in cur_stats
  assert "/" in cur_stats["writesBytes"]
  assert "reads" in cur_stats
  assert "/" in cur_stats["reads"]
  assert "readsBytes" in cur_stats
  assert "/" in cur_stats["readsBytes"]
  assert "total" in cur_stats
  assert "/writes" in cur_stats["total"]
  assert "/writeBytes" in cur_stats["total"]
  assert "/reads" in cur_stats["total"]
  assert "/readBytes" in cur_stats["total"]

  #add some traffic
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)
  cur_stats = wait_for_stats(zkt, stats, "set_data", 1)

  #writes for / should stay 0
  assert cur_stats["writes"]["/"] == 0
  assert cur_stats["total"]["/writes"] == 20

  stats.stop()
def test_per_path_stats():
  stats = TestablePerPathAccumulators(aggregation_depth=1)
  stats.start()
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)

  cur_stats = wait_for_stats(zkt, stats, "set_data", NUMBER_OF_REQUESTS_SET_DATA)

  assert cur_stats["writes"]["/load-testing"] == 20
  assert cur_stats["SetDataRequest"]["/load-testing"] == 20

  stats.stop()
def test_watches():
  stats = TestablePerPathAccumulators(aggregation_depth=1)
  stats.start()
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)

  cur_stats = wait_for_stats(zkt, stats, "getdata_watches", NUMBER_OF_REQUESTS_WATCHES)

  assert cur_stats["watches"]["/test"] == 2
  assert cur_stats["GetDataRequest"]["/test"] == 2
  assert cur_stats["GetChildrenRequest"]["/test"] == 2

  stats.stop()
class EndpointsServer(HttpServer):
  MAX_RESULTS = 10

  def __init__(
      self, iface, zkport, request_handler, reply_handler=None, event_handler=None):
    HttpServer.__init__(self)

    config = SnifferConfig(iface=iface)
    config.zookeeper_port = zkport
    config.update_filter()

    self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)
    self._sniffer.start()
def test_aggregation_depth():
  stats = AccumulatedStats(StatsConfig(aggregation_depth=1))
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)

  consume_packets('set_data', zkt)

  assert stats.global_stats.ops_written == 20
  assert stats.global_stats.by_op_counters[OpCodes.SETDATA] == 20

  # Did aggregation work?
  assert stats.by_path["/load-testing"].ops_written == 20
  assert stats.by_path["/load-testing"].by_op_counters[OpCodes.SETDATA] == 20
def test_aggregation_depth():
  datapoints = PerPathDatapoints()

  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(datapoints.handle_request)

  consume_packets('set_data', zkt)

  d = datapoints.sum_minute()

  assert d['global']['ops_written'] == 20
  assert d['global']['by_op_counters']['SetDataRequest'] == 20

  for i in range(0, 4):
    assert d['paths']['/load-testing/%d' % (i)]['ops_written'] == 4
    assert d['paths']['/load-testing/%d' % (i)]['by_op_counters']['SetDataRequest'] == 4
def test_per_path_stats_aggregated():
  stats = TestablePerPathAccumulators(aggregation_depth=2)
  stats.start()
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)

  cur_stats = wait_for_stats(zkt, stats, "set_data", NUMBER_OF_REQUESTS_SET_DATA)

  for i in range(0, 5):
    assert cur_stats["writes"]["/load-testing/%d" % (i)] == 4
    assert cur_stats["SetDataRequest"]["/load-testing/%d" % (i)] == 4

  assert cur_stats["total"]["/writes"] == 20
  assert cur_stats["total"]["/reads"] == 16

  stats.stop()
Exemple #8
0
class EndpointsServer(HttpServer):
  MAX_RESULTS = 10

  def __init__(
      self, iface, zkport, request_handler, reply_handler=None, event_handler=None, start_sniffer=True):
    config = SnifferConfig(iface=iface)
    config.zookeeper_port = zkport
    config.update_filter()

    self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)

    if start_sniffer:  # pragma: no cover
      self._sniffer.start()

    super(EndpointsServer, self).__init__()

  @property
  def sniffer(self):
    return self._sniffer
Exemple #9
0
def get_sniffer(request_handler, reply_handler=None, event_handler=None):
  config = SnifferConfig()
  config.track_replies = True

  sniffer = Sniffer(config)
  sniffer.add_request_handler(request_handler)
  if reply_handler:
    sniffer.add_reply_handler(reply_handler)
  if event_handler:
    sniffer.add_event_handler(event_handler)

  return sniffer
Exemple #10
0
def main(_, options):
    config = SnifferConfig(options.iface)
    config.track_replies = True
    config.zookeeper_port = options.zookeeper_port
    config.max_queued_requests = options.max_queued_requests
    config.client_port = options.client_port if options.client_port != 0 else config.client_port

    if options.excluded_hosts and options.included_hosts:
        sys.stderr.write(
            "The flags --include-host and --exclude-host can't be mixed")
        sys.exit(1)

    if options.excluded_hosts:
        config.excluded_ips += expand_hosts(options.excluded_hosts)
    elif options.included_hosts:
        config.included_ips += expand_hosts(options.included_hosts)

    config.update_filter()

    if options.include_pings:
        config.include_pings()

    loopback = options.iface in ["lo", "lo0"]

    if options.unpaired:
        p = UnpairedPrinter(options.colors, loopback)
    else:
        p = DefaultPrinter(options.colors, loopback)
    p.start()

    sniffer = Sniffer(config, p.request_handler, p.reply_handler,
                      p.event_handler)
    sniffer.start()

    try:
        while True:
            time.sleep(60)
    except (KeyboardInterrupt, SystemExit):
        pass

    sys.stdout.write("\033[0m")
    sys.stdout.flush()
Exemple #11
0
def main(_, options):
  config = SnifferConfig(options.iface)
  config.track_replies = True
  config.zookeeper_port = options.zookeeper_port
  config.max_queued_requests = options.max_queued_requests
  config.client_port = options.client_port if options.client_port != 0 else config.client_port

  if options.excluded_hosts and options.included_hosts:
    sys.stderr.write("The flags --include-host and --exclude-host can't be mixed")
    sys.exit(1)

  if options.excluded_hosts:
    config.excluded_ips += expand_hosts(options.excluded_hosts)
  elif options.included_hosts:
    config.included_ips += expand_hosts(options.included_hosts)

  config.update_filter()

  if options.include_pings:
    config.include_pings()

  config.dump_bad_packet = options.dump_bad_packet

  loopback = options.iface in ["lo", "lo0"]

  if options.unpaired:
    p = UnpairedPrinter(options.colors, loopback)
  else:
    p = DefaultPrinter(options.colors, loopback)
  p.start()

  sniffer = Sniffer(config, p.request_handler, p.reply_handler, p.event_handler)
  sniffer.start()

  try:
    while True:
      time.sleep(60)
  except (KeyboardInterrupt, SystemExit):
    pass

  sys.stdout.write("\033[0m")
  sys.stdout.flush()
def get_sniffer(printer, with_pings=True):
    config = SnifferConfig()
    config.track_replies = True

    if with_pings:
        config.include_pings()

    sniffer = Sniffer(config)
    sniffer.add_request_handler(printer.request_handler)
    sniffer.add_reply_handler(printer.reply_handler)
    sniffer.add_event_handler(printer.event_handler)

    return sniffer
    def __init__(self,
                 iface,
                 zkport,
                 request_handler,
                 reply_handler=None,
                 event_handler=None,
                 start_sniffer=True,
                 sampling=1.0):
        config = SnifferConfig(iface=iface)
        config.zookeeper_port = zkport
        config.update_filter()
        config.sampling = sampling

        self._sniffer = Sniffer(config, request_handler, reply_handler,
                                event_handler)

        if start_sniffer:  # pragma: no cover
            self._sniffer.start()

        super(EndpointsServer, self).__init__()
Exemple #14
0
def test_setwatches():
  requests = []

  def handler(request):
    if isinstance(request, SetWatchesRequest):
      requests.append(request)

  sniffer = Sniffer(SnifferConfig())
  sniffer.add_request_handler(handler)
  consume_packets('setwatches', sniffer)

  assert len(requests) == 1

  req = requests[0]

  assert len(req.child) == 5
  assert "/foo" in req.child
  assert "/in" in req.child
  assert "/zookeeper" in req.child
  assert "/in/portland" in req.child
  assert "/" in req.child
Exemple #15
0
def test_setwatches():
  requests = []

  def handler(request):
    if isinstance(request, SetWatchesRequest):
      requests.append(request)

  sniffer = Sniffer(SnifferConfig())
  sniffer.add_request_handler(handler)
  consume_packets('setwatches', sniffer)

  assert len(requests) == 1

  req = requests[0]

  assert len(req.child) == 5
  assert "/foo" in req.child
  assert "/in" in req.child
  assert "/zookeeper" in req.child
  assert "/in/portland" in req.child
  assert "/" in req.child
  def __init__(
      self, iface, zkport, request_handler, reply_handler=None, event_handler=None, start_sniffer=True):
    config = SnifferConfig(iface=iface)
    config.zookeeper_port = zkport
    config.update_filter()

    self._sniffer = Sniffer(config, request_handler, reply_handler, event_handler)

    if start_sniffer:  # pragma: no cover
      self._sniffer.start()

    super(EndpointsServer, self).__init__()
Exemple #17
0
def get_sniffer(request_handler, reply_handler=None, event_handler=None):
  config = SnifferConfig()
  config.track_replies = True

  sniffer = Sniffer(config)
  sniffer.add_request_handler(request_handler)
  if reply_handler:
    sniffer.add_reply_handler(reply_handler)
  if event_handler:
    sniffer.add_event_handler(event_handler)

  return sniffer
Exemple #18
0
def get_sniffer(printer, with_pings=True):
    config = SnifferConfig()
    config.track_replies = True

    if with_pings:
        config.include_pings()

    sniffer = Sniffer(config)
    sniffer.add_request_handler(printer.request_handler)
    sniffer.add_reply_handler(printer.reply_handler)
    sniffer.add_event_handler(printer.event_handler)

    return sniffer
Exemple #19
0
def _test_requests_replies(pcap_name, request_cls, reply_cls, nreqs, nreps):
  requests = []
  replies = []

  def handler(msg):
    if isinstance(msg, request_cls):
      requests.append(msg)
    elif isinstance(msg, reply_cls):
      replies.append(msg)

  config = SnifferConfig()
  config.track_replies = True
  sniffer = Sniffer(config)
  sniffer.add_request_handler(handler)
  sniffer.add_reply_handler(handler)

  consume_packets(pcap_name, sniffer)

  assert len(requests) == nreqs
  assert len(replies) == nreps

  sniffer.stop()
Exemple #20
0
def _test_requests_replies(pcap_name, request_cls, reply_cls, nreqs, nreps):
  requests = []
  replies = []

  def handler(msg):
    if isinstance(msg, request_cls):
      requests.append(msg)
    elif isinstance(msg, reply_cls):
      replies.append(msg)

  config = SnifferConfig()
  config.track_replies = True
  sniffer = Sniffer(config)
  sniffer.add_request_handler(handler)
  sniffer.add_reply_handler(handler)

  consume_packets(pcap_name, sniffer)

  assert len(requests) == nreqs
  assert len(replies) == nreps

  sniffer.stop()
Exemple #21
0
def default_zkt():
    stats = AccumulatedStats(StatsConfig())
    zkt = Sniffer(SnifferConfig())
    zkt.add_request_handler(stats.handle_request)
    return (zkt, stats)
Exemple #22
0
def main(_, options):

  if options.version:
    sys.stdout.write("%s\n" % __version__)
    sys.exit(0)

  config = SnifferConfig(options.iface)
  config.track_replies = True
  config.zookeeper_port = options.zookeeper_port
  config.max_queued_requests = options.max_queued_requests
  config.client_port = options.client_port if options.client_port != 0 else config.client_port

  if options.excluded_hosts and options.included_hosts:
    sys.stderr.write("The flags --include-host and --exclude-host can't be mixed.\n")
    sys.exit(1)

  if options.excluded_hosts:
    config.excluded_ips += expand_hosts(options.excluded_hosts)
  elif options.included_hosts:
    config.included_ips += expand_hosts(options.included_hosts)

  config.update_filter()

  if options.include_pings:
    config.include_pings()

  config.dump_bad_packet = options.dump_bad_packet

  loopback = options.iface in ["lo", "lo0"]

  if options.count_requests > 0 and options.measure_latency > 0:
    sys.stderr.write("The flags --count-requests and --measure-latency can't be mixed.\n")
    sys.exit(1)

  if options.count_requests > 0:
    validate_group_by(options.group_by)
    validate_aggregation_depth(options.aggregation_depth)
    p = CountPrinter(options.count_requests, options.group_by, loopback, options.aggregation_depth)
  elif options.measure_latency > 0:
    validate_group_by(options.group_by)
    validate_aggregation_depth(options.aggregation_depth)
    validate_sort_by(options.sort_by)
    p = LatencyPrinter(
      options.measure_latency, options.group_by, loopback, options.aggregation_depth,
      options.sort_by)
  elif options.unpaired:
    p = UnpairedPrinter(options.colors, loopback)
  else:
    p = DefaultPrinter(options.colors, loopback)
  p.start()

  sniffer = Sniffer(
    config,
    p.request_handler,
    p.reply_handler,
    p.event_handler,
    error_to_stderr=True
  )
  sniffer.start()

  try:
    while p.isAlive():
      time.sleep(0.5)
  except (KeyboardInterrupt, SystemExit):
    p.cancel()

  # shutdown sniffer
  sniffer.stop()
  while sniffer.isAlive():
    time.sleep(0.001)

  try:
    sys.stdout.write("\033[0m")
    sys.stdout.flush()
  except IOError: pass
Exemple #23
0
 def setUp(self):
   self.zkt = Sniffer(SnifferConfig())
Exemple #24
0
class TestSniffer(TestCase):
  def setUp(self):
    self.zkt = Sniffer(SnifferConfig())

  @mock.patch('os.kill', spec=os.kill)
  @mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
  def test_run_socket_error(self, mock_sniff, mock_kill):
    mock_sniff.side_effect = socket.error

    self.zkt.run()

    mock_sniff.assert_called_once_with(
        filter=self.zkt.config.filter,
        store=0,
        prn=self.zkt.handle_packet,
        iface=self.zkt.config.iface,
        stop_filter=self.zkt.wants_stop
    )
    mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)

  @mock.patch('os.kill', spec=os.kill)
  @mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
  def test_run(self, mock_sniff, mock_kill):
    self.zkt.run()

    mock_sniff.assert_called_once_with(
        filter=self.zkt.config.filter,
        store=0,
        prn=self.zkt.handle_packet,
        iface=self.zkt.config.iface,
        stop_filter=self.zkt.wants_stop
    )
    mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)

  def test_exclude(self):
    filter_text = 'port 2181'
    excluded_ip = '8.8.8.8'
    assert self.zkt.config.filter == filter_text

    self.zkt.config.excluded_ips = [excluded_ip]
    assert self.zkt.config.filter == filter_text
    self.zkt.config.update_filter()
    assert self.zkt.config.filter == '%s and host not %s' % (filter_text, excluded_ip)

    excluded_ip_two = '8.8.4.4'
    excluded_ip_three = '172.24.6.11'
    self.zkt.config.excluded_ips = [excluded_ip, excluded_ip_two, excluded_ip_three]
    self.zkt.config.update_filter()
    assert self.zkt.config.filter == '%s and host not %s and host not %s and host not %s' % (
        filter_text, excluded_ip, excluded_ip_two, excluded_ip_three)

  def test_include(self):
    filter_text = 'port 2181'
    included_ip = '8.8.8.8'
    assert self.zkt.config.filter == filter_text
    self.zkt.config.included_ips = [included_ip]
    assert self.zkt.config.filter == filter_text
    self.zkt.config.update_filter()
    assert self.zkt.config.filter == '%s and (host %s)' % (filter_text, included_ip)

    included_ip_two = '8.8.4.4'
    included_ip_three = '172.24.6.11'
    self.zkt.config.included_ips = [included_ip, included_ip_two, included_ip_three]
    self.zkt.config.update_filter()
    assert self.zkt.config.filter == '%s and (host %s or host %s or host %s)' % (
        filter_text, included_ip, included_ip_two, included_ip_three)
Exemple #25
0
def default_zkt():
  stats = AccumulatedStats(StatsConfig())
  zkt = Sniffer(SnifferConfig())
  zkt.add_request_handler(stats.handle_request)
  return (zkt, stats)
Exemple #26
0
def default_sniffer(aggregation_depth=1):
  stats = PerPathStatsAccumulator(aggregation_depth=aggregation_depth)
  sniffer = Sniffer(SnifferConfig())
  sniffer.add_request_handler(stats.update_request_stats)
  return (sniffer, stats)
Exemple #27
0
class TestSniffer(TestCase):
    def setUp(self):
        self.zkt = Sniffer(SnifferConfig())

    @mock.patch('os.kill', spec=os.kill)
    @mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
    def test_run_socket_error(self, mock_sniff, mock_kill):
        mock_sniff.side_effect = socket.error

        self.zkt.run()

        mock_sniff.assert_called_once_with(filter=self.zkt.config.filter,
                                           store=0,
                                           prn=self.zkt.handle_packet,
                                           iface=self.zkt.config.iface)
        mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)

    @mock.patch('os.kill', spec=os.kill)
    @mock.patch('zktraffic.base.sniffer.sniff', spec=sniff)
    def test_run(self, mock_sniff, mock_kill):
        self.zkt.run()

        mock_sniff.assert_called_once_with(filter=self.zkt.config.filter,
                                           store=0,
                                           prn=self.zkt.handle_packet,
                                           iface=self.zkt.config.iface)
        mock_kill.assert_called_once_with(os.getpid(), signal.SIGINT)

    def test_exclude(self):
        filter_text = 'port 2181'
        excluded_ip = '8.8.8.8'
        assert self.zkt.config.filter == filter_text

        self.zkt.config.excluded_ips = [excluded_ip]
        assert self.zkt.config.filter == filter_text
        self.zkt.config.update_filter()
        assert self.zkt.config.filter == '%s and host not %s' % (filter_text,
                                                                 excluded_ip)

        excluded_ip_two = '8.8.4.4'
        excluded_ip_three = '172.24.6.11'
        self.zkt.config.excluded_ips = [
            excluded_ip, excluded_ip_two, excluded_ip_three
        ]
        self.zkt.config.update_filter()
        assert self.zkt.config.filter == '%s and host not %s and host not %s and host not %s' % (
            filter_text, excluded_ip, excluded_ip_two, excluded_ip_three)

    def test_include(self):
        filter_text = 'port 2181'
        included_ip = '8.8.8.8'
        assert self.zkt.config.filter == filter_text
        self.zkt.config.included_ips = [included_ip]
        assert self.zkt.config.filter == filter_text
        self.zkt.config.update_filter()
        assert self.zkt.config.filter == '%s and (host %s)' % (filter_text,
                                                               included_ip)

        included_ip_two = '8.8.4.4'
        included_ip_three = '172.24.6.11'
        self.zkt.config.included_ips = [
            included_ip, included_ip_two, included_ip_three
        ]
        self.zkt.config.update_filter()
        assert self.zkt.config.filter == '%s and (host %s or host %s or host %s)' % (
            filter_text, included_ip, included_ip_two, included_ip_three)
Exemple #28
0
def main(_, options):

    if options.version:
        sys.stdout.write("%s\n" % __version__)
        sys.exit(0)

    config = SnifferConfig(options.iface)
    config.track_replies = True
    config.zookeeper_port = options.zookeeper_port
    config.max_queued_requests = options.max_queued_requests
    config.client_port = options.client_port if options.client_port != 0 else config.client_port

    if options.excluded_hosts and options.included_hosts:
        sys.stderr.write(
            "The flags --include-host and --exclude-host can't be mixed.\n")
        sys.exit(1)

    if options.excluded_hosts:
        config.excluded_ips += expand_hosts(options.excluded_hosts)
    elif options.included_hosts:
        config.included_ips += expand_hosts(options.included_hosts)

    config.update_filter()

    if options.include_pings:
        config.include_pings()

    config.dump_bad_packet = options.dump_bad_packet

    loopback = options.iface in ["lo", "lo0"]

    if options.count_requests > 0 and options.measure_latency > 0:
        sys.stderr.write(
            "The flags --count-requests and --measure-latency can't be mixed.\n"
        )
        sys.exit(1)

    if options.count_requests > 0:
        validate_group_by(options.group_by)
        validate_aggregation_depth(options.aggregation_depth)
        p = CountPrinter(options.count_requests, options.group_by, loopback,
                         options.aggregation_depth)
    elif options.measure_latency > 0:
        validate_group_by(options.group_by)
        validate_aggregation_depth(options.aggregation_depth)
        validate_sort_by(options.sort_by)
        p = LatencyPrinter(options.measure_latency, options.group_by, loopback,
                           options.aggregation_depth, options.sort_by)
    elif options.unpaired:
        p = UnpairedPrinter(options.colors, loopback)
    else:
        p = DefaultPrinter(options.colors, loopback)
    p.start()

    sniffer = Sniffer(config,
                      p.request_handler,
                      p.reply_handler,
                      p.event_handler,
                      error_to_stderr=True)
    sniffer.start()

    try:
        while p.isAlive():
            time.sleep(0.5)
    except (KeyboardInterrupt, SystemExit):
        p.cancel()

    # shutdown sniffer
    sniffer.stop()
    while sniffer.isAlive():
        time.sleep(0.001)

    try:
        sys.stdout.write("\033[0m")
        sys.stdout.flush()
    except IOError:
        pass
Exemple #29
0
def default_sniffer(aggregation_depth=1):
  stats = PerPathStatsAccumulator(aggregation_depth=aggregation_depth)
  sniffer = Sniffer(SnifferConfig())
  sniffer.add_request_handler(stats.update_request_stats)
  return (sniffer, stats)
Exemple #30
0
 def setUp(self):
     self.zkt = Sniffer(SnifferConfig())