def test_get_srv_keyspace(self):
        utils.run_vtctl('CreateKeyspace test_keyspace')
        t = tablet.Tablet(tablet_uid=1, cell="nj")
        t.init_tablet("master", "test_keyspace", "0")
        utils.run_vtctl(
            'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s'
            % (t.tablet_alias, t.mysql_port, t.port + 500))
        self.rebuild(use_served_types=True)
        reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace")
        self.assertEqual(reply['TabletTypes'], ['master'])

        # zkocc API test
        utils.prog_compile(['zkclient2'])
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace'
            % utils.zkocc_port_base,
            trap_output=True)
        self.assertEqual(
            err, "Partitions[master] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Partitions[rdonly] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Partitions[replica] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n")

        # vttopo API test
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace'
            % utils.vttopo_port_base,
            trap_output=True)
        self.assertEqual(
            err, "Partitions[master] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Partitions[rdonly] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Partitions[replica] =\n" + "  Shards[0]={Start: , End: }\n" +
            "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n")
示例#2
0
  def test_get_srv_keyspace(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t.tablet_alias, t.mysql_port, t.port + 500))
    self.rebuild(use_served_types=True)
    reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace")
    self.assertEqual(reply['TabletTypes'], ['master'])

    # zkocc API test
    utils.prog_compile(['zkclient2'])
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.zkocc_port_base, trap_output=True)
    self.assertEqual(err,
                     "Partitions[master] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[rdonly] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[replica] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Shards[0]={Start: , End: }\n" +
                     "TabletTypes[0] = master\n")

    # vttopo API test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.vttopo_port_base, trap_output=True)
    self.assertEqual(err, "Partitions[master] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[rdonly] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[replica] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Shards[0]={Start: , End: }\n" +
                     "TabletTypes[0] = master\n")
    def test_get_end_points(self):
        utils.run_vtctl('CreateKeyspace test_keyspace')
        t = tablet.Tablet(tablet_uid=1, cell="nj")
        t.init_tablet("master", "test_keyspace", "0")
        t.update_addrs(mysql_ip_addr="127.0.0.1:%s" % t.mysql_port,
                       secure_addr="localhost:%s" % (t.port + 500))
        self.rebuild()
        self.assertEqual(
            len(
                self.topo.get_end_points("test_nj", "test_keyspace", "0",
                                         "master")['Entries']), 1)

        # zkocc API test
        utils.prog_compile(['zkclient2'])
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master'
            % utils.zkocc_port_base,
            trap_output=True)
        self.assertEqual(err, "Entries[0] = 1 localhost\n")

        # vttopo API test
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master'
            % utils.vttopo_port_base,
            trap_output=True)
        self.assertEqual(err, "Entries[0] = 1 localhost\n")
示例#4
0
  def test_get_srv_keyspace_names(self):
    utils.run_vtctl('CreateKeyspace test_keyspace1')
    utils.run_vtctl('CreateKeyspace test_keyspace2')
    t1 = tablet.Tablet(tablet_uid=1, cell="nj")
    t1.init_tablet("master", "test_keyspace1", "0")
    t1.update_addrs()
    t2 = tablet.Tablet(tablet_uid=2, cell="nj")
    t2.init_tablet("master", "test_keyspace2", "0")
    t2.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True)
    self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"])

    # zkocc API test
    utils.prog_compile(['zkclient2'])
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.zkocc_port_base, trap_output=True)
    self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                          "KeyspaceNames[1] = test_keyspace2\n")

    # vtgate zk API test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zk_port, trap_output=True)
    self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                          "KeyspaceNames[1] = test_keyspace2\n")

    # vtgate zkocc API test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zkocc_port, trap_output=True)
    self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                          "KeyspaceNames[1] = test_keyspace2\n")
示例#5
0
 def start(self):
     utils.prog_compile(['vtctld'])
     args = [
         os.path.join(utils.vtroot, 'bin', 'vtctld'), '-debug',
         '-templates', utils.vttop + '/go/cmd/vtctld/templates'
     ]
     stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w")
     self.proc = utils.run_bg(args, stderr=stderr_fd)
     return self.proc
示例#6
0
  def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
    """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
    utils.prog_compile(['vtaction',
                        'vttablet',
                        ])

    args = [os.path.join(utils.vtroot, 'bin', 'vttablet'),
            '-port', '%s' % (port or self.port),
            '-tablet-path', self.tablet_alias,
            '-log_dir', self.tablet_dir]

    dbconfigs = self._get_db_configs_file(repl_extra_flags)
    for key1 in dbconfigs:
      for key2 in dbconfigs[key1]:
        args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])

    if memcache:
      memcache = os.path.join(self.tablet_dir, "memcache.sock")
      args.extend(["-rowcache-bin", "memcached"])
      args.extend(["-rowcache-s", memcache])

    if auth:
      args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')])

    if customrules:
      args.extend(['-customrules', customrules])

    if schema_override:
      args.extend(['-schema-override', schema_override])

    if cert:
      self.secure_port = utils.reserve_ports(1)
      args.extend(['-secure-port', '%s' % self.secure_port,
                   '-cert', cert,
                   '-key', key])
      if ca_cert:
        args.extend(['-ca-cert', ca_cert])

    stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
    # increment count only the first time
    if not self.proc:
      Tablet.tablets_running += 1
    self.proc = utils.run_bg(args, stderr=stderr_fd)
    stderr_fd.close()

    # wait for zookeeper PID just to be sure we have it
    utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull)

    # wait for query service to be in the right state
    self.wait_for_vttablet_state(wait_for_state, port=port)

    return self.proc
示例#7
0
    def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None):
        utils.prog_compile(['mysqlctl'])

        env = None
        if extra_my_cnf:
            env = os.environ.copy()
            env['EXTRA_MY_CNF'] = extra_my_cnf

        return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') +
                            ' -tablet-uid %u ' % self.tablet_uid + cmd,
                            env=env)
示例#8
0
 def start(self):
     utils.prog_compile(["vtctld"])
     args = [
         os.path.join(utils.vtroot, "bin", "vtctld"),
         "-debug",
         "-templates",
         utils.vttop + "/go/cmd/vtctld/templates",
     ]
     stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w")
     self.proc = utils.run_bg(args, stderr=stderr_fd)
     return self.proc
示例#9
0
  def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="OPEN", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
    """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
    utils.prog_compile(['vtaction',
                        'vttablet',
                        ])

    args = [os.path.join(utils.vtroot, 'bin', 'vttablet'),
            '-port', '%s' % (port or self.port),
            '-tablet-path', self.tablet_alias,
            '-log_dir', self.tablet_dir,
            '-db-configs-file', self._write_db_configs_file(repl_extra_flags)]

    if memcache:
      memcache = os.path.join(self.tablet_dir, "memcache.sock")
      config = os.path.join(self.tablet_dir, "config.json")
      with open(config, 'w') as f:
        json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
      args.extend(["-queryserver-config-file", config])

    if auth:
      args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')])

    if customrules:
      args.extend(['-customrules', customrules])

    if schema_override:
      args.extend(['-schema-override', schema_override])

    if cert:
      self.secure_port = utils.reserve_ports(1)
      args.extend(['-secure-port', '%s' % self.secure_port,
                   '-cert', cert,
                   '-key', key])
      if ca_cert:
        args.extend(['-ca-cert', ca_cert])

    stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
    # increment count only the first time
    if not self.proc:
      Tablet.tablets_running += 1
    self.proc = utils.run_bg(args, stderr=stderr_fd)
    stderr_fd.close()

    # wait for zookeeper PID just to be sure we have it
    utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull)

    # wait for query service to be in the right state
    self.wait_for_vttablet_state(wait_for_state, port=port)

    return self.proc
示例#10
0
  def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None):
    utils.prog_compile(['mysqlctl'])


    env = None
    if extra_my_cnf:
      env = os.environ.copy()
      env['EXTRA_MY_CNF'] = extra_my_cnf

    return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') +
                        ' -tablet-uid %u ' % self.tablet_uid + cmd,
                        env=env)
示例#11
0
  def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="OPEN", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
    """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
    utils.prog_compile(['vtaction',
                        'vttablet',
                        ])

    args = [os.path.join(utils.vtroot, 'bin', 'vttablet'),
            '-port', '%s' % (port or self.port),
            '-tablet-path', self.tablet_alias,
            '-logfile', self.logfile,
            '-log.level', 'INFO',
            '-db-configs-file', self._write_db_configs_file(repl_extra_flags),
            '-debug-querylog-file', self.querylog_file]

    if memcache:
      memcache = os.path.join(self.tablet_dir, "memcache.sock")
      config = os.path.join(self.tablet_dir, "config.json")
      with open(config, 'w') as f:
        json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
      args.extend(["-queryserver-config-file", config])

    if auth:
      args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')])

    if customrules:
      args.extend(['-customrules', customrules])

    if schema_override:
      args.extend(['-schema-override', schema_override])

    if cert:
      self.secure_port = utils.reserve_ports(1)
      args.extend(['-secure-port', '%s' % self.secure_port,
                   '-cert', cert,
                   '-key', key])
      if ca_cert:
        args.extend(['-ca-cert', ca_cert])

    stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
    self.proc = utils.run_bg(args, stderr=stderr_fd)
    stderr_fd.close()

    # wait for zookeeper PID just to be sure we have it
    utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull)

    # wait for query service to be in the right state
    self.wait_for_vttablet_state(wait_for_state, port=port)

    Tablet.tablets_running += 1
    return self.proc
示例#12
0
def setup():
  utils.zk_setup()
  utils.prog_compile(['mysqlctl',
                      'vtaction',
                      'vtctl',
                      'vttablet',
                      ])

  # start mysql instance external to the test
  setup_procs = [master_tablet.init_mysql(),
                 replica_tablet.init_mysql()
                ]
  utils.wait_procs(setup_procs)
  setup_tablets()
示例#13
0
  def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None):
    utils.prog_compile(['mysqlctl'])

    logLevel = ''
    if utils.options.verbose and not quiet:
      logLevel = ' -log.level=INFO'

    env = None
    if extra_my_cnf:
      env = os.environ.copy()
      env['EXTRA_MY_CNF'] = extra_my_cnf

    return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') +
                        logLevel + ' -tablet-uid %u ' % self.tablet_uid + cmd,
                        env=env)
示例#14
0
  def test_get_end_points(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs(mysql_ip_addr="127.0.0.1:%s" % t.mysql_port, secure_addr="localhost:%s" % (t.port + 500))
    self.rebuild()
    self.assertEqual(len(self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1)

    # zkocc API test
    utils.prog_compile(['zkclient2'])
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.zkocc_port_base, trap_output=True)
    self.assertEqual(err, "Entry[0] = 1 localhost\n")

    # vttopo API test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.vttopo_port_base, trap_output=True)
    self.assertEqual(err, "Entry[0] = 1 localhost\n")
示例#15
0
文件: tablet.py 项目: hub501/go-pack
    def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None):
        utils.prog_compile(['mysqlctl'])

        logLevel = ''
        if utils.options.verbose and not quiet:
            logLevel = ' -log.level=INFO'

        env = None
        if extra_my_cnf:
            env = os.environ.copy()
            env['EXTRA_MY_CNF'] = extra_my_cnf

        return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') +
                            logLevel + ' -tablet-uid %u ' % self.tablet_uid +
                            cmd,
                            env=env)
示例#16
0
  def test_get_srv_keyspace_names(self):
    utils.run_vtctl('CreateKeyspace test_keyspace1')
    utils.run_vtctl('CreateKeyspace test_keyspace2')
    t1 = tablet.Tablet(tablet_uid=1, cell="nj")
    t1.init_tablet("master", "test_keyspace1", "0")
    utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t1.tablet_alias, t1.mysql_port, t1.port + 500))
    t2 = tablet.Tablet(tablet_uid=2, cell="nj")
    t2.init_tablet("master", "test_keyspace2", "0")
    utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t2.tablet_alias, t2.mysql_port, t2.port + 500))
    utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True)
    self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"])

    # zkocc API test
    utils.prog_compile(['zkclient2'])
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.zkocc_port_base, trap_output=True)
    self.assertEqual(err, "KeyspaceName[0] = test_keyspace1\n" +
                          "KeyspaceName[1] = test_keyspace2\n")

    # vttopo API test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.vttopo_port_base, trap_output=True)
    self.assertEqual(err, "KeyspaceName[0] = test_keyspace1\n" +
                          "KeyspaceName[1] = test_keyspace2\n")
    def test_get_srv_keyspace_names(self):
        utils.run_vtctl('CreateKeyspace test_keyspace1')
        utils.run_vtctl('CreateKeyspace test_keyspace2')
        t1 = tablet.Tablet(tablet_uid=1, cell="nj")
        t1.init_tablet("master", "test_keyspace1", "0")
        utils.run_vtctl(
            'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s'
            % (t1.tablet_alias, t1.mysql_port, t1.port + 500))
        t2 = tablet.Tablet(tablet_uid=2, cell="nj")
        t2.init_tablet("master", "test_keyspace2", "0")
        utils.run_vtctl(
            'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s'
            % (t2.tablet_alias, t2.mysql_port, t2.port + 500))
        utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*',
                        auto_log=True)
        self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'),
                              ["test_keyspace1", "test_keyspace2"])

        # zkocc API test
        utils.prog_compile(['zkclient2'])
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj'
            % utils.zkocc_port_base,
            trap_output=True)
        self.assertEqual(
            err, "KeyspaceNames[0] = test_keyspace1\n" +
            "KeyspaceNames[1] = test_keyspace2\n")

        # vttopo API test
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj'
            % utils.vttopo_port_base,
            trap_output=True)
        self.assertEqual(
            err, "KeyspaceNames[0] = test_keyspace1\n" +
            "KeyspaceNames[1] = test_keyspace2\n")
示例#18
0
  def test_zkocc(self):
    # preload the test_nj cell
    zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
    time.sleep(1)

    # create a python client. The first address is bad, will test the retry logic
    bad_port = utils.reserve_ports(3)
    zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30)
    zkocc_client.dial()

    # test failure for a python client that cannot connect
    bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
    try:
      bad_zkocc_client.dial()
      raise utils.TestError('exception expected')
    except zkocc.ZkOccError as e:
      if str(e) != "Cannot dial to any server":
        raise
    level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)

    # FIXME(ryszard): This can be changed into a self.assertRaises.
    try:
      bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
      self.fail('exception expected')
    except zkocc.ZkOccError as e:
      if str(e) != "Cannot dial to any server":
        raise

    logging.getLogger().setLevel(level)

    # get test
    utils.prog_compile(['zkclient2'])
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True)
    self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")

    zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
    self.assertDictContainsSubset({'Data': "Test data 1",
                                   'Cached': True,
                                   'Stale': False,},
                                  zk_data)
    self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])

    # getv test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % utils.zkocc_port_base, trap_output=True)
    self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
  [1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
  [2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
  """)
    zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
    self.assertEqual(len(zk_data), 3)
    for i, d in enumerate(zk_data):
      self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
      self.assertTrue(d['Cached'])
      self.assertFalse(d['Stale'])
      self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])

    # children test
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj/vt' % utils.zkocc_port_base, trap_output=True)
    self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
  Child[0] = zkocc1
  Child[1] = zkocc2
  NumChildren = 2
  CVersion = 2
  Cached = false
  Stale = false
  """)

    # zk command tests
    self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
    self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
  -rw-rw-rw- zk zk       11  %s data1
  -rw-rw-rw- zk zk       11  %s data2
  -rw-rw-rw- zk zk       11  %s data3
  """ % (_format_time(zk_data[0]['Stat']['MTime']),
         _format_time(zk_data[1]['Stat']['MTime']),
         _format_time(zk_data[2]['Stat']['MTime'])))

    # test /zk/local is not resolved and rejected
    out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False)
    self.assertIn("zkocc: cannot resolve local cell", err)

    # start a background process to query the same value over and over again
    # while we kill the zk server and restart it
    outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
    filename = outfd.name
    querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file)
    outfd.close()
    time.sleep(1)

    # kill zk server, sleep a bit, restart zk server, sleep a bit
    utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
    time.sleep(3)
    utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2))
    time.sleep(3)

    utils.kill_sub_process(querier)

    logging.debug("Checking %s", filename)
    fd = open(filename, "r")
    state = 0
    for line in fd:
      if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
        stale = False
      elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
        stale = True
      else:
        raise utils.TestError('unexpected line: ', line)
      if state == 0:
        if stale:
          state = 1
      elif state == 1:
        if not stale:
          state = 2
      else:
        if stale:
          self.fail('unexpected stale state')
    self.assertEqual(state, 2)
    fd.close()

    utils.zkocc_kill(zkocc_14850)

    # check that after the server is gone, the python client fails correctly
    level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)
    try:
      zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
      self.fail('exception expected')
    except zkocc.ZkOccError as e:
      if str(e) != "Cannot dial to any server":
        raise
    logging.getLogger().setLevel(level)
示例#19
0
def setup():
    utils.prog_compile([
        'zkclient2',
    ])
    utils.zk_setup()
示例#20
0
    def test_zkocc(self):
        # preload the test_nj cell
        zkocc_14850 = utils.zkocc_start(
            extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
        time.sleep(1)

        # create a python client. The first address is bad, will test the retry logic
        bad_port = utils.reserve_ports(3)
        zkocc_client = zkocc.ZkOccConnection(
            "localhost:%u,localhost:%u,localhost:%u" %
            (bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30)
        zkocc_client.dial()

        # test failure for a python client that cannot connect
        bad_zkocc_client = zkocc.ZkOccConnection(
            "localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj",
            30)
        try:
            bad_zkocc_client.dial()
            raise utils.TestError('exception expected')
        except zkocc.ZkOccError as e:
            if str(e) != "Cannot dial to any server":
                raise
        logging.getLogger().setLevel(logging.ERROR)

        # FIXME(ryszard): This can be changed into a self.assertRaises.
        try:
            bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
            self.fail('exception expected')
        except zkocc.ZkOccError as e:
            if str(e) != "Cannot dial to any server":
                raise

        logging.getLogger().setLevel(logging.WARNING)

        # get test
        utils.prog_compile(['zkclient2'])
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' %
            utils.zkocc_port_base,
            trap_output=True)
        self.assertEqual(
            err,
            "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n"
        )

        zk_data = zkocc_client.get("/zk/test_nj/zkocc1/data1")
        self.assertDictContainsSubset(
            {
                'Data': "Test data 1",
                'Cached': True,
                'Stale': False,
            }, zk_data)
        self.assertDictContainsSubset({
            'NumChildren': 0,
            'Version': 0
        }, zk_data['Stat'])

        # getv test
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3'
            % utils.zkocc_port_base,
            trap_output=True)
        self.assertEqualNormalized(
            err,
            """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
  [1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
  [2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
  """)
        zk_data = zkocc_client.getv([
            "/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2",
            "/zk/test_nj/zkocc1/data3"
        ])['Nodes']
        self.assertEqual(len(zk_data), 3)
        for i, d in enumerate(zk_data):
            self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
            self.assertTrue(d['Cached'])
            self.assertFalse(d['Stale'])
            self.assertDictContainsSubset({
                'NumChildren': 0,
                'Version': 0
            }, d['Stat'])

        # children test
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' %
            utils.zkocc_port_base,
            trap_output=True)
        self.assertEqualNormalized(
            err, """Path = /zk/test_nj
  Child[0] = zkocc1
  Child[1] = zkocc2
  NumChildren = 2
  CVersion = 4
  Cached = false
  Stale = false
  """)

        # zk command tests
        self._check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
        self._check_zk_output(
            "ls -l /zk/test_nj/zkocc1", """total: 3
  -rw-rw-rw- zk zk       11  %s data1
  -rw-rw-rw- zk zk       11  %s data2
  -rw-rw-rw- zk zk       11  %s data3
  """ % (_format_time(zk_data[0]['Stat']['MTime']),
         _format_time(zk_data[1]['Stat']['MTime']),
         _format_time(zk_data[2]['Stat']['MTime'])))

        # test /zk/local is not resolved and rejected
        out, err = utils.run(
            utils.vtroot +
            '/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' %
            utils.zkocc_port_base,
            trap_output=True,
            raise_on_error=False)
        self.assertIn("zkocc: cannot resolve local cell", err)

        # start a background process to query the same value over and over again
        # while we kill the zk server and restart it
        outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
        filename = outfd.name
        querier = utils.run_bg(
            '/bin/bash -c "while true ; do ' + utils.vtroot +
            '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"'
            % utils.zkocc_port_base,
            stderr=outfd.file)
        outfd.close()
        time.sleep(1)

        # kill zk server, sleep a bit, restart zk server, sleep a bit
        utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
                  ':%u:%u:%u shutdown' %
                  (utils.zk_port_base, utils.zk_port_base + 1,
                   utils.zk_port_base + 2))
        time.sleep(3)
        utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
                  ':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base +
                                       1, utils.zk_port_base + 2))
        time.sleep(3)

        utils.kill_sub_process(querier)

        utils.debug("Checking " + filename)
        fd = open(filename, "r")
        state = 0
        for line in fd:
            if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
                stale = False
            elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
                stale = True
            else:
                raise utils.TestError('unexpected line: ', line)
            if state == 0:
                if stale:
                    state = 1
            elif state == 1:
                if not stale:
                    state = 2
            else:
                if stale:
                    self.fail('unexpected stale state')
        self.assertEqual(state, 2)
        fd.close()

        utils.zkocc_kill(zkocc_14850)

        # check that after the server is gone, the python client fails correctly
        logging.getLogger().setLevel(logging.ERROR)
        try:
            zkocc_client.get("/zk/test_nj/zkocc1/data1")
            self.fail('exception expected')
        except zkocc.ZkOccError as e:
            if str(e) != "Cannot dial to any server":
                raise
        logging.getLogger().setLevel(logging.WARNING)
示例#21
0
 def start(self):
   utils.prog_compile(['vtctld'])
   args = [os.path.join(utils.vtroot, 'bin', 'vtctld'), '-debug', '-templates', utils.vttop + '/go/cmd/vtctld/templates']
   stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w")
   self.proc = utils.run_bg(args, stderr=stderr_fd)
   return self.proc
示例#22
0
def setup():
  utils.prog_compile(['zkclient2',
                      ])
  utils.zk_setup()
示例#23
0
文件: tablet.py 项目: hub501/go-pack
    def start_vttablet(self,
                       port=None,
                       auth=False,
                       memcache=False,
                       wait_for_state="OPEN",
                       customrules=None,
                       schema_override=None,
                       cert=None,
                       key=None,
                       ca_cert=None,
                       repl_extra_flags={}):
        """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
        utils.prog_compile([
            'vtaction',
            'vttablet',
        ])

        args = [
            os.path.join(utils.vtroot, 'bin', 'vttablet'), '-port',
            '%s' % (port or self.port), '-tablet-path', self.tablet_alias,
            '-logfile', self.logfile, '-log.level', 'INFO', '-db-configs-file',
            self._write_db_configs_file(repl_extra_flags),
            '-debug-querylog-file', self.querylog_file
        ]

        if memcache:
            self.start_memcache()
            args.extend(['-rowcache', self.memcache_path])

        if auth:
            args.extend([
                '-auth-credentials',
                os.path.join(utils.vttop, 'test', 'test_data',
                             'authcredentials_test.json')
            ])

        if customrules:
            args.extend(['-customrules', customrules])

        if schema_override:
            args.extend(['-schema-override', schema_override])

        if cert:
            self.secure_port = utils.reserve_ports(1)
            args.extend([
                '-secure-port',
                '%s' % self.secure_port, '-cert', cert, '-key', key
            ])
            if ca_cert:
                args.extend(['-ca-cert', ca_cert])

        stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
        self.proc = utils.run_bg(args, stderr=stderr_fd)
        stderr_fd.close()

        # wait for zookeeper PID just to be sure we have it
        utils.run(utils.vtroot + '/bin/zk wait -e ' + self.zk_pid,
                  stdout=utils.devnull)

        # wait for query service to be in the right state
        self.wait_for_vttablet_state(wait_for_state, port=port)

        return self.proc