def test_get_srv_keyspace(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") utils.run_vtctl( 'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t.tablet_alias, t.mysql_port, t.port + 500)) self.rebuild(use_served_types=True) reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace") self.assertEqual(reply['TabletTypes'], ['master']) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.zkocc_port_base, trap_output=True) self.assertEqual( err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n") # vttopo API test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.vttopo_port_base, trap_output=True) self.assertEqual( err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n")
def test_get_srv_keyspace(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t.tablet_alias, t.mysql_port, t.port + 500)) self.rebuild(use_served_types=True) reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace") self.assertEqual(reply['TabletTypes'], ['master']) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n") # vttopo API test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % utils.vttopo_port_base, trap_output=True) self.assertEqual(err, "Partitions[master] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[rdonly] =\n" + " Shards[0]={Start: , End: }\n" + "Partitions[replica] =\n" + " Shards[0]={Start: , End: }\n" + "Shards[0]={Start: , End: }\n" + "TabletTypes[0] = master\n")
def test_get_end_points(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") t.update_addrs(mysql_ip_addr="127.0.0.1:%s" % t.mysql_port, secure_addr="localhost:%s" % (t.port + 500)) self.rebuild() self.assertEqual( len( self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "Entries[0] = 1 localhost\n") # vttopo API test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.vttopo_port_base, trap_output=True) self.assertEqual(err, "Entries[0] = 1 localhost\n")
def test_get_srv_keyspace_names(self): utils.run_vtctl('CreateKeyspace test_keyspace1') utils.run_vtctl('CreateKeyspace test_keyspace2') t1 = tablet.Tablet(tablet_uid=1, cell="nj") t1.init_tablet("master", "test_keyspace1", "0") t1.update_addrs() t2 = tablet.Tablet(tablet_uid=2, cell="nj") t2.init_tablet("master", "test_keyspace2", "0") t2.update_addrs() utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True) self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"]) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n") # vtgate zk API test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zk_port, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n") # vtgate zkocc API test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zkocc_port, trap_output=True) self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n")
def start(self): utils.prog_compile(['vtctld']) args = [ os.path.join(utils.vtroot, 'bin', 'vtctld'), '-debug', '-templates', utils.vttop + '/go/cmd/vtctld/templates' ] stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ utils.prog_compile(['vtaction', 'vttablet', ]) args = [os.path.join(utils.vtroot, 'bin', 'vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-log_dir', self.tablet_dir] dbconfigs = self._get_db_configs_file(repl_extra_flags) for key1 in dbconfigs: for key2 in dbconfigs[key1]: args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]]) if memcache: memcache = os.path.join(self.tablet_dir, "memcache.sock") args.extend(["-rowcache-bin", "memcached"]) args.extend(["-rowcache-s", memcache]) if auth: args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = utils.reserve_ports(1) args.extend(['-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc
def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None): utils.prog_compile(['mysqlctl']) env = None if extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = extra_my_cnf return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') + ' -tablet-uid %u ' % self.tablet_uid + cmd, env=env)
def start(self): utils.prog_compile(["vtctld"]) args = [ os.path.join(utils.vtroot, "bin", "vtctld"), "-debug", "-templates", utils.vttop + "/go/cmd/vtctld/templates", ] stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="OPEN", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ utils.prog_compile(['vtaction', 'vttablet', ]) args = [os.path.join(utils.vtroot, 'bin', 'vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-log_dir', self.tablet_dir, '-db-configs-file', self._write_db_configs_file(repl_extra_flags)] if memcache: memcache = os.path.join(self.tablet_dir, "memcache.sock") config = os.path.join(self.tablet_dir, "config.json") with open(config, 'w') as f: json.dump({"RowCache": ["memcached", "-s", memcache]}, f) args.extend(["-queryserver-config-file", config]) if auth: args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = utils.reserve_ports(1) args.extend(['-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") # increment count only the first time if not self.proc: Tablet.tablets_running += 1 self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="OPEN", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ utils.prog_compile(['vtaction', 'vttablet', ]) args = [os.path.join(utils.vtroot, 'bin', 'vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-logfile', self.logfile, '-log.level', 'INFO', '-db-configs-file', self._write_db_configs_file(repl_extra_flags), '-debug-querylog-file', self.querylog_file] if memcache: memcache = os.path.join(self.tablet_dir, "memcache.sock") config = os.path.join(self.tablet_dir, "config.json") with open(config, 'w') as f: json.dump({"RowCache": ["memcached", "-s", memcache]}, f) args.extend(["-queryserver-config-file", config]) if auth: args.extend(['-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json')]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = utils.reserve_ports(1) args.extend(['-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it utils.run(utils.vtroot+'/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state self.wait_for_vttablet_state(wait_for_state, port=port) Tablet.tablets_running += 1 return self.proc
def setup(): utils.zk_setup() utils.prog_compile(['mysqlctl', 'vtaction', 'vtctl', 'vttablet', ]) # start mysql instance external to the test setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql() ] utils.wait_procs(setup_procs) setup_tablets()
def mysqlctl(self, cmd, quiet=False, extra_my_cnf=None): utils.prog_compile(['mysqlctl']) logLevel = '' if utils.options.verbose and not quiet: logLevel = ' -log.level=INFO' env = None if extra_my_cnf: env = os.environ.copy() env['EXTRA_MY_CNF'] = extra_my_cnf return utils.run_bg(os.path.join(utils.vtroot, 'bin', 'mysqlctl') + logLevel + ' -tablet-uid %u ' % self.tablet_uid + cmd, env=env)
def test_get_end_points(self): utils.run_vtctl('CreateKeyspace test_keyspace') t = tablet.Tablet(tablet_uid=1, cell="nj") t.init_tablet("master", "test_keyspace", "0") t.update_addrs(mysql_ip_addr="127.0.0.1:%s" % t.mysql_port, secure_addr="localhost:%s" % (t.port + 500)) self.rebuild() self.assertEqual(len(self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "Entry[0] = 1 localhost\n") # vttopo API test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % utils.vttopo_port_base, trap_output=True) self.assertEqual(err, "Entry[0] = 1 localhost\n")
def test_get_srv_keyspace_names(self): utils.run_vtctl('CreateKeyspace test_keyspace1') utils.run_vtctl('CreateKeyspace test_keyspace2') t1 = tablet.Tablet(tablet_uid=1, cell="nj") t1.init_tablet("master", "test_keyspace1", "0") utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t1.tablet_alias, t1.mysql_port, t1.port + 500)) t2 = tablet.Tablet(tablet_uid=2, cell="nj") t2.init_tablet("master", "test_keyspace2", "0") utils.run_vtctl('UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t2.tablet_alias, t2.mysql_port, t2.port + 500)) utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True) self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"]) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "KeyspaceName[0] = test_keyspace1\n" + "KeyspaceName[1] = test_keyspace2\n") # vttopo API test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.vttopo_port_base, trap_output=True) self.assertEqual(err, "KeyspaceName[0] = test_keyspace1\n" + "KeyspaceName[1] = test_keyspace2\n")
def test_get_srv_keyspace_names(self): utils.run_vtctl('CreateKeyspace test_keyspace1') utils.run_vtctl('CreateKeyspace test_keyspace2') t1 = tablet.Tablet(tablet_uid=1, cell="nj") t1.init_tablet("master", "test_keyspace1", "0") utils.run_vtctl( 'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t1.tablet_alias, t1.mysql_port, t1.port + 500)) t2 = tablet.Tablet(tablet_uid=2, cell="nj") t2.init_tablet("master", "test_keyspace2", "0") utils.run_vtctl( 'UpdateTabletAddrs %s -mysql-ip-addr 127.0.0.1:%s -secure-addr 127.0.0.1:%s' % (t2.tablet_alias, t2.mysql_port, t2.port + 500)) utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True) self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"]) # zkocc API test utils.prog_compile(['zkclient2']) out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.zkocc_port_base, trap_output=True) self.assertEqual( err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n") # vttopo API test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode getSrvKeyspaceNames test_nj' % utils.vttopo_port_base, trap_output=True) self.assertEqual( err, "KeyspaceNames[0] = test_keyspace1\n" + "KeyspaceNames[1] = test_keyspace2\n")
def test_zkocc(self): # preload the test_nj cell zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s']) time.sleep(1) # create a python client. The first address is bad, will test the retry logic bad_port = utils.reserve_ports(3) zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port+1), "test_nj", 30) zkocc_client.dial() # test failure for a python client that cannot connect bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30) try: bad_zkocc_client.dial() raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.ERROR) # FIXME(ryszard): This can be changed into a self.assertRaises. try: bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(level) # get test utils.prog_compile(['zkclient2']) out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True) self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n") zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.assertDictContainsSubset({'Data': "Test data 1", 'Cached': True, 'Stale': False,}, zk_data) self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat']) # getv test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false) [1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false) [2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false) """) zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes'] self.assertEqual(len(zk_data), 3) for i, d in enumerate(zk_data): self.assertEqual(d['Data'], 'Test data %s' % (i + 1)) self.assertTrue(d['Cached']) self.assertFalse(d['Stale']) self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat']) # children test out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj/vt' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized(err, """Path = /zk/test_nj/vt Child[0] = zkocc1 Child[1] = zkocc2 NumChildren = 2 CVersion = 2 Cached = false Stale = false """) # zk command tests self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1") self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3 -rw-rw-rw- zk zk 11 %s data1 -rw-rw-rw- zk zk 11 %s data2 -rw-rw-rw- zk zk 11 %s data3 """ % (_format_time(zk_data[0]['Stat']['MTime']), _format_time(zk_data[1]['Stat']['MTime']), _format_time(zk_data[2]['Stat']['MTime']))) # test /zk/local is not resolved and rejected out, err = utils.run(utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/local/vt/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False) self.assertIn("zkocc: cannot resolve local cell", err) # start a background process to query the same value over and over again # while we kill the zk server and restart it outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False) filename = outfd.name querier = utils.run_bg('/bin/bash -c "while true ; do '+utils.vtroot+'/bin/zkclient2 -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file) outfd.close() time.sleep(1) # kill zk server, sleep a bit, restart zk server, sleep a bit utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2)) time.sleep(3) utils.run(utils.vtroot+'/bin/zkctl -zk.cfg 1@'+utils.hostname+':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base+1, utils.zk_port_base+2)) time.sleep(3) utils.kill_sub_process(querier) logging.debug("Checking %s", filename) fd = open(filename, "r") state = 0 for line in fd: if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n": stale = False elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n": stale = True else: raise utils.TestError('unexpected line: ', line) if state == 0: if stale: state = 1 elif state == 1: if not stale: state = 2 else: if stale: self.fail('unexpected stale state') self.assertEqual(state, 2) fd.close() utils.zkocc_kill(zkocc_14850) # check that after the server is gone, the python client fails correctly level = logging.getLogger().getEffectiveLevel() logging.getLogger().setLevel(logging.ERROR) try: zkocc_client.get("/zk/test_nj/vt/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(level)
def setup(): utils.prog_compile([ 'zkclient2', ]) utils.zk_setup()
def test_zkocc(self): # preload the test_nj cell zkocc_14850 = utils.zkocc_start( extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s']) time.sleep(1) # create a python client. The first address is bad, will test the retry logic bad_port = utils.reserve_ports(3) zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30) zkocc_client.dial() # test failure for a python client that cannot connect bad_zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30) try: bad_zkocc_client.dial() raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.ERROR) # FIXME(ryszard): This can be changed into a self.assertRaises. try: bad_zkocc_client.get("/zk/test_nj/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.WARNING) # get test utils.prog_compile(['zkclient2']) out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' % utils.zkocc_port_base, trap_output=True) self.assertEqual( err, "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n" ) zk_data = zkocc_client.get("/zk/test_nj/zkocc1/data1") self.assertDictContainsSubset( { 'Data': "Test data 1", 'Cached': True, 'Stale': False, }, zk_data) self.assertDictContainsSubset({ 'NumChildren': 0, 'Version': 0 }, zk_data['Stat']) # getv test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized( err, """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false) [1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false) [2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false) """) zk_data = zkocc_client.getv([ "/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2", "/zk/test_nj/zkocc1/data3" ])['Nodes'] self.assertEqual(len(zk_data), 3) for i, d in enumerate(zk_data): self.assertEqual(d['Data'], 'Test data %s' % (i + 1)) self.assertTrue(d['Cached']) self.assertFalse(d['Stale']) self.assertDictContainsSubset({ 'NumChildren': 0, 'Version': 0 }, d['Stat']) # children test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized( err, """Path = /zk/test_nj Child[0] = zkocc1 Child[1] = zkocc2 NumChildren = 2 CVersion = 4 Cached = false Stale = false """) # zk command tests self._check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1") self._check_zk_output( "ls -l /zk/test_nj/zkocc1", """total: 3 -rw-rw-rw- zk zk 11 %s data1 -rw-rw-rw- zk zk 11 %s data2 -rw-rw-rw- zk zk 11 %s data3 """ % (_format_time(zk_data[0]['Stat']['MTime']), _format_time(zk_data[1]['Stat']['MTime']), _format_time(zk_data[2]['Stat']['MTime']))) # test /zk/local is not resolved and rejected out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False) self.assertIn("zkocc: cannot resolve local cell", err) # start a background process to query the same value over and over again # while we kill the zk server and restart it outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False) filename = outfd.name querier = utils.run_bg( '/bin/bash -c "while true ; do ' + utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file) outfd.close() time.sleep(1) # kill zk server, sleep a bit, restart zk server, sleep a bit utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.kill_sub_process(querier) utils.debug("Checking " + filename) fd = open(filename, "r") state = 0 for line in fd: if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n": stale = False elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n": stale = True else: raise utils.TestError('unexpected line: ', line) if state == 0: if stale: state = 1 elif state == 1: if not stale: state = 2 else: if stale: self.fail('unexpected stale state') self.assertEqual(state, 2) fd.close() utils.zkocc_kill(zkocc_14850) # check that after the server is gone, the python client fails correctly logging.getLogger().setLevel(logging.ERROR) try: zkocc_client.get("/zk/test_nj/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.WARNING)
def start(self): utils.prog_compile(['vtctld']) args = [os.path.join(utils.vtroot, 'bin', 'vtctld'), '-debug', '-templates', utils.vttop + '/go/cmd/vtctld/templates'] stderr_fd = open(os.path.join(utils.tmp_root, "vtctld.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) return self.proc
def setup(): utils.prog_compile(['zkclient2', ]) utils.zk_setup()
def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="OPEN", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}): """ Starts a vttablet process, and returns it. The process is also saved in self.proc, so it's easy to kill as well. """ utils.prog_compile([ 'vtaction', 'vttablet', ]) args = [ os.path.join(utils.vtroot, 'bin', 'vttablet'), '-port', '%s' % (port or self.port), '-tablet-path', self.tablet_alias, '-logfile', self.logfile, '-log.level', 'INFO', '-db-configs-file', self._write_db_configs_file(repl_extra_flags), '-debug-querylog-file', self.querylog_file ] if memcache: self.start_memcache() args.extend(['-rowcache', self.memcache_path]) if auth: args.extend([ '-auth-credentials', os.path.join(utils.vttop, 'test', 'test_data', 'authcredentials_test.json') ]) if customrules: args.extend(['-customrules', customrules]) if schema_override: args.extend(['-schema-override', schema_override]) if cert: self.secure_port = utils.reserve_ports(1) args.extend([ '-secure-port', '%s' % self.secure_port, '-cert', cert, '-key', key ]) if ca_cert: args.extend(['-ca-cert', ca_cert]) stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w") self.proc = utils.run_bg(args, stderr=stderr_fd) stderr_fd.close() # wait for zookeeper PID just to be sure we have it utils.run(utils.vtroot + '/bin/zk wait -e ' + self.zk_pid, stdout=utils.devnull) # wait for query service to be in the right state self.wait_for_vttablet_state(wait_for_state, port=port) return self.proc