Esempio n. 1
0
  def test_get_srv_keyspace_names(self):
    utils.run_vtctl('CreateKeyspace test_keyspace1')
    utils.run_vtctl('CreateKeyspace test_keyspace2')
    t1 = tablet.Tablet(tablet_uid=1, cell="nj")
    t1.init_tablet("master", "test_keyspace1", "0")
    t1.update_addrs()
    t2 = tablet.Tablet(tablet_uid=2, cell="nj")
    t2.init_tablet("master", "test_keyspace2", "0")
    t2.update_addrs()
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True)

    # vtgate API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zk_port, trap_output=True)
    self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                          "KeyspaceNames[1] = test_keyspace2\n")

    if environment.topo_server().flavor() == 'zookeeper':
      self.assertItemsEqual(self.topo.get_srv_keyspace_names('local'), ["test_keyspace1", "test_keyspace2"])

      # zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % environment.topo_server().zkocc_port_base, trap_output=True)
      self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                       "KeyspaceNames[1] = test_keyspace2\n")

      # vtgate zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspaceNames test_nj' % self.vtgate_zkocc_port, trap_output=True)
      self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                       "KeyspaceNames[1] = test_keyspace2\n")
Esempio n. 2
0
  def _check_zk_output(self, cmd, expected):
    # directly for sanity
    out, err = utils.run(environment.binary_argstr('zk')+' ' + cmd, trap_output=True)
    self.assertEqualNormalized(out, expected, 'unexpected direct zk output')

    # using zkocc
    out, err = utils.run(environment.binary_argstr('zk')+' --zk.zkocc-addr=localhost:%u %s' % (environment.topo_server().zkocc_port_base, cmd), trap_output=True)
    self.assertEqualNormalized(out, expected, 'unexpected zk zkocc output')
    logging.debug("Matched: %s", out)
Esempio n. 3
0
 def create_customrules(self, filename):
   with open(filename, "w") as f:
     f.write("""[{
       "Name": "r1",
       "Description": "disallow bindvar 'asdfg'",
       "BindVarConds":[{
         "Name": "asdfg",
         "OnAbsent": false,
         "Operator": "NOOP"
       }]
     }]""")
   if self.env == "vttablet":
     if environment.topo_server().flavor() == 'zookeeper':
       utils.run(environment.binary_argstr('zk') + ' touch -p /zk/test_ca/config/customrules/testrules')
       utils.run(environment.binary_argstr('zk') + ' cp ' + filename + ' /zk/test_ca/config/customrules/testrules')
Esempio n. 4
0
 def restore_customrules(self):
   customrules = os.path.join(environment.tmproot, 'customrules.json')
   self.create_customrules(customrules)
   if environment.topo_server().flavor() == 'zookeeper':
     utils.run(
         environment.binary_argstr('zk') + ' cp ' + customrules +
         ' /zk/test_ca/config/customrules/testrules')
    def test_regular_operation(self):
        # Use a dedicated worker to run all vtworker commands.
        worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(["--cell", "test_nj"], auto_log=True)
        vtworker_endpoint = "localhost:" + str(worker_rpc_port)

        automation_server_proc, automation_server_port = utils.run_automation_server()

        source_shard_list = "0"
        dest_shard_list = "-80,80-"
        _, vtctld_endpoint = utils.vtctld.rpc_endpoint()
        utils.run(
            environment.binary_argstr("automation_client")
            + " --server localhost:"
            + str(automation_server_port)
            + " --task HorizontalReshardingTask"
            + " --param keyspace="
            + self.KEYSPACE
            + " --param source_shard_list="
            + source_shard_list
            + " --param dest_shard_list="
            + dest_shard_list
            + " --param vtctld_endpoint="
            + vtctld_endpoint
            + " --param vtworker_endpoint="
            + vtworker_endpoint
            + " --param min_healthy_rdonly_tablets=1"
        )

        self.verify()

        utils.kill_sub_process(automation_server_proc, soft=True)
        utils.kill_sub_process(worker_proc, soft=True)
Esempio n. 6
0
  def test_zkocc_qps(self):
    # preload the test_nj cell
    zkocc_14850 = utils.zkocc_start()

    qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.zkocc_port_base)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the zkocc vars, make sure we have what we need
    v = utils.get_vars(environment.zkocc_port_base)
    if v['ZkReader']['test_nj']['State'] != 'Connected':
      self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])

    # some checks on performance / stats
    # a typical workstation will do 45-47k QPS, check we have more than 10k
    rpcCalls = v['ZkReader']['RpcCalls']
    if rpcCalls < 100000:
      self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    cacheReads = v['ZkReader']['test_nj']['CacheReads']
    if cacheReads < 100000:
      self.fail('Cache QPS is too low: %u < 10000' % (cacheReads / 10))
    totalCacheReads = v['ZkReader']['total']['CacheReads']
    self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
    self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
    utils.zkocc_kill(zkocc_14850)
Esempio n. 7
0
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start()
    qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    # a typical workstation will do 38-40k QPS, check we have more than 10k
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < 100000:
      self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
  def test_regular_operation(self):
    # Use a dedicated worker to run all vtworker commands.
    worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
        ['--cell', 'test_nj'],
        auto_log=True)
    vtworker_endpoint = "localhost:" + str(worker_rpc_port)

    automation_server_proc, automation_server_port = utils.run_automation_server()

    keyspace = 'test_keyspace'
    source_shard_list = '0'
    dest_shard_list = '-80,80-'
    _, vtctld_endpoint = utils.vtctld.rpc_endpoint()
    utils.run(environment.binary_argstr('automation_client') +
              ' --server localhost:' + str(automation_server_port) +
              ' --task HorizontalReshardingTask' +
              ' --param keyspace=' + keyspace +
              ' --param source_shard_list=' + source_shard_list +
              ' --param dest_shard_list=' + dest_shard_list +
              ' --param source_shard_rdonly_list=' +
              worker.shard_rdonly1.tablet_alias +
              ' --param dest_shard_rdonly_list=' +
              worker.shard_0_rdonly1.tablet_alias + ',' +
              worker.shard_1_rdonly1.tablet_alias +
              ' --param vtctld_endpoint=' + vtctld_endpoint +
              ' --param vtworker_endpoint=' + vtworker_endpoint)

    self.assert_shard_data_equal(0, worker.shard_master,
                                 worker.shard_0_tablets.replica)
    self.assert_shard_data_equal(1, worker.shard_master,
                                 worker.shard_1_tablets.replica)

    utils.kill_sub_process(automation_server_proc, soft=True)
    utils.kill_sub_process(worker_proc, soft=True)
Esempio n. 9
0
  def test_zkocc_qps(self):
    # preload the test_nj cell
    zkocc_14850 = utils.zkocc_start()

    qpser = utils.run_bg(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode qps /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2' % environment.topo_server().zkocc_port_base)
    qpser.wait()

    # get the zkocc vars, make sure we have what we need
    v = utils.get_vars(environment.topo_server().zkocc_port_base)
    if v['ZkReader']['test_nj']['State'] != 'Connected':
      self.fail('invalid zk global state: ' + v['ZkReader']['test_nj']['State'])

    # some checks on performance / stats
    rpcCalls = v['ZkReader']['RpcCalls']
    if rpcCalls < MIN_QPS * 10:
      self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    cacheReads = v['ZkReader']['test_nj']['CacheReads']
    if cacheReads < MIN_QPS * 10:
      self.fail('Cache QPS is too low: %u < %u' % (cacheReads, MIN_QPS * 10))
    totalCacheReads = v['ZkReader']['total']['CacheReads']
    self.assertEqual(cacheReads, totalCacheReads, 'Rollup stats are wrong')
    self.assertEqual(v['ZkReader']['UnknownCellErrors'], 0, 'unexpected UnknownCellErrors')
    utils.zkocc_kill(zkocc_14850)
  def test_regular_operation(self):
    # Use a dedicated worker to run all vtworker commands.
    worker_proc, _, worker_rpc_port = utils.run_vtworker_bg(
        ['--cell', 'test_nj'],
        auto_log=True)
    vtworker_endpoint = 'localhost:' + str(worker_rpc_port)

    automation_server_proc, automation_server_port = (
        utils.run_automation_server())

    source_shard_list = '0'
    dest_shard_list = '-80,80-'
    _, vtctld_endpoint = utils.vtctld.rpc_endpoint()
    utils.run(
        environment.binary_argstr('automation_client') +
        ' --server localhost:' + str(automation_server_port) +
        ' --task HorizontalReshardingTask' +
        ' --param keyspace=' + self.KEYSPACE +
        ' --param source_shard_list=' + source_shard_list +
        ' --param dest_shard_list=' + dest_shard_list +
        ' --param vtctld_endpoint=' + vtctld_endpoint +
        ' --param vtworker_endpoint=' + vtworker_endpoint +
        ' --param min_healthy_rdonly_endpoints=1')

    self.verify()

    utils.kill_sub_process(automation_server_proc, soft=True)
    utils.kill_sub_process(worker_proc, soft=True)
Esempio n. 11
0
  def test_get_srv_keyspace(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s -vts-port %s %s' % (t.mysql_port, t.port + 500, t.tablet_alias))
    self.rebuild()

    # vtgate zk API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zk_port, trap_output=True)
    self.assertEqual(err, "Partitions[master] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[rdonly] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Partitions[replica] =\n" +
                     "  Shards[0]={Start: , End: }\n" +
                     "Shards[0]={Start: , End: }\n" +
                     "TabletTypes[0] = master\n",
                     "Got wrong content: %s" % err)

    if environment.topo_server().flavor() == 'zookeeper':
      reply = self.topo.get_srv_keyspace("test_nj", "test_keyspace")
      self.assertEqual(reply['TabletTypes'], ['master'])

      # zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % environment.topo_server().zkocc_port_base, trap_output=True)
      self.assertEqual(err,
                       "Partitions[master] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Partitions[rdonly] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Partitions[replica] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Shards[0]={Start: , End: }\n" +
                       "TabletTypes[0] = master\n",
                       "Got wrong content: %s" % err)

      # vtgate zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getSrvKeyspace test_nj test_keyspace' % self.vtgate_zkocc_port, trap_output=True)
      self.assertEqual(err, "Partitions[master] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Partitions[rdonly] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Partitions[replica] =\n" +
                       "  Shards[0]={Start: , End: }\n" +
                       "Shards[0]={Start: , End: }\n" +
                       "TabletTypes[0] = master\n",
                       "Got wrong content: %s" % err)
Esempio n. 12
0
 def restore_customrules(self):
     customrules = os.path.join(environment.tmproot, "customrules.json")
     self.create_customrules(customrules)
     if self.env == "vttablet":
         if environment.topo_server().flavor() == "zookeeper":
             utils.run(
                 environment.binary_argstr("zk") + " cp " + customrules + " /zk/test_ca/config/customrules/testrules"
             )
Esempio n. 13
0
  def test_get_end_points(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    self.rebuild()

    # vtgate zk API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True)
    self.assertEqual(err, "Entries[0] = 1 localhost\n")
Esempio n. 14
0
  def test_get_end_points(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    self.rebuild()

    # vtgate zk API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zk_port, trap_output=True)
    self.assertEqual(err, "Entries[0] = 1 localhost\n")

    if environment.topo_server().flavor() == 'zookeeper':
      self.assertEqual(len(self.topo.get_end_points("test_nj", "test_keyspace", "0", "master")['Entries']), 1)

      # zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % environment.topo_server().zkocc_port_base, trap_output=True)
      self.assertEqual(err, "Entries[0] = 1 localhost\n")

      # vtgate zkocc API test
      out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode getEndPoints test_nj test_keyspace 0 master' % self.vtgate_zkocc_port, trap_output=True)
      self.assertEqual(err, "Entries[0] = 1 localhost\n")
Esempio n. 15
0
  def setUp(self):
    environment.topo_server().wipe()
    if environment.topo_server().flavor() == 'zookeeper':
      utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc1')
      utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc2')
      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename1 = fd.name
      fd.write("Test data 1")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename1+' /zk/test_nj/vt/zkocc1/data1')

      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename2 = fd.name
      fd.write("Test data 2")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename2+' /zk/test_nj/vt/zkocc1/data2')

      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename3 = fd.name
      fd.write("Test data 3")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename3+' /zk/test_nj/vt/zkocc1/data3')
Esempio n. 16
0
def zk_setup(add_bad_host=False):
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run('%s -log_dir %s -zk.cfg [email protected]%s:%s init' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports))
  config = environment.tmproot+'/test-zk-client-conf.json'
  with open(config, 'w') as f:
    ca_server = 'localhost:%u' % (zk_port_base+2)
    if add_bad_host:
      ca_server += ',does.not.exists:1234'
    zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
                       'test_ny': 'localhost:%u'%(zk_port_base+2),
                       'test_ca': ca_server,
                       'global': 'localhost:%u'%(zk_port_base+2),
                       'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
                       'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
                       'test_ca:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
                       'global:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),}
    json.dump(zk_cell_mapping, f)
  os.environ['ZK_CLIENT_CONFIG'] = config
  run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt')
  run(environment.binary_argstr('zk')+' touch -p /zk/test_ny/vt')
  run(environment.binary_argstr('zk')+' touch -p /zk/test_ca/vt')
Esempio n. 17
0
def zk_wipe():
  # Work around safety check on recursive delete.
  run(environment.binary_argstr('zk')+' rm -rf /zk/test_nj/vt/*')
  run(environment.binary_argstr('zk')+' rm -rf /zk/test_ny/vt/*')
  run(environment.binary_argstr('zk')+' rm -rf /zk/global/vt/*')

  run(environment.binary_argstr('zk')+' rm -f /zk/test_nj/vt')
  run(environment.binary_argstr('zk')+' rm -f /zk/test_ny/vt')
  run(environment.binary_argstr('zk')+' rm -f /zk/global/vt')
Esempio n. 18
0
  def test_get_srv_keyspace_names(self):
    utils.run_vtctl('CreateKeyspace test_keyspace1')
    utils.run_vtctl('CreateKeyspace test_keyspace2')
    t1 = tablet.Tablet(tablet_uid=1, cell="nj")
    t1.init_tablet("master", "test_keyspace1", "0")
    t1.update_addrs()
    t2 = tablet.Tablet(tablet_uid=2, cell="nj")
    t2.init_tablet("master", "test_keyspace2", "0")
    t2.update_addrs()
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace*'], auto_log=True)

    # vtgate API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspaceNames test_nj' % utils.vtgate.port, trap_output=True)
    self.assertEqual(err, "KeyspaceNames[0] = test_keyspace1\n" +
                          "KeyspaceNames[1] = test_keyspace2\n")
Esempio n. 19
0
 def change_customrules(self):
   customrules = os.path.join(environment.tmproot, 'customrules.json')
   with open(customrules, "w") as f:
     f.write("""[{
       "Name": "r2",
       "Description": "disallow bindvar 'gfdsa'",
       "BindVarConds":[{
         "Name": "gfdsa",
         "OnAbsent": false,
         "Operator": "NOOP"
       }]
     }]""")
   if self.env == "vttablet":
     if environment.topo_server().flavor() == 'zookeeper':
       utils.run(environment.binary_argstr('zk') + ' cp ' + customrules + ' /zk/test_ca/config/customrules/testrules')
Esempio n. 20
0
  def test_get_srv_keyspace(self):
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    utils.run_vtctl('UpdateTabletAddrs -hostname localhost -ip-addr 127.0.0.1 -mysql-port %s %s' % (t.mysql_port, t.tablet_alias))
    self.rebuild()

    # vtgate zk API test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%d -mode getSrvKeyspace test_nj test_keyspace' % utils.vtgate.port, trap_output=True)
    self.assertEqual(err, "Partitions[master] =\n" +
                     "  ShardReferences[0]={Start: , End: }\n" +
                     "Partitions[rdonly] =\n" +
                     "  ShardReferences[0]={Start: , End: }\n" +
                     "Partitions[replica] =\n" +
                     "  ShardReferences[0]={Start: , End: }\n",
                     "Got wrong content: %s" % err)
Esempio n. 21
0
  def test_sharding(self):

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    # run checks now before we start the tablets
    utils.validate_topology()

    # create databases, start the tablets, wait for them to start
    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None)
    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.wait_for_vttablet_state('SERVING')

    # apply the schema on the first shard through vtctl, so all tablets
    # are the same (replication is not enabled yet, so allow_replication=false
    # is just there to be tested)
    utils.run_vtctl(['ApplySchema',
                     '-stop-replication',
                     '-sql=' + create_vt_select_test.replace("\n", ""),
                     shard_0_master.tablet_alias])
    utils.run_vtctl(['ApplySchema',
                     '-stop-replication',
                     '-sql=' + create_vt_select_test.replace("\n", ""),
                     shard_0_replica.tablet_alias])

    # start vtgate, we'll use it later
    vtgate_server, vtgate_port = utils.vtgate_start()

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.reset_replication()
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/-80',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/80-',
                     shard_1_master.tablet_alias], auto_log=True)

    # apply the schema on the second shard using a simple schema upgrade
    utils.run_vtctl(['ApplySchemaShard',
                     '-simple',
                     '-sql=' + create_vt_select_test_reverse.replace("\n", ""),
                     'test_keyspace/80-'])

    # insert some values directly (db is RO after minority reparent)
    # FIXME(alainjobart) these values don't match the shard map
    utils.run_vtctl(['SetReadWrite', shard_0_master.tablet_alias])
    utils.run_vtctl(['SetReadWrite', shard_1_master.tablet_alias])
    shard_0_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True)
    shard_1_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True)

    utils.validate_topology(ping_tablets=True)

    utils.pause("Before the sql scatter query")

    # note the order of the rows is not guaranteed, as the go routines
    # doing the work can go out of order
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "10\ttest 10"])

    # write a value, re-read them all
    utils.vtclient2(3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True)
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "2\ttest 2",
                      "10\ttest 10"])

    # make sure the '2' value was written on first shard
    rows = shard_0_master.mquery('vt_test_keyspace', "select id, msg from vt_select_test order by id")
    self.assertEqual(rows, ((1, 'test 1'), (2, 'test 2'), ),
                     'wrong mysql_query output: %s' % str(rows))

    utils.pause("After db writes")

    # now use various topo servers and streaming or both for the same query
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "2\ttest 2",
                      "10\ttest 10"],
                     driver="vtdb-streaming")
    if environment.topo_server().flavor() == 'zookeeper':
      self._check_rows(["Index\tid\tmsg",
                        "1\ttest 1",
                        "2\ttest 2",
                        "10\ttest 10"],
                       driver="vtdb-zk")
      self._check_rows(["Index\tid\tmsg",
                        "1\ttest 1",
                        "2\ttest 2",
                        "10\ttest 10"],
                       driver="vtdb-zk-streaming")

    # make sure the schema checking works
    self._check_rows_schema_diff("vtdb")
    if environment.topo_server().flavor() == 'zookeeper':
      self._check_rows_schema_diff("vtdb-zk")

    # throw in some schema validation step
    # we created the schema differently, so it should show
    utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/-80'])
    utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/80-'])
    out, err = utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
                               trap_output=True, raise_on_error=False)
    if 'test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err or \
       'test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err:
      self.fail('wrong ValidateSchemaKeyspace output: ' + err)

    # validate versions
    utils.run_vtctl(['ValidateVersionShard', 'test_keyspace/-80'],
                    auto_log=True)
    utils.run_vtctl(['ValidateVersionKeyspace', 'test_keyspace'], auto_log=True)

    # show and validate permissions
    utils.run_vtctl(['GetPermissions', 'test_nj-0000062344'], auto_log=True)
    utils.run_vtctl(['ValidatePermissionsShard', 'test_keyspace/-80'],
                    auto_log=True)
    utils.run_vtctl(['ValidatePermissionsKeyspace', 'test_keyspace'],
                    auto_log=True)

    if environment.topo_server().flavor() == 'zookeeper':
      # and create zkns on this complex keyspace, make sure a few files are created
      utils.run_vtctl(['ExportZknsForKeyspace', 'test_keyspace'])
      out, err = utils.run(environment.binary_argstr('zk')+' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True)
      lines = out.splitlines()
      for base in ['-80', '80-']:
        for db_type in ['master', 'replica']:
          for sub_path in ['', '.vdns', '/0', '/vt.vdns']:
            expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
            if expected not in lines:
              self.fail('missing zkns part:\n%s\nin:%s' %(expected, out))

    # now try to connect using the python client and shard-aware connection
    # to both shards
    # first get the topology and check it
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                          "test_nj", 30.0)
    topology.read_keyspaces(vtgate_client)

    shard_0_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.-80.master:vt")
    if len(shard_0_master_addrs) != 1:
      self.fail('topology.get_host_port_by_name failed for "test_keyspace.-80.master:vt", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
    logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))

    # connect to shard -80
    conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0],
                                               shard_0_master_addrs[0][1]),
                                    "", "test_keyspace", "-80", 10.0)
    conn.dial()
    (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
    self.assertEqual(results, [(1, 'test 1'), (2, 'test 2'), ],
                     'wrong conn._execute output: %s' % str(results))

    # connect to shard 80-
    shard_1_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.80-.master:vt")
    conn = tablet3.TabletConnection("%s:%u" % (shard_1_master_addrs[0][0],
                                               shard_1_master_addrs[0][1]),
                                    "", "test_keyspace", "80-", 10.0)
    conn.dial()
    (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
    self.assertEqual(results, [(10, 'test 10'), ],
                     'wrong conn._execute output: %s' % str(results))
    vtgate_client.close()

    # try to connect with bad shard
    try:
      conn = tablet3.TabletConnection("localhost:%u" % shard_0_master.port,
                                      "", "test_keyspace", "-90", 10.0)
      conn.dial()
      self.fail('expected an exception')
    except Exception as e:
      if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
        self.fail('unexpected exception: ' + str(e))

    utils.vtgate_kill(vtgate_server)
    tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
                         shard_1_replica])
Esempio n. 22
0
def zk_cat(path):
  out, err = run(environment.binary_argstr('zk')+' cat '+path, trap_output=True)
  return out
Esempio n. 23
0
def zk_ls(path):
  out, err = run(environment.binary_argstr('zk')+' ls '+path, trap_output=True)
  return sorted(out.splitlines())
Esempio n. 24
0
  def _test_reparent_from_outside(self, brutal=False):
    utils.run_vtctl('CreateKeyspace test_keyspace')

    # create the database so vttablets start, as they are serving
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.create_db('vt_test_keyspace')

    # Start up a master mysql and vttablet
    tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True, wait_for_start=False)

    # Create a few slaves for testing reparenting.
    tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
    tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)
    tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True, wait_for_start=False)

    # wait for all tablets to start
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.wait_for_vttablet_state("SERVING")

    # Reparent as a starting point
    for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
      t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias, auto_log=True)

    # now manually reparent 1 out of 2 tablets
    # 62044 will be the new master
    # 31981 won't be re-parented, so it will be busted
    tablet_62044.mquery('', [
        "RESET MASTER",
        "STOP SLAVE",
        "RESET SLAVE",
        "CHANGE MASTER TO MASTER_HOST = ''",
        ])
    new_pos = tablet_62044.mquery('', 'show master status')
    logging.debug("New master position: %s" % str(new_pos))

    # 62344 will now be a slave of 62044
    tablet_62344.mquery('', [
        "RESET MASTER",
        "RESET SLAVE",
        "change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
        'start slave'
        ])

    # 41983 will be a slave of 62044
    tablet_41983.mquery('', [
        'stop slave',
        "change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
        'start slave'
        ])

    # in brutal mode, we scrap the old master first
    if brutal:
      tablet_62344.scrap(force=True)
      # we have some automated tools that do this too, so it's good to simulate
      if environment.topo_server_implementation == 'zookeeper':
        utils.run(environment.binary_argstr('zk')+' rm -rf ' + tablet_62344.zk_tablet_path)

    # update zk with the new graph
    utils.run_vtctl('ShardExternallyReparented test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True)

    self._test_reparent_from_outside_check(brutal)

    utils.run_vtctl('RebuildReplicationGraph test_nj test_keyspace')

    self._test_reparent_from_outside_check(brutal)

    tablet.kill_tablets([tablet_31981, tablet_62344, tablet_62044, tablet_41983])
Esempio n. 25
0
  def test_zkocc(self):
    # preload the test_nj cell
    zkocc_14850 = utils.zkocc_start(extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
    time.sleep(1)

    # create a python client. The first address is bad, will test the retry logic
    bad_port = environment.reserve_ports(3)
    zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u,localhost:%u" % (bad_port, environment.topo_server().zkocc_port_base, bad_port+1), "test_nj", 30)
    zkocc_client.dial()

    # test failure for a python client that cannot connect
    bad_zkocc_client = zkocc.ZkOccConnection("localhost:%u,localhost:%u" % (bad_port+2, bad_port), "test_nj", 30)
    try:
      bad_zkocc_client.dial()
      self.fail('exception expected')
    except zkocc.ZkOccError as e:
      if not str(e).startswith("Cannot dial to any server, tried: "):
        self.fail('unexpected exception: %s' % str(e))
    level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)

    # FIXME(ryszard): This can be changed into a self.assertRaises.
    try:
      bad_zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
      self.fail('exception expected')
    except zkocc.ZkOccError as e:
      if not str(e).startswith("Cannot dial to any server, tried: "):
        self.fail('unexpected exception: %s' % str(e))

    logging.getLogger().setLevel(level)

    # get test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1' % environment.topo_server().zkocc_port_base, trap_output=True)
    self.assertEqual(err, "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n")

    zk_data = zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
    self.assertDictContainsSubset({'Data': "Test data 1",
                                   'Cached': True,
                                   'Stale': False,},
                                  zk_data)
    self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, zk_data['Stat'])

    # getv test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 /zk/test_nj/vt/zkocc1/data2 /zk/test_nj/vt/zkocc1/data3' % environment.topo_server().zkocc_port_base, trap_output=True)
    self.assertEqualNormalized(err, """[0] /zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
  [1] /zk/test_nj/vt/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
  [2] /zk/test_nj/vt/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
  """)
    zk_data = zkocc_client.getv(["/zk/test_nj/vt/zkocc1/data1", "/zk/test_nj/vt/zkocc1/data2", "/zk/test_nj/vt/zkocc1/data3"])['Nodes']
    self.assertEqual(len(zk_data), 3)
    for i, d in enumerate(zk_data):
      self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
      self.assertTrue(d['Cached'])
      self.assertFalse(d['Stale'])
      self.assertDictContainsSubset({'NumChildren': 0, 'Version': 0}, d['Stat'])

    # children test
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u -mode children /zk/test_nj/vt' % environment.topo_server().zkocc_port_base, trap_output=True)
    self.assertEqualNormalized(err, """Path = /zk/test_nj/vt
  Child[0] = zkocc1
  Child[1] = zkocc2
  NumChildren = 2
  CVersion = 2
  Cached = false
  Stale = false
  """)

    # zk command tests
    self._check_zk_output("cat /zk/test_nj/vt/zkocc1/data1", "Test data 1")
    self._check_zk_output("ls -l /zk/test_nj/vt/zkocc1", """total: 3
  -rw-rw-rw- zk zk       11  %s data1
  -rw-rw-rw- zk zk       11  %s data2
  -rw-rw-rw- zk zk       11  %s data3
  """ % (_format_time(zk_data[0]['Stat']['MTime']),
         _format_time(zk_data[1]['Stat']['MTime']),
         _format_time(zk_data[2]['Stat']['MTime'])))

    # test /zk/local is not resolved and rejected
    out, err = utils.run(environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/local/vt/zkocc1/data1' % environment.topo_server().zkocc_port_base, trap_output=True, raise_on_error=False)
    self.assertIn("zkocc: cannot resolve local cell", err)

    # start a background process to query the same value over and over again
    # while we kill the zk server and restart it
    outfd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
    filename = outfd.name
    querier = utils.run_bg('/bin/bash -c "while true ; do '+environment.binary_argstr('zkclient2')+' -server localhost:%u /zk/test_nj/vt/zkocc1/data1 ; sleep 0.1 ; done"' % environment.topo_server().zkocc_port_base, stderr=outfd.file)
    outfd.close()
    time.sleep(1)

    # kill zk server, sleep a bit, restart zk server, sleep a bit
    utils.run(environment.binary_argstr('zkctl')+' -zk.cfg [email protected]'+utils.hostname+':%s shutdown' % environment.topo_server().zk_ports)
    time.sleep(3)
    utils.run(environment.binary_argstr('zkctl')+' -zk.cfg [email protected]'+utils.hostname+':%s start' % environment.topo_server().zk_ports)
    time.sleep(3)

    utils.kill_sub_process(querier)

    logging.debug("Checking %s", filename)
    fd = open(filename, "r")
    state = 0
    for line in fd:
      if line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
        stale = False
      elif line == "/zk/test_nj/vt/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
        stale = True
      else:
        self.fail('unexpected line: %s' % line)
      if state == 0:
        if stale:
          state = 1
      elif state == 1:
        if not stale:
          state = 2
      else:
        if stale:
          self.fail('unexpected stale state')
    self.assertEqual(state, 2)
    fd.close()

    utils.zkocc_kill(zkocc_14850)

    # check that after the server is gone, the python client fails correctly
    level = logging.getLogger().getEffectiveLevel()
    logging.getLogger().setLevel(logging.ERROR)
    try:
      zkocc_client.get("/zk/test_nj/vt/zkocc1/data1")
      self.fail('exception expected')
    except zkocc.ZkOccError as e:
      if not str(e).startswith("Cannot dial to any server, tried: "):
        self.fail('unexpected exception: %s', str(e))
    logging.getLogger().setLevel(level)
Esempio n. 26
0
def zk_ls(path):
    out, _ = run(environment.binary_argstr("zk") + " ls " + path, trap_output=True)
    return sorted(out.splitlines())
Esempio n. 27
0
def zk_teardown():
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run('%s -log_dir %s -zk.cfg [email protected]%s:%s teardown' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports), raise_on_error=False)
Esempio n. 28
0
def zk_cat(path):
    out, _ = run(environment.binary_argstr("zk") + " cat " + path, trap_output=True)
    return out
Esempio n. 29
0
def zk_teardown():
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  action = 'shutdown' if options.keep_logs else 'teardown'
  run('%s -log_dir %s -zk.cfg [email protected]%s:%s %s' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports, action), raise_on_error=False)
Esempio n. 30
0
    def test_sharding(self):

        shard_0_master.init_tablet("master", "test_keyspace", "-80")
        shard_0_replica.init_tablet("replica", "test_keyspace", "-80")
        shard_1_master.init_tablet("master", "test_keyspace", "80-")
        shard_1_replica.init_tablet("replica", "test_keyspace", "80-")

        utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)

        # run checks now before we start the tablets
        utils.validate_topology()

        # create databases, start the tablets, wait for them to start
        for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
            t.create_db("vt_test_keyspace")
            t.start_vttablet(wait_for_state=None)
        for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
            t.wait_for_vttablet_state("SERVING")

        # apply the schema on the first shard through vtctl, so all tablets
        # are the same.
        shard_0_master.mquery("vt_test_keyspace", create_vt_select_test.replace("\n", ""), write=True)
        shard_0_replica.mquery("vt_test_keyspace", create_vt_select_test.replace("\n", ""), write=True)

        # apply the schema on the second shard.
        shard_1_master.mquery("vt_test_keyspace", create_vt_select_test_reverse.replace("\n", ""), write=True)
        shard_1_replica.mquery("vt_test_keyspace", create_vt_select_test_reverse.replace("\n", ""), write=True)

        for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
            utils.run_vtctl(["ReloadSchema", t.tablet_alias])

        # start vtgate, we'll use it later
        utils.VtGate().start()

        for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
            t.reset_replication()
        utils.run_vtctl(["InitShardMaster", "test_keyspace/-80", shard_0_master.tablet_alias], auto_log=True)
        utils.run_vtctl(["InitShardMaster", "test_keyspace/80-", shard_1_master.tablet_alias], auto_log=True)

        # insert some values directly (db is RO after minority reparent)
        # FIXME(alainjobart) these values don't match the shard map
        utils.run_vtctl(["SetReadWrite", shard_0_master.tablet_alias])
        utils.run_vtctl(["SetReadWrite", shard_1_master.tablet_alias])
        shard_0_master.mquery(
            "vt_test_keyspace", "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True
        )
        shard_1_master.mquery(
            "vt_test_keyspace", "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True
        )

        utils.validate_topology(ping_tablets=True)

        utils.pause("Before the sql scatter query")

        # make sure the '1' value was written on first shard
        rows = shard_0_master.mquery("vt_test_keyspace", "select id, msg from vt_select_test order by id")
        self.assertEqual(rows, ((1, "test 1"),), "wrong mysql_query output: %s" % str(rows))

        utils.pause("After db writes")

        # throw in some schema validation step
        # we created the schema differently, so it should show
        utils.run_vtctl(["ValidateSchemaShard", "test_keyspace/-80"])
        utils.run_vtctl(["ValidateSchemaShard", "test_keyspace/80-"])
        out, err = utils.run_vtctl(["ValidateSchemaKeyspace", "test_keyspace"], trap_output=True, raise_on_error=False)
        if (
            "test_nj-0000062344 and test_nj-0000062346 disagree on schema "
            "for table vt_select_test:\nCREATE TABLE" not in err
            or "test_nj-0000062344 and test_nj-0000062347 disagree on schema "
            "for table vt_select_test:\nCREATE TABLE" not in err
        ):
            self.fail("wrong ValidateSchemaKeyspace output: " + err)

        # validate versions
        utils.run_vtctl(["ValidateVersionShard", "test_keyspace/-80"], auto_log=True)
        utils.run_vtctl(["ValidateVersionKeyspace", "test_keyspace"], auto_log=True)

        # show and validate permissions
        utils.run_vtctl(["GetPermissions", "test_nj-0000062344"], auto_log=True)
        utils.run_vtctl(["ValidatePermissionsShard", "test_keyspace/-80"], auto_log=True)
        utils.run_vtctl(["ValidatePermissionsKeyspace", "test_keyspace"], auto_log=True)

        if environment.topo_server().flavor() == "zookeeper":
            # and create zkns on this complex keyspace, make sure a few
            # files are created
            utils.run_vtctl(["ExportZknsForKeyspace", "test_keyspace"])
            out, err = utils.run(
                environment.binary_argstr("zk") + " ls -R /zk/test_nj/zk?s/vt/test_keysp*", trap_output=True
            )
            lines = out.splitlines()
            for base in ["-80", "80-"]:
                for db_type in ["master", "replica"]:
                    for sub_path in ["", ".vdns", "/0", "/vt.vdns"]:
                        expected = "/zk/test_nj/zkns/vt/test_keyspace/" + base + "/" + db_type + sub_path
                        if expected not in lines:
                            self.fail("missing zkns part:\n%s\nin:%s" % (expected, out))

        # connect to the tablets directly, make sure they know / validate
        # their own shard
        sql = "select id, msg from vt_select_test order by id"

        qr = shard_0_master.execute(sql)
        self.assertEqual(qr["Rows"], [["1", "test 1"]])

        qr = shard_1_master.execute(sql)
        self.assertEqual(qr["Rows"], [["10", "test 10"]])

        _, stderr = utils.run_vtctl(
            ["VtTabletExecute", "-keyspace", "test_keyspace", "-shard", "-90", shard_0_master.tablet_alias, sql],
            expect_fail=True,
        )
        self.assertIn("fatal: Shard mismatch, expecting -80, received -90", stderr)

        utils.vtgate.kill()
        tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])