コード例 #1
0
ファイル: keyspace_test.py プロジェクト: cihangir/vitess
def tearDownModule():
  if utils.options.skip_teardown:
    return

  global vtgate_server
  utils.vtgate_kill(vtgate_server)
  tablet.kill_tablets([shard_0_master, shard_0_replica,
                      shard_1_master, shard_1_replica])
  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_replica.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_replica.teardown_mysql(),
      unsharded_master.teardown_mysql(),
      unsharded_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
  unsharded_master.remove_tree()
  unsharded_replica.remove_tree()
コード例 #2
0
ファイル: zkocc_test.py プロジェクト: iamima/vitess
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start()
    qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    # a typical workstation will do 38-40k QPS, check we have more than 15k
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < 150000:
      self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
コード例 #3
0
ファイル: zkocc_test.py プロジェクト: lcwy220/vitess
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start(
        extra_args=['-cpu_profile', os.path.join(environment.tmproot,
                                                 'vtgate.pprof')])
    qpser = utils.run_bg(environment.binary_args('zkclient2') + [
        '-server', 'localhost:%u' % vtgate_port,
        '-mode', 'qps',
        '-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
        'test_nj', 'test_keyspace'])
    qpser.wait()

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < MIN_QPS * 10:
      self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
コード例 #4
0
ファイル: vtgatev2_test.py プロジェクト: bill2004158/vitess
def tearDownModule():
  global vtgate_server
  logging.debug("in tearDownModule")
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  utils.vtgate_kill(vtgate_server)
  tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
                       shard_1_replica])
  teardown_procs = [shard_0_master.teardown_mysql(),
                    shard_0_replica.teardown_mysql(),
                    shard_1_master.teardown_mysql(),
                    shard_1_replica.teardown_mysql(),
                   ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server_teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
コード例 #5
0
ファイル: tabletmanager.py プロジェクト: ZhuoRoger/vitess
  def test_vtgate(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    utils.run_vtctl('CreateShard -force test_keyspace/0')
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.mquery("", ["set global read_only = off"])
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()
    gate_proc = utils.vtgate_start()

    try:
      conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0)

      # _execute
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

      # _stream_execute
      (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
      count = 0
      while 1:
        r = conn._stream_next()
        if not r:
          break
        count += 1
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-rollback
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.rollback()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-commit
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.commit()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 5, "want 5, got %d" % (count))

      # close
      conn.close()
    finally:
      utils.vtgate_kill(gate_proc)
      tablet_62344.kill_vttablet()
コード例 #6
0
 def shutdown(self):
   utils.vtgate_kill(self.vtgate_server)
   tablet.kill_tablets(self.tablets)
   teardown_procs = [t.teardown_mysql() for t in self.tablets]
   utils.wait_procs(teardown_procs, raise_on_error=False)
   environment.topo_server().teardown()
   utils.kill_sub_processes()
   utils.remove_tmp_files()
   for t in self.tablets:
     t.remove_tree()
コード例 #7
0
ファイル: vtgatev3_test.py プロジェクト: asim/vitess
def tearDownModule():
  logging.debug("in tearDownModule")
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  utils.vtgate_kill(vtgate_server)
  keyspace_env.teardown()

  environment.topo_server().teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()
コード例 #8
0
ファイル: vtdb_test.py プロジェクト: nosix-me/vitess
 def setUp(self):
     global vtgate_server, vtgate_port
     self.shard_index = 0
     self.replica_tablet = shard_0_replica
     self.replica_tablet.kill_vttablet()
     self.replica_tablet.start_vttablet(auth=True)
     utils.vtgate_kill(vtgate_server)
     vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
     credentials_file_name = os.path.join(environment.vttop, "test", "test_data", "authcredentials_test.json")
     credentials_file = open(credentials_file_name, "r")
     credentials = json.load(credentials_file)
     self.user = str(credentials.keys()[0])
     self.password = str(credentials[self.user][0])
     self.secondary_password = str(credentials[self.user][1])
コード例 #9
0
ファイル: vtctld_test.py プロジェクト: acid009/vitess
def tearDownModule():
  if utils.options.skip_teardown:
    return

  utils.vtgate_kill(vtgate_server)

  teardown_procs = [t.teardown_mysql() for t in tablets]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server_teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  for t in tablets:
    t.remove_tree()
コード例 #10
0
def tearDownModule():
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  tablet.kill_tablets([master_tablet, replica_tablet])
  teardown_procs = [master_tablet.teardown_mysql(),
                    replica_tablet.teardown_mysql()]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server_teardown()
  utils.vtgate_kill(vtgate_server)
  utils.kill_sub_processes()
  utils.remove_tmp_files()
  master_tablet.remove_tree()
  replica_tablet.remove_tree()
コード例 #11
0
ファイル: client_test.py プロジェクト: egbonrelu/vitess-1
def tearDownModule():
  global vtgate_server
  global __tablets
  if utils.options.skip_teardown:
    return
  utils.vtgate_kill(vtgate_server)
  if __tablets is not None:
    tablet.kill_tablets(__tablets)
    teardown_procs = []
    for t in __tablets:
      teardown_procs.append(t.teardown_mysql())
    utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()

  if __tablets is not None:
    for t in __tablets:
      t.remove_tree()
コード例 #12
0
ファイル: demo.py プロジェクト: Eter365/vitess
def main():
  parser = optparse.OptionParser(usage="usage: %prog [options]")
  utils.add_options(parser)
  (options, args) = parser.parse_args()
  options.debug = True
  utils.set_options(options)
  env = keyspace_util.TestEnv()
  vtgate_server=None
  try:
    environment.topo_server().setup()
    env.launch(
        "user",
        shards=["-80", "80-"],
        ddls=[
            'create table user(user_id bigint, name varchar(128), primary key(user_id))',
            'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
            'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
            'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
            ],
        )
    env.launch(
        "lookup",
        ddls=[
            'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
            'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
            'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
            ],
        )
    utils.apply_vschema(vschema)
    vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
    utils.Vtctld().start()
    print "vtgate:", vtgate_port
    print "vtctld:", utils.vtctld.port
    utils.pause("the cluster is up, press enter to shut it down...")
  finally:
    utils.vtgate_kill(vtgate_server)
    env.teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()
    environment.topo_server().teardown()
コード例 #13
0
ファイル: custom_sharding.py プロジェクト: travis-ci/vitess
def tearDownModule():
  global vtgate_server

  if utils.options.skip_teardown:
    return

  utils.vtgate_kill(vtgate_server)
  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_rdonly.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_rdonly.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_rdonly.remove_tree()
  shard_1_master.remove_tree()
  shard_1_rdonly.remove_tree()
コード例 #14
0
ファイル: vertical_split.py プロジェクト: dvska/vitess
 def tearDown(self):
   self.vtgate_client.close()
   utils.vtgate_kill(self.vtgate_server)
コード例 #15
0
ファイル: vtgatev2_test.py プロジェクト: bill2004158/vitess
def restart_vtgate(extra_args={}):
  global vtgate_server, vtgate_port
  utils.vtgate_kill(vtgate_server)
  vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port, extra_args=extra_args)
コード例 #16
0
ファイル: vtgatev2_test.py プロジェクト: bill2004158/vitess
    try:
      results = vtgate_conn._execute(
          "select 1 from vt_insert_test", {},
          KEYSPACE_NAME, 'replica',
          keyranges=[self.keyrange])
    except Exception, e:
      self.fail("Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))

  def test_vtgate_restart_read(self):
    global vtgate_server, vtgate_port
    try:
      vtgate_conn = get_connection()
    except Exception, e:
      self.fail("Connection to vtgate failed with error %s" % (str(e)))
    utils.vtgate_kill(vtgate_server)
    with self.assertRaises(dbexceptions.OperationalError):
      vtgate_conn._execute(
          "select 1 from vt_insert_test", {},
          KEYSPACE_NAME, 'replica',
          keyranges=[self.keyrange])
    vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
    vtgate_conn = get_connection()
    try:
      results = vtgate_conn._execute(
          "select 1 from vt_insert_test", {},
          KEYSPACE_NAME, 'replica',
          keyranges=[self.keyrange])
    except Exception, e:
      self.fail("Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))
コード例 #17
0
ファイル: vtgate_test.py プロジェクト: qinbo/vitess
      self.fail("Connection to shard %s replica failed with error %s" % (vtdb_test.shard_names[self.shard_index], str(e)))
    self.replica_tablet.kill_vttablet()
    with self.assertRaises(dbexceptions.DatabaseError):
      replica_conn._execute("select 1 from vt_insert_test", {})
    proc = self.replica_tablet.start_vttablet()
    try:
      results = replica_conn._execute("select 1 from vt_insert_test", {})
    except Exception, e:
      self.fail("Communication with shard %s replica failed with error %s" % (vtdb_test.shard_names[self.shard_index], str(e)))

  def test_vtgate_restart_read(self):
    try:
      replica_conn = vtdb_test.get_connection(db_type='replica', shard_index=self.shard_index)
    except Exception, e:
      self.fail("Connection to shard %s replica failed with error %s" % (vtdb_test.shard_names[self.shard_index], str(e)))
    utils.vtgate_kill(vtdb_test.vtgate_server)
    with self.assertRaises(dbexceptions.OperationalError):
      replica_conn._execute("select 1 from vt_insert_test", {})
    vtdb_test.vtgate_server, vtdb_test.vtgate_port = utils.vtgate_start(vtdb_test.vtgate_port)
    try:
      results = replica_conn._execute("select 1 from vt_insert_test", {})
    except Exception, e:
      self.fail("Communication with shard %s replica failed with error %s" % (vtdb_test.shard_names[self.shard_index], str(e)))


  def test_tablet_restart_stream_execute(self):
    try:
      replica_conn = vtdb_test.get_connection(db_type='replica', shard_index=self.shard_index)
    except Exception, e:
      self.fail("Connection to %s replica failed with error %s" % (vtdb_test.shard_names[self.shard_index], str(e)))
    stream_cursor = cursor.StreamCursor(replica_conn)
コード例 #18
0
ファイル: zkocc_test.py プロジェクト: lcwy220/vitess
 def tearDown(self):
   utils.vtgate_kill(self.vtgate_zk)
コード例 #19
0
ファイル: zkocc_test.py プロジェクト: Carney/vitess
 def tearDown(self):
   utils.vtgate_kill(self.vtgate_zk)
   if environment.topo_server().flavor() == 'zookeeper':
     self.topo.close()
     utils.zkocc_kill(self.zkocc_server)
     utils.vtgate_kill(self.vtgate_zkocc)
コード例 #20
0
ファイル: tabletmanager.py プロジェクト: Acidburn0zzz/vitess
  def test_vtgate(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['CreateShard', 'test_keyspace/0'])
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.mquery("", ["set global read_only = off"])
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()
    gate_proc, gate_port = utils.vtgate_start()

    conn = vtgate.connect("localhost:%s"%(gate_port), "master", "test_keyspace", "0", 2.0)

    # _execute
    (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
    self.assertEqual(count, 4, "want 4, got %d" % (count))
    self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

    # _execute_batch
    queries = [
      "select * from vt_select_test where id = :id",
      "select * from vt_select_test where id = :id",
      ]
    bindvars = [
      {"id": 1},
      {"id": 2},
      ]
    rowsets = conn._execute_batch(queries, bindvars)
    self.assertEqual(rowsets[0][0][0][0], 1)
    self.assertEqual(rowsets[1][0][0][0], 2)

    # _stream_execute
    (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
    self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
    count = 0
    while 1:
      r = conn._stream_next()
      if not r:
        break
      count += 1
    self.assertEqual(count, 4, "want 4, got %d" % (count))

    # begin-rollback
    conn.begin()
    conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
    conn.rollback()
    (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
    self.assertEqual(count, 4, "want 4, got %d" % (count))

    # begin-commit
    conn.begin()
    conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
    conn.commit()
    (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
    self.assertEqual(count, 5, "want 5, got %d" % (count))

    # error on dml. We still need to get a transaction id
    conn.begin()
    with self.assertRaises(dbexceptions.IntegrityError):
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
    self.assertTrue(conn.session["ShardSessions"][0]["TransactionId"] != 0)
    conn.commit()

    # interleaving
    conn2 = vtgate.connect("localhost:%s"%(gate_port), "master", "test_keyspace", "0", 2.0)
    thd = threading.Thread(target=self._query_lots, args=(conn2,))
    thd.start()
    for i in xrange(250):
      (result, count, lastrow, fields) = conn._execute("select id from vt_select_test where id = 2", {})
      self.assertEqual(result, [(2,)])
      if i % 10 == 0:
        conn._stream_execute("select id from vt_select_test where id = 3", {})
        while 1:
          result = conn._stream_next()
          if not result:
            break
          self.assertEqual(result, (3,))
    thd.join()

    # close
    conn.close()

    utils.vtgate_kill(gate_proc)
    tablet_62344.kill_vttablet()
コード例 #21
0
ファイル: zkocc_test.py プロジェクト: iamima/vitess
 def tearDown(self):
   utils.zkocc_kill(self.zkocc_server)
   utils.vtgate_kill(self.vtgate_zk)
   utils.vtgate_kill(self.vtgate_zkocc)
コード例 #22
0
ファイル: zkocc_test.py プロジェクト: Abioy/vitess
 def tearDown(self):
   utils.vtgate_kill(self.vtgate_zk)
   if environment.topo_server_implementation == 'zookeeper':
     self.topo.close()
     utils.zkocc_kill(self.zkocc_server)
     utils.vtgate_kill(self.vtgate_zkocc)
コード例 #23
0
ファイル: sharded.py プロジェクト: Eter365/vitess
  def test_sharding(self):

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    # run checks now before we start the tablets
    utils.validate_topology()

    # create databases, start the tablets, wait for them to start
    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None)
    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.wait_for_vttablet_state('SERVING')

    # apply the schema on the first shard through vtctl, so all tablets
    # are the same (replication is not enabled yet, so allow_replication=false
    # is just there to be tested)
    utils.run_vtctl(['ApplySchema',
                     '-stop-replication',
                     '-sql=' + create_vt_select_test.replace("\n", ""),
                     shard_0_master.tablet_alias])
    utils.run_vtctl(['ApplySchema',
                     '-stop-replication',
                     '-sql=' + create_vt_select_test.replace("\n", ""),
                     shard_0_replica.tablet_alias])

    # start vtgate, we'll use it later
    vtgate_server, vtgate_port = utils.vtgate_start()

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
      t.reset_replication()
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/-80',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['InitShardMaster', 'test_keyspace/80-',
                     shard_1_master.tablet_alias], auto_log=True)

    # apply the schema on the second shard using a simple schema upgrade
    utils.run_vtctl(['ApplySchemaShard',
                     '-simple',
                     '-sql=' + create_vt_select_test_reverse.replace("\n", ""),
                     'test_keyspace/80-'])

    # insert some values directly (db is RO after minority reparent)
    # FIXME(alainjobart) these values don't match the shard map
    utils.run_vtctl(['SetReadWrite', shard_0_master.tablet_alias])
    utils.run_vtctl(['SetReadWrite', shard_1_master.tablet_alias])
    shard_0_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True)
    shard_1_master.mquery('vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True)

    utils.validate_topology(ping_tablets=True)

    utils.pause("Before the sql scatter query")

    # note the order of the rows is not guaranteed, as the go routines
    # doing the work can go out of order
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "10\ttest 10"])

    # write a value, re-read them all
    utils.vtclient2(3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True)
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "2\ttest 2",
                      "10\ttest 10"])

    # make sure the '2' value was written on first shard
    rows = shard_0_master.mquery('vt_test_keyspace', "select id, msg from vt_select_test order by id")
    self.assertEqual(rows, ((1, 'test 1'), (2, 'test 2'), ),
                     'wrong mysql_query output: %s' % str(rows))

    utils.pause("After db writes")

    # now use various topo servers and streaming or both for the same query
    self._check_rows(["Index\tid\tmsg",
                      "1\ttest 1",
                      "2\ttest 2",
                      "10\ttest 10"],
                     driver="vtdb-streaming")
    if environment.topo_server().flavor() == 'zookeeper':
      self._check_rows(["Index\tid\tmsg",
                        "1\ttest 1",
                        "2\ttest 2",
                        "10\ttest 10"],
                       driver="vtdb-zk")
      self._check_rows(["Index\tid\tmsg",
                        "1\ttest 1",
                        "2\ttest 2",
                        "10\ttest 10"],
                       driver="vtdb-zk-streaming")

    # make sure the schema checking works
    self._check_rows_schema_diff("vtdb")
    if environment.topo_server().flavor() == 'zookeeper':
      self._check_rows_schema_diff("vtdb-zk")

    # throw in some schema validation step
    # we created the schema differently, so it should show
    utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/-80'])
    utils.run_vtctl(['ValidateSchemaShard', 'test_keyspace/80-'])
    out, err = utils.run_vtctl(['ValidateSchemaKeyspace', 'test_keyspace'],
                               trap_output=True, raise_on_error=False)
    if 'test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err or \
       'test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE' not in err:
      self.fail('wrong ValidateSchemaKeyspace output: ' + err)

    # validate versions
    utils.run_vtctl(['ValidateVersionShard', 'test_keyspace/-80'],
                    auto_log=True)
    utils.run_vtctl(['ValidateVersionKeyspace', 'test_keyspace'], auto_log=True)

    # show and validate permissions
    utils.run_vtctl(['GetPermissions', 'test_nj-0000062344'], auto_log=True)
    utils.run_vtctl(['ValidatePermissionsShard', 'test_keyspace/-80'],
                    auto_log=True)
    utils.run_vtctl(['ValidatePermissionsKeyspace', 'test_keyspace'],
                    auto_log=True)

    if environment.topo_server().flavor() == 'zookeeper':
      # and create zkns on this complex keyspace, make sure a few files are created
      utils.run_vtctl(['ExportZknsForKeyspace', 'test_keyspace'])
      out, err = utils.run(environment.binary_argstr('zk')+' ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True)
      lines = out.splitlines()
      for base in ['-80', '80-']:
        for db_type in ['master', 'replica']:
          for sub_path in ['', '.vdns', '/0', '/vt.vdns']:
            expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
            if expected not in lines:
              self.fail('missing zkns part:\n%s\nin:%s' %(expected, out))

    # now try to connect using the python client and shard-aware connection
    # to both shards
    # first get the topology and check it
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                          "test_nj", 30.0)
    topology.read_keyspaces(vtgate_client)

    shard_0_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.-80.master:vt")
    if len(shard_0_master_addrs) != 1:
      self.fail('topology.get_host_port_by_name failed for "test_keyspace.-80.master:vt", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
    logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))

    # connect to shard -80
    conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0],
                                               shard_0_master_addrs[0][1]),
                                    "", "test_keyspace", "-80", 10.0)
    conn.dial()
    (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
    self.assertEqual(results, [(1, 'test 1'), (2, 'test 2'), ],
                     'wrong conn._execute output: %s' % str(results))

    # connect to shard 80-
    shard_1_master_addrs = topology.get_host_port_by_name(vtgate_client, "test_keyspace.80-.master:vt")
    conn = tablet3.TabletConnection("%s:%u" % (shard_1_master_addrs[0][0],
                                               shard_1_master_addrs[0][1]),
                                    "", "test_keyspace", "80-", 10.0)
    conn.dial()
    (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {})
    self.assertEqual(results, [(10, 'test 10'), ],
                     'wrong conn._execute output: %s' % str(results))
    vtgate_client.close()

    # try to connect with bad shard
    try:
      conn = tablet3.TabletConnection("localhost:%u" % shard_0_master.port,
                                      "", "test_keyspace", "-90", 10.0)
      conn.dial()
      self.fail('expected an exception')
    except Exception as e:
      if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
        self.fail('unexpected exception: ' + str(e))

    utils.vtgate_kill(vtgate_server)
    tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
                         shard_1_replica])
コード例 #24
0
ファイル: secure.py プロジェクト: bigrats/vitess
  def test_secure(self):
    zkocc_server = utils.zkocc_start()

    # start the tablets
    shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
                                  key=cert_dir + "/vt-server-key.pem")
    shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
                                 key=cert_dir + "/vt-server-key.pem",
                                 repl_extra_flags={
        'flags': "2048",
        'ssl-ca': cert_dir + "/ca-cert.pem",
        'ssl-cert': cert_dir + "/client-cert.pem",
        'ssl-key': cert_dir + "/client-key.pem",
        })

    # Reparent using SSL
    for t in [shard_0_master, shard_0_slave]:
      t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True)

    # then get the topology and check it
    zkocc_client = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base,
                                         "test_nj", 30.0)
    topology.read_keyspaces(zkocc_client)

    shard_0_master_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vts")
    if len(shard_0_master_addrs) != 1:
      self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
    if shard_0_master_addrs[0][2] != True:
      self.fail('topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted')
    logging.debug("shard 0 master addrs: %s", " ".join(["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))

    # make sure asking for optionally secure connections works too
    auto_addrs = topology.get_host_port_by_name(zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
    if auto_addrs != shard_0_master_addrs:
      self.fail('topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs)))

    # try to connect with regular client
    try:
      conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
                                      "", "test_keyspace", "0", 10.0)
      conn.dial()
      self.fail("No exception raised to secure port")
    except dbexceptions.FatalError as e:
      if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
        self.fail("Unexpected exception: %s" % str(e))

    sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
    if sconn != 0:
      self.fail("unexpected conns %s" % sconn)

    # connect to encrypted port
    conn = tablet3.TabletConnection("%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
                                    "", "test_keyspace", "0", 5.0, encrypted=True)
    conn.dial()
    (results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {})
    self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))

    sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
    if sconn != 1:
      self.fail("unexpected conns %s" % sconn)
    saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
    if saccept == 0:
      self.fail("unexpected accepts %s" % saccept)

    # trigger a time out on a secure connection, see what exception we get
    try:
      conn._execute("select sleep(100) from dual", {})
      self.fail("No timeout exception")
    except dbexceptions.TimeoutError as e:
      logging.debug("Got the right exception for SSL timeout: %s", str(e))

    # start a vtgate to connect to that tablet
    gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
        tablet_bson_encrypted=True,
        cert=cert_dir + "/vt-server-cert.pem",
        key=cert_dir + "/vt-server-key.pem")

    # try to connect to vtgate with regular client
    timeout = 2.0
    try:
      conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
                               timeout)
      self.fail("No exception raised to VTGate secure port")
    except dbexceptions.OperationalError as e:
      exception_type = e.args[2]
      exception_msg = str(e.args[2][0][0])
      self.assertIsInstance(exception_type, dbexceptions.FatalError,
                            "unexpected exception type")
      if not exception_msg.startswith('Unexpected EOF in handshake to'):
        self.fail("Unexpected exception message: %s" % exception_msg)

    sconn = utils.get_vars(gate_port)["SecureConnections"]
    if sconn != 0:
      self.fail("unexpected conns %s" % sconn)

    # connect to vtgate with encrypted port
    conn = vtgatev2.connect(["localhost:%s" % (gate_secure_port),],
                             timeout, encrypted=True)
    (results, rowcount, lastrowid, fields) = conn._execute(
        "select 1 from dual",
        {},
        "test_keyspace",
        "master",
        keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
    self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
    self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
    self.assertEqual(results, [(1,),], 'wrong conn._execute output: %s' % str(results))

    sconn = utils.get_vars(gate_port)["SecureConnections"]
    if sconn != 1:
      self.fail("unexpected conns %s" % sconn)
    saccept = utils.get_vars(gate_port)["SecureAccepts"]
    if saccept == 0:
      self.fail("unexpected accepts %s" % saccept)

    # trigger a time out on a vtgate secure connection, see what exception we get
    try:
      conn._execute("select sleep(4) from dual",
                    {},
                    "test_keyspace",
                    "master",
                    keyranges=[keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),])
      self.fail("No timeout exception")
    except dbexceptions.TimeoutError as e:
      logging.debug("Got the right exception for SSL timeout: %s", str(e))
    conn.close()
    utils.vtgate_kill(gate_proc)

    # kill everything
    utils.kill_sub_process(zkocc_server)