Beispiel #1
0
 def setUp(self):
   utils.zk_wipe()
   self.zkocc_server = utils.zkocc_start()
   # the default topo implementation for vtgate is zookeeper
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
   self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
   self.topo = zkocc.ZkOccConnection("localhost:%u" % utils.zkocc_port_base, 'test_nj', 30)
   self.topo.dial()
Beispiel #2
0
 def setUp(self):
   environment.topo_server_wipe()
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
   if environment.topo_server_implementation == 'zookeeper':
     self.zkocc_server = utils.zkocc_start()
     self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
     self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.zkocc_port_base, 'test_nj', 30)
     self.topo.dial()
Beispiel #3
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  setup_sharded_keyspace()
  setup_unsharded_keyspace()
  vtgate_server, vtgate_port = utils.vtgate_start()
Beispiel #4
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  setup_sharded_keyspace()
  setup_unsharded_keyspace()
  vtgate_server, vtgate_port = utils.vtgate_start()
Beispiel #5
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', USER_KEYSACE])
  utils.run_vtctl(['CreateKeyspace', LOOKUP_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', USER_KEYSACE,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=USER_KEYSACE, shard='-80')
  shard_1_master.init_tablet('master', keyspace=USER_KEYSACE, shard='80-')
  lookup_master.init_tablet('master', keyspace=LOOKUP_KEYSPACE, shard='0')

  for t in [shard_0_master, shard_1_master]:
    t.create_db('vt_user_keyspace')
    t.mquery('vt_user_keyspace', create_vt_user)
    t.mquery('vt_user_keyspace', create_vt_user2)
    t.mquery('vt_user_keyspace', create_vt_user_extra)
    t.mquery('vt_user_keyspace', create_vt_music)
    t.mquery('vt_user_keyspace', create_vt_music_extra)
    t.start_vttablet(wait_for_state='SERVING')
    utils.run_vtctl(['SetReadWrite', t.tablet_alias])
  lookup_master.create_db('vt_lookup_keyspace')
  lookup_master.mquery('vt_lookup_keyspace', create_vt_user_idx)
  lookup_master.mquery('vt_lookup_keyspace', create_music_user_map)
  lookup_master.mquery('vt_lookup_keyspace', create_name_user2_map)
  lookup_master.start_vttablet(wait_for_state='SERVING')
  utils.run_vtctl(['SetReadWrite', lookup_master.tablet_alias])

  utils.run_vtctl(['RebuildKeyspaceGraph', USER_KEYSACE], auto_log=True)
  utils.run_vtctl(['RebuildKeyspaceGraph', LOOKUP_KEYSPACE], auto_log=True)

  vtgate_server, vtgate_port = utils.vtgate_start(schema=schema)
Beispiel #6
0
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start()
    qpser = utils.run_bg(environment.binary_path('zkclient2')+' -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    # a typical workstation will do 38-40k QPS, check we have more than 10k
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < 100000:
      self.fail('QPS is too low: %u < 10000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
Beispiel #7
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', USER_KEYSACE])
  utils.run_vtctl(['CreateKeyspace', LOOKUP_KEYSPACE])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', USER_KEYSACE,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=USER_KEYSACE, shard='-80')
  shard_1_master.init_tablet('master', keyspace=USER_KEYSACE, shard='80-')
  lookup_master.init_tablet('master', keyspace=LOOKUP_KEYSPACE, shard='0')

  for t in [shard_0_master, shard_1_master]:
    t.create_db('vt_user_keyspace')
    t.mquery('vt_user_keyspace', create_vt_user)
    t.mquery('vt_user_keyspace', create_vt_user2)
    t.mquery('vt_user_keyspace', create_vt_user_extra)
    t.mquery('vt_user_keyspace', create_vt_music)
    t.mquery('vt_user_keyspace', create_vt_music_extra)
    t.start_vttablet(wait_for_state='SERVING')
    utils.run_vtctl(['SetReadWrite', t.tablet_alias])
  lookup_master.create_db('vt_lookup_keyspace')
  lookup_master.mquery('vt_lookup_keyspace', create_vt_user_idx)
  lookup_master.mquery('vt_lookup_keyspace', create_music_user_map)
  lookup_master.mquery('vt_lookup_keyspace', create_name_user2_map)
  lookup_master.start_vttablet(wait_for_state='SERVING')
  utils.run_vtctl(['SetReadWrite', lookup_master.tablet_alias])

  utils.run_vtctl(['RebuildKeyspaceGraph', USER_KEYSACE], auto_log=True)
  utils.run_vtctl(['RebuildKeyspaceGraph', LOOKUP_KEYSPACE], auto_log=True)

  vtgate_server, vtgate_port = utils.vtgate_start(schema=schema)
Beispiel #8
0
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start(
        extra_args=['-cpu_profile', os.path.join(environment.tmproot,
                                                 'vtgate.pprof')])
    qpser = utils.run_bg(environment.binary_args('zkclient2') + [
        '-server', 'localhost:%u' % vtgate_port,
        '-mode', 'qps',
        '-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
        'test_nj', 'test_keyspace'])
    qpser.wait()

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < MIN_QPS * 10:
      self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
Beispiel #9
0
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start(
        extra_args=['-cpu_profile', os.path.join(environment.tmproot,
                                                 'vtgate.pprof')])
    qpser = utils.run_bg(environment.binary_args('zkclient2') + [
        '-server', 'localhost:%u' % vtgate_port,
        '-mode', 'qps',
        '-zkclient_cpu_profile', os.path.join(environment.tmproot, 'zkclient2.pprof'),
        'test_nj', 'test_keyspace'])
    qpser.wait()

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < MIN_QPS * 10:
      self.fail('QPS is too low: %u < %u' % (rpcCalls / 10, MIN_QPS))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
Beispiel #10
0
  def test_vtgate_qps(self):
    # create the topology
    utils.run_vtctl('CreateKeyspace test_keyspace')
    t = tablet.Tablet(tablet_uid=1, cell="nj")
    t.init_tablet("master", "test_keyspace", "0")
    t.update_addrs()
    utils.run_vtctl('RebuildShardGraph test_keyspace/0', auto_log=True)
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    # start vtgate and the qps-er
    vtgate_proc, vtgate_port = utils.vtgate_start()
    qpser = utils.run_bg(utils.vtroot+'/bin/zkclient2 -server localhost:%u -mode qps2 test_nj test_keyspace' % vtgate_port)
    time.sleep(10)
    utils.kill_sub_process(qpser)

    # get the vtgate vars, make sure we have what we need
    v = utils.get_vars(vtgate_port)

    # some checks on performance / stats
    # a typical workstation will do 38-40k QPS, check we have more than 15k
    rpcCalls = v['TopoReaderRpcQueryCount']['test_nj']
    if rpcCalls < 150000:
      self.fail('QPS is too low: %u < 15000' % (rpcCalls / 10))
    else:
      logging.debug("Recorded qps: %u", rpcCalls / 10)
    utils.vtgate_kill(vtgate_proc)
 def set_up(self):
   try:
     environment.topo_server_setup()
     utils.wait_procs([t.init_mysql() for t in self.tablets])
     utils.run_vtctl(['CreateKeyspace', self.keyspace])
     utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
     for t in self.tablets:
       t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     for t in self.tablets:
       t.create_db('vt_' + self.keyspace)
       t.mquery(t.dbname, create_table)
       t.start_vttablet(wait_for_state=None)
     for t in self.tablets:
       t.wait_for_vttablet_state('SERVING')
     for t in self.tablets:
       if t.type == "master":
         utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     self.vtgate_server, self.vtgate_port = utils.vtgate_start()
     vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
     topology.read_topology(vtgate_client)
   except:
     self.shutdown()
     raise
Beispiel #12
0
 def set_up(self):
     try:
         environment.topo_server_setup()
         utils.wait_procs([t.init_mysql() for t in self.tablets])
         utils.run_vtctl(['CreateKeyspace', self.keyspace])
         utils.run_vtctl([
             'SetKeyspaceShardingInfo', '-force', self.keyspace,
             'keyspace_id', 'uint64'
         ])
         for t in self.tablets:
             t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
         utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
                         auto_log=True)
         for t in self.tablets:
             t.create_db('vt_' + self.keyspace)
             t.mquery(t.dbname, create_table)
             t.start_vttablet(wait_for_state=None)
         for t in self.tablets:
             t.wait_for_vttablet_state('SERVING')
         for t in self.tablets:
             if t.type == "master":
                 utils.run_vtctl([
                     'ReparentShard', '-force',
                     self.keyspace + '/' + t.shard, t.tablet_alias
                 ],
                                 auto_log=True)
         utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
                         auto_log=True)
         self.vtgate_server, self.vtgate_port = utils.vtgate_start()
         vtgate_client = zkocc.ZkOccConnection(
             "localhost:%u" % self.vtgate_port, "test_nj", 30.0)
         topology.read_topology(vtgate_client)
     except:
         self.shutdown()
         raise
 def set_up(self):
   try:
     environment.topo_server().setup()
     utils.wait_procs([t.init_mysql() for t in self.tablets])
     utils.run_vtctl(['CreateKeyspace', self.keyspace])
     utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
     for t in self.tablets:
       t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     for t in self.tablets:
       t.create_db('vt_' + self.keyspace)
       t.start_vttablet(
         wait_for_state=None,
         extra_args=['-queryserver-config-schema-reload-time', '1'],
       )
     for t in self.tablets:
       t.wait_for_vttablet_state('SERVING')
     for t in self.tablets:
       if t.type == "master":
         utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     if self.schema:
       utils.run_vtctl(['ApplySchemaKeyspace', '-simple', '-sql', self.schema, self.keyspace])
     if self.vschema:
       if self.vschema[0] == '{':
         utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
       else:
         utils.run_vtctl(['ApplyVSchema', "-vschema_file", self.vschema])
     self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='500s', vtport=self.vtgate_port)
     vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
     topology.read_topology(vtgate_client)
   except:
     self.shutdown()
     raise
Beispiel #14
0
    def setUpClass(klass):
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        utils.run_vtctl([
            'CreateKeyspace', '--served-from',
            'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
            'redirected_keyspace'
        ])

        shard_0_master.init_tablet('master', 'test_keyspace', '-80')
        shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
        shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
        shard_1_master.init_tablet('master', 'test_keyspace', '80-')
        shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
        idle.init_tablet('idle')
        scrap.init_tablet('idle')

        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                        auto_log=True)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                        auto_log=True)

        for t in [shard_0_master, shard_1_master, shard_1_replica]:
            t.create_db('vt_test_keyspace')
            t.start_vttablet(extra_args=vtctld.process_args())
        shard_0_replica.create_db('vt_test_keyspace')
        shard_0_replica.start_vttablet(extra_args=vtctld.process_args(),
                                       target_tablet_type='replica',
                                       wait_for_state='NOT_SERVING')

        for t in scrap, idle, shard_0_spare:
            t.start_vttablet(wait_for_state='NOT_SERVING',
                             extra_args=vtctld.process_args())

        scrap.scrap()

        for t in [
                shard_0_master, shard_0_replica, shard_0_spare, shard_1_master,
                shard_1_replica, idle, scrap
        ]:
            t.reset_replication()
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/-80',
            shard_0_master.tablet_alias
        ],
                        auto_log=True)
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/80-',
            shard_1_master.tablet_alias
        ],
                        auto_log=True)
        shard_0_replica.wait_for_vttablet_state('SERVING')

        # run checks now before we start the tablets
        utils.validate_topology()

        # start a vtgate server too
        global vtgate_server, vtgate_port
        vtgate_server, vtgate_port = utils.vtgate_start(
            cache_ttl='0s', extra_args=vtctld.process_args())
Beispiel #15
0
def setUpModule():
    global vtgate_server
    global vtgate_port
    global vtgate_socket_file
    global master_start_position

    try:
        environment.topo_server().setup()

        # start mysql instance external to the test
        setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
        utils.wait_procs(setup_procs)

        # Start up a master mysql and vttablet
        logging.debug('Setting up tablets')
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        master_tablet.init_tablet('master', 'test_keyspace', '0')
        replica_tablet.init_tablet('replica', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
        utils.validate_topology()
        master_tablet.create_db('vt_test_keyspace')
        master_tablet.create_db('other_database')
        replica_tablet.create_db('vt_test_keyspace')
        replica_tablet.create_db('other_database')

        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

        vtgate_socket_file = environment.tmproot + '/vtgate.sock'
        vtgate_server, vtgate_port = utils.vtgate_start(
            socket_file=vtgate_socket_file)

        master_tablet.start_vttablet()
        replica_tablet.start_vttablet()
        utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
        utils.check_db_read_write(master_tablet.tablet_uid)

        for t in [master_tablet, replica_tablet]:
            t.reset_replication()
        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/0',
            master_tablet.tablet_alias
        ],
                        auto_log=True)

        # reset counter so tests don't assert
        tablet.Tablet.tablets_running = 0

        master_start_position = _get_master_current_position()
        master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
        master_tablet.mquery('vt_test_keyspace', _create_vt_a)
        master_tablet.mquery('vt_test_keyspace', _create_vt_b)

        utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
        utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

    except:
        tearDownModule()
        raise
Beispiel #16
0
  def setUpClass(klass):
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl(['CreateKeyspace',
                     '--served-from', 'master:test_keyspace,replica:test_keyspace,rdonly:test_keyspace',
                     'redirected_keyspace'])

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('spare',   'test_keyspace', '-80')
    shard_0_spare.init_tablet(  'spare',   'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
    idle.init_tablet('idle')
    scrap.init_tablet('idle')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)
    utils.run_vtctl(['RebuildKeyspaceGraph', 'redirected_keyspace'],
                    auto_log=True)

    # start running all the tablets
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())
    shard_0_replica.create_db('vt_test_keyspace')
    shard_0_replica.start_vttablet(extra_args=utils.vtctld.process_args(),
                                   target_tablet_type='replica',
                                   wait_for_state=None)

    for t in scrap, idle, shard_0_spare:
      t.start_vttablet(wait_for_state=None,
                       extra_args=utils.vtctld.process_args())

    # wait for the right states
    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.wait_for_vttablet_state('SERVING')
    for t in [scrap, idle, shard_0_replica, shard_0_spare]:
      t.wait_for_vttablet_state('NOT_SERVING')

    scrap.scrap()

    for t in [shard_0_master, shard_0_replica, shard_0_spare,
              shard_1_master, shard_1_replica, idle, scrap]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
                     shard_1_master.tablet_alias], auto_log=True)
    shard_0_replica.wait_for_vttablet_state('SERVING')

    # run checks now before we start the tablets
    utils.validate_topology()

    # start a vtgate server too
    global vtgate_server, vtgate_port
    vtgate_server, vtgate_port = utils.vtgate_start(
        cache_ttl='0s', extra_args=utils.vtctld.process_args())
Beispiel #17
0
def setup_tablets():
    global vtgate_server
    global vtgate_port

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    utils.run_vtctl([
        'SetKeyspaceShardingInfo', '-force', 'test_keyspace', 'keyspace_id',
        'uint64'
    ])
    shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='-80')
    shard_0_replica.init_tablet('replica',
                                keyspace='test_keyspace',
                                shard='-80')
    shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='80-')
    shard_1_replica.init_tablet('replica',
                                keyspace='test_keyspace',
                                shard='80-')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.create_db('vt_test_keyspace')
        t.mquery(shard_0_master.dbname, create_vt_insert_test)
        t.mquery(shard_0_master.dbname, create_vt_a)
        t.start_vttablet(wait_for_state=None)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.wait_for_vttablet_state('SERVING')

    utils.run_vtctl([
        'ReparentShard', '-force', 'test_keyspace/-80',
        shard_0_master.tablet_alias
    ],
                    auto_log=True)
    utils.run_vtctl([
        'ReparentShard', '-force', 'test_keyspace/80-',
        shard_1_master.tablet_alias
    ],
                    auto_log=True)

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

    utils.check_srv_keyspace(
        'test_nj', 'test_keyspace', 'Partitions(master): -80 80-\n' +
        'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')

    vtgate_server, vtgate_port = utils.vtgate_start()
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                          "test_nj", 30.0)
    topology.read_topology(vtgate_client)
Beispiel #18
0
  def test_vtgate(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    utils.run_vtctl('CreateShard -force test_keyspace/0')
    tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
    utils.run_vtctl('RebuildShardGraph test_keyspace/0')
    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
    utils.validate_topology()

    # if these statements don't run before the tablet it will wedge waiting for the
    # db to become accessible. this is more a bug than a feature.
    tablet_62344.mquery("", ["set global read_only = off"])
    tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                          self._populate_vt_select_test)

    tablet_62344.start_vttablet()
    gate_proc = utils.vtgate_start()

    try:
      conn = vtgate.connect("localhost:%s"%(utils.vtgate_port_base), "master", "test_keyspace", "0", 2.0)

      # _execute
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

      # _stream_execute
      (result, count, lastrow, fields) = conn._stream_execute("select * from vt_select_test", {})
      self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
      count = 0
      while 1:
        r = conn._stream_next()
        if not r:
          break
        count += 1
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-rollback
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.rollback()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 4, "want 4, got %d" % (count))

      # begin-commit
      conn.begin()
      conn._execute("insert into vt_select_test values(:id, :msg)", {"id": 5, "msg": "test4"})
      conn.commit()
      (result, count, lastrow, fields) = conn._execute("select * from vt_select_test", {})
      self.assertEqual(count, 5, "want 5, got %d" % (count))

      # close
      conn.close()
    finally:
      utils.vtgate_kill(gate_proc)
      tablet_62344.kill_vttablet()
Beispiel #19
0
def setUpModule():
  global vtgate_server
  global vtgate_port
  global vtgate_socket_file
  global master_start_position

  try:
    environment.topo_server_setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()
                   ]
    utils.wait_procs(setup_procs)

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    master_tablet.create_db('vt_test_keyspace')
    master_tablet.create_db('other_database')
    replica_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('other_database')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

    vtgate_socket_file = environment.tmproot + '/vtgate.sock'
    vtgate_server, vtgate_port = utils.vtgate_start(socket_file=vtgate_socket_file)

    master_tablet.start_vttablet()
    replica_tablet.start_vttablet()
    utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
    utils.check_db_read_write(master_tablet.tablet_uid)

    for t in [master_tablet, replica_tablet]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0

    master_start_position = _get_master_current_position()
    master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
    master_tablet.mquery('vt_test_keyspace', _create_vt_a)
    master_tablet.mquery('vt_test_keyspace', _create_vt_b)

    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

  except:
    tearDownModule()
    raise
Beispiel #20
0
  def setUp(self):
    self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
    self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
                                               "test_nj", 30.0)
    self.vtgate_addrs = None
    if vtgate_protocol == VTGATE_PROTOCOL_V1BSON:
      global vtgate_addrs
      self.vtgate_addrs = {"_vt": ["localhost:%s"%(self.vtgate_port),]}

    self.insert_index = 0
Beispiel #21
0
def setup_tablets():
    global vtgate_server
    global vtgate_port

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
    utils.run_vtctl([
        'SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME, 'keyspace_id',
        'uint64'
    ])
    shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
    shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
    shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
    shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')

    utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.create_db('vt_test_keyspace')
        for create_table in create_tables:
            t.mquery(shard_0_master.dbname, create_table)
        t.start_vttablet(wait_for_state=None)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.wait_for_vttablet_state('SERVING')

    utils.run_vtctl([
        'ReparentShard', '-force', KEYSPACE_NAME + '/-80',
        shard_0_master.tablet_alias
    ],
                    auto_log=True)
    utils.run_vtctl([
        'ReparentShard', '-force', KEYSPACE_NAME + '/80-',
        shard_1_master.tablet_alias
    ],
                    auto_log=True)

    utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

    utils.check_srv_keyspace(
        'test_nj', KEYSPACE_NAME, 'Partitions(master): -80 80-\n' +
        'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')

    vtgate_server, vtgate_port = utils.vtgate_start()
    # FIXME(shrutip): this should be removed once vtgate_cursor's
    # dependency on topology goes away.
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                          "test_nj", 30.0)
    topology.read_topology(vtgate_client)
Beispiel #22
0
  def setUp(self):
    self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
    self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
                                               "test_nj", 30.0)
    self.vtgate_addrs = None
    if client_type == VTGATE:
      self.vtgate_addrs = {"vt": ["localhost:%s"%(self.vtgate_port),]}

    self.insert_index = 0
    # Lowering the keyspace refresh throttle so things are testable.
    self.throttle_sleep_interval = 0.1
    topology.set_keyspace_fetch_throttle(0.01)
Beispiel #23
0
  def setUp(self):
    self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='0s')
    self.vtgate_client = zkocc.ZkOccConnection("localhost:%u"%self.vtgate_port,
                                               "test_nj", 30.0)
    self.vtgate_addrs = None
    if vtgate_protocol == VTGATE_PROTOCOL_V1BSON:
      global vtgate_addrs
      self.vtgate_addrs = {"_vt": ["localhost:%s"%(self.vtgate_port),]}

    self.insert_index = 0
    # Lowering the keyspace refresh throttle so things are testable.
    self.throttle_sleep_interval = 0.1
    topology.set_keyspace_fetch_throttle(0.01)
Beispiel #24
0
 def setUp(self):
     global vtgate_server, vtgate_port
     self.shard_index = 0
     self.replica_tablet = shard_0_replica
     self.replica_tablet.kill_vttablet()
     self.replica_tablet.start_vttablet(auth=True)
     utils.vtgate_kill(vtgate_server)
     vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
     credentials_file_name = os.path.join(environment.vttop, "test", "test_data", "authcredentials_test.json")
     credentials_file = open(credentials_file_name, "r")
     credentials = json.load(credentials_file)
     self.user = str(credentials.keys()[0])
     self.password = str(credentials[self.user][0])
     self.secondary_password = str(credentials[self.user][1])
Beispiel #25
0
def setup_tablets():
    global vtgate_server
    global vtgate_port

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
    utils.run_vtctl([
        'SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME, 'keyspace_id',
        'uint64'
    ])
    shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
    shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
    shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
    shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')

    utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.create_db('vt_test_keyspace')
        for create_table in create_tables:
            t.mquery(shard_0_master.dbname, create_table)
        t.start_vttablet(wait_for_state=None)

    for t in [
            shard_0_master, shard_0_replica, shard_1_master, shard_1_replica
    ]:
        t.wait_for_vttablet_state('SERVING')

    utils.run_vtctl([
        'ReparentShard', '-force', KEYSPACE_NAME + '/-80',
        shard_0_master.tablet_alias
    ],
                    auto_log=True)
    utils.run_vtctl([
        'ReparentShard', '-force', KEYSPACE_NAME + '/80-',
        shard_1_master.tablet_alias
    ],
                    auto_log=True)

    utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

    utils.check_srv_keyspace(
        'test_nj', KEYSPACE_NAME, 'Partitions(master): -80 80-\n' +
        'Partitions(replica): -80 80-\n' + 'TabletTypes: master,replica')

    vtgate_server, vtgate_port = utils.vtgate_start()
Beispiel #26
0
 def set_up(self):
     try:
         environment.topo_server().setup()
         utils.wait_procs([t.init_mysql() for t in self.tablets])
         utils.run_vtctl(['CreateKeyspace', self.keyspace])
         utils.run_vtctl([
             'SetKeyspaceShardingInfo', '-force', self.keyspace,
             'keyspace_id', 'uint64'
         ])
         for t in self.tablets:
             t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
         utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
                         auto_log=True)
         for t in self.tablets:
             t.create_db('vt_' + self.keyspace)
             t.start_vttablet(
                 wait_for_state=None,
                 extra_args=['-queryserver-config-schema-reload-time', '1'],
             )
         for t in self.tablets:
             t.wait_for_vttablet_state('SERVING')
         for t in self.tablets:
             if t.type == "master":
                 utils.run_vtctl([
                     'ReparentShard', '-force',
                     self.keyspace + '/' + t.shard, t.tablet_alias
                 ],
                                 auto_log=True)
         utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace],
                         auto_log=True)
         if self.schema:
             utils.run_vtctl([
                 'ApplySchemaKeyspace', '-simple', '-sql', self.schema,
                 self.keyspace
             ])
         if self.vschema:
             if self.vschema[0] == '{':
                 utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
             else:
                 utils.run_vtctl(
                     ['ApplyVSchema', "-vschema_file", self.vschema])
         self.vtgate_server, self.vtgate_port = utils.vtgate_start(
             cache_ttl='500s')
         vtgate_client = zkocc.ZkOccConnection(
             "localhost:%u" % self.vtgate_port, "test_nj", 30.0)
         topology.read_topology(vtgate_client)
     except:
         self.shutdown()
         raise
Beispiel #27
0
 def setUp(self):
   global vtgate_server, vtgate_port
   self.shard_index = 0
   self.replica_tablet = shard_0_replica
   self.replica_tablet.kill_vttablet()
   self.replica_tablet.start_vttablet(auth=True)
   utils.vtgate_kill(vtgate_server)
   vtgate_server, vtgate_port = utils.vtgate_start(auth=True)
   credentials_file_name = os.path.join(environment.vttop, 'test', 'test_data',
                                        'authcredentials_test.json')
   credentials_file = open(credentials_file_name, 'r')
   credentials = json.load(credentials_file)
   self.user = str(credentials.keys()[0])
   self.password = str(credentials[self.user][0])
   self.secondary_password = str(credentials[self.user][1])
Beispiel #28
0
def setUpModule():
  global vtgate_server, vtgate_port
  try:
    environment.topo_server().setup()
    setup_topology()

    # start mysql instance external to the test
    global __tablets
    setup_procs = []
    for tablet in __tablets:
      setup_procs.append(tablet.init_mysql())
    utils.wait_procs(setup_procs)
    create_db()
    start_tablets()
    vtgate_server, vtgate_port = utils.vtgate_start()
  except:
    tearDownModule()
    raise
Beispiel #29
0
def setUpModule():
    global vtgate_server, vtgate_port
    try:
        environment.topo_server().setup()
        setup_topology()

        # start mysql instance external to the test
        global __tablets
        setup_procs = []
        for tablet in __tablets:
            setup_procs.append(tablet.init_mysql())
        utils.wait_procs(setup_procs)
        create_db()
        start_tablets()
        vtgate_server, vtgate_port = utils.vtgate_start()
    except:
        tearDownModule()
        raise
Beispiel #30
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
  shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
  shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
  shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')

  utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.create_db('vt_test_keyspace')
    for create_table in create_tables:
      t.mquery(shard_0_master.dbname, create_table)
    t.start_vttablet(wait_for_state=None)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/-80',
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/80-',
                   shard_1_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', KEYSPACE_NAME,
                           'Partitions(master): -80 80-\n' +
                           'Partitions(replica): -80 80-\n' +
                           'TabletTypes: master,replica')

  vtgate_server, vtgate_port = utils.vtgate_start()
  # FIXME(shrutip): this should be removed once vtgate_cursor's
  # dependency on topology goes away.
  vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                        "test_nj", 30.0)
  topology.read_topology(vtgate_client)
Beispiel #31
0
  def setUpClass(klass):
    utils.run_vtctl('CreateKeyspace test_keyspace')

    shard_0_master.init_tablet( 'master',  'test_keyspace', '-80')
    shard_0_replica.init_tablet('spare', 'test_keyspace', '-80')
    shard_0_spare.init_tablet('spare', 'test_keyspace', '-80')
    shard_1_master.init_tablet( 'master',  'test_keyspace', '80-')
    shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
    idle.init_tablet('idle')
    scrap.init_tablet('idle')

    utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

    for t in [shard_0_master, shard_1_master, shard_1_replica]:
      t.create_db('vt_test_keyspace')
      t.start_vttablet(extra_args=vtctld.process_args())
    shard_0_replica.create_db('vt_test_keyspace')
    shard_0_replica.start_vttablet(extra_args=vtctld.process_args(),
                                   target_tablet_type='replica',
                                   wait_for_state='NOT_SERVING')

    for t in scrap, idle, shard_0_spare:
      t.start_vttablet(wait_for_state='NOT_SERVING',
                       extra_args=vtctld.process_args())

    scrap.scrap()

    for t in [shard_0_master, shard_0_replica, shard_0_spare,
              shard_1_master, shard_1_replica, idle, scrap]:
      t.reset_replication()
    utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
                    shard_0_master.tablet_alias, auto_log=True)
    utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
                    shard_1_master.tablet_alias, auto_log=True)
    shard_0_replica.wait_for_vttablet_state('SERVING')

    # run checks now before we start the tablets
    utils.validate_topology()

    # start a vtgate server too
    global vtgate_server, vtgate_port
    vtgate_server, vtgate_port = utils.vtgate_start(
        cache_ttl='0s', extra_args=vtctld.process_args())
Beispiel #32
0
def setUpModule():
  global vtgate_server
  global vtgate_port

  try:
    environment.topo_server().setup()

    setup_procs = [
        shard_0_master.init_mysql(),
        shard_0_rdonly.init_mysql(),
        shard_1_master.init_mysql(),
        shard_1_rdonly.init_mysql(),
        ]
    utils.Vtctld().start()
    vtgate_server, vtgate_port = utils.vtgate_start()
    utils.wait_procs(setup_procs)
  except:
    tearDownModule()
    raise
Beispiel #33
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', 'test_keyspace',
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='-80')
  shard_0_replica.init_tablet('replica', keyspace='test_keyspace', shard='-80')
  shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='80-')
  shard_1_replica.init_tablet('replica', keyspace='test_keyspace', shard='80-')

  utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.create_db('vt_test_keyspace')
    t.mquery(shard_0_master.dbname, create_vt_insert_test)
    t.mquery(shard_0_master.dbname, create_vt_a)
    t.start_vttablet(wait_for_state=None)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/-80',
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/80-',
                   shard_1_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', 'test_keyspace',
                           'Partitions(master): -80 80-\n' +
                           'Partitions(replica): -80 80-\n' +
                           'TabletTypes: master,replica')

  vtgate_server, vtgate_port = utils.vtgate_start()
  vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                        "test_nj", 30.0)
  topology.read_topology(vtgate_client)
Beispiel #34
0
def setUpModule():
    global vtgate_server
    global vtgate_port

    try:
        environment.topo_server_setup()

        # start mysql instance external to the test
        setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
        utils.wait_procs(setup_procs)

        # Start up a master mysql and vttablet
        logging.debug("Setting up tablets")
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        master_tablet.init_tablet('master', 'test_keyspace', '0')
        replica_tablet.init_tablet('replica', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
        utils.validate_topology()

        master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
        replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)

        vtgate_server, vtgate_port = utils.vtgate_start()

        master_tablet.start_vttablet(memcache=True, wait_for_state=None)
        replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
        master_tablet.wait_for_vttablet_state('SERVING')
        replica_tablet.wait_for_vttablet_state('SERVING')

        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/0',
            master_tablet.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        # restart the replica tablet so the stats are reset
        replica_tablet.kill_vttablet()
        replica_tablet.start_vttablet(memcache=True)
    except:
        tearDownModule()
        raise
Beispiel #35
0
def setUpModule():
  global keyspace_env
  global shard_0_master
  global shard_1_master
  global lookup_master
  global vtgate_server
  global vtgate_port
  logging.debug("in setUpModule")

  try:
    environment.topo_server().setup()
    logging.debug("Setting up tablets")
    keyspace_env = keyspace_util.TestEnv()
    keyspace_env.launch(
        "user",
        shards=["-80", "80-"],
        ddls=[
            create_vt_user,
            create_vt_user2,
            create_vt_user_extra,
            create_vt_music,
            create_vt_music_extra,
            ],
        )
    keyspace_env.launch(
        "lookup",
        ddls=[
            create_vt_user_idx,
            create_music_user_map,
            create_name_user2_map,
            ],
        )
    shard_0_master = keyspace_env.tablet_map["user.-80.master"]
    shard_1_master = keyspace_env.tablet_map["user.80-.master"]
    lookup_master = keyspace_env.tablet_map["lookup.0.master"]

    utils.apply_vschema(schema)
    vtgate_server, vtgate_port = utils.vtgate_start()
  except:
    tearDownModule()
    raise
Beispiel #36
0
def setUpModule():
    global keyspace_env
    global shard_0_master
    global shard_1_master
    global lookup_master
    global vtgate_server
    global vtgate_port
    logging.debug("in setUpModule")

    try:
        environment.topo_server().setup()
        logging.debug("Setting up tablets")
        keyspace_env = keyspace_util.TestEnv()
        keyspace_env.launch(
            "user",
            shards=["-80", "80-"],
            ddls=[
                create_vt_user,
                create_vt_user2,
                create_vt_user_extra,
                create_vt_music,
                create_vt_music_extra,
            ],
        )
        keyspace_env.launch(
            "lookup",
            ddls=[
                create_vt_user_idx,
                create_music_user_map,
                create_name_user2_map,
            ],
        )
        shard_0_master = keyspace_env.tablet_map["user.-80.master"]
        shard_1_master = keyspace_env.tablet_map["user.80-.master"]
        lookup_master = keyspace_env.tablet_map["lookup.0.master"]

        utils.apply_vschema(schema)
        vtgate_server, vtgate_port = utils.vtgate_start()
    except:
        tearDownModule()
        raise
Beispiel #37
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options]")
    utils.add_options(parser)
    (options, args) = parser.parse_args()
    options.debug = True
    utils.set_options(options)
    env = keyspace_util.TestEnv()
    vtgate_server = None
    try:
        environment.topo_server().setup()
        env.launch(
            "user",
            shards=["-80", "80-"],
            ddls=[
                'create table user(user_id bigint, name varchar(128), primary key(user_id))',
                'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
                'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
                'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
            ],
        )
        env.launch(
            "lookup",
            ddls=[
                'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
                'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
                'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
            ],
        )
        utils.apply_vschema(vschema)
        vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
        utils.Vtctld().start()
        print "vtgate:", vtgate_port
        print "vtctld:", utils.vtctld.port
        utils.pause("the cluster is up, press enter to shut it down...")
    finally:
        utils.vtgate_kill(vtgate_server)
        env.teardown()
        utils.kill_sub_processes()
        utils.remove_tmp_files()
        environment.topo_server().teardown()
def setUpModule():
  global vtgate_server
  global vtgate_port

  try:
    environment.topo_server_setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()]
    utils.wait_procs(setup_procs)

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
    utils.validate_topology()

    master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
    replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)

    vtgate_server, vtgate_port = utils.vtgate_start()

    master_tablet.start_vttablet(memcache=True, wait_for_state=None)
    replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
    master_tablet.wait_for_vttablet_state('SERVING')
    replica_tablet.wait_for_vttablet_state('SERVING')

    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)
    utils.validate_topology()

    # restart the replica tablet so the stats are reset
    replica_tablet.kill_vttablet()
    replica_tablet.start_vttablet(memcache=True)
  except:
    tearDownModule()
    raise
Beispiel #39
0
def main():
  parser = optparse.OptionParser(usage="usage: %prog [options]")
  utils.add_options(parser)
  (options, args) = parser.parse_args()
  options.debug = True
  utils.set_options(options)
  env = keyspace_util.TestEnv()
  vtgate_server=None
  try:
    environment.topo_server().setup()
    env.launch(
        "user",
        shards=["-80", "80-"],
        ddls=[
            'create table user(user_id bigint, name varchar(128), primary key(user_id))',
            'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
            'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
            'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
            ],
        )
    env.launch(
        "lookup",
        ddls=[
            'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
            'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
            'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
            ],
        )
    utils.apply_vschema(vschema)
    vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
    utils.Vtctld().start()
    print "vtgate:", vtgate_port
    print "vtctld:", utils.vtctld.port
    utils.pause("the cluster is up, press enter to shut it down...")
  finally:
    utils.vtgate_kill(vtgate_server)
    env.teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()
    environment.topo_server().teardown()
Beispiel #40
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', KEYSPACE_NAME])
  utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', KEYSPACE_NAME,
                   'keyspace_id', 'uint64'])
  shard_0_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='-80')
  shard_0_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='-80')
  shard_1_master.init_tablet('master', keyspace=KEYSPACE_NAME, shard='80-')
  shard_1_replica.init_tablet('replica', keyspace=KEYSPACE_NAME, shard='80-')

  utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME], auto_log=True)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.create_db('vt_test_keyspace')
    for create_table in create_tables:
      t.mquery(shard_0_master.dbname, create_table)
    t.start_vttablet(wait_for_state=None)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/-80',
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['ReparentShard', '-force', KEYSPACE_NAME+'/80-',
                   shard_1_master.tablet_alias], auto_log=True)

  utils.run_vtctl(['RebuildKeyspaceGraph', KEYSPACE_NAME],
                   auto_log=True)

  utils.check_srv_keyspace('test_nj', KEYSPACE_NAME,
                           'Partitions(master): -80 80-\n' +
                           'Partitions(replica): -80 80-\n' +
                           'TabletTypes: master,replica')

  vtgate_server, vtgate_port = utils.vtgate_start()
Beispiel #41
0
def setup_tablets():
    global vtgate_server
    global vtgate_port

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(["CreateKeyspace", "test_keyspace"])
    utils.run_vtctl(["SetKeyspaceShardingInfo", "-force", "test_keyspace", "keyspace_id", "uint64"])
    shard_0_master.init_tablet("master", keyspace="test_keyspace", shard="-80")
    shard_0_replica.init_tablet("replica", keyspace="test_keyspace", shard="-80")
    shard_1_master.init_tablet("master", keyspace="test_keyspace", shard="80-")
    shard_1_replica.init_tablet("replica", keyspace="test_keyspace", shard="80-")

    utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
        t.create_db("vt_test_keyspace")
        t.mquery(shard_0_master.dbname, create_vt_insert_test)
        t.mquery(shard_0_master.dbname, create_vt_a)
        t.start_vttablet(wait_for_state=None)

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
        t.wait_for_vttablet_state("SERVING")

    utils.run_vtctl(["ReparentShard", "-force", "test_keyspace/-80", shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(["ReparentShard", "-force", "test_keyspace/80-", shard_1_master.tablet_alias], auto_log=True)

    utils.run_vtctl(["RebuildKeyspaceGraph", "test_keyspace"], auto_log=True)

    utils.check_srv_keyspace(
        "test_nj",
        "test_keyspace",
        "Partitions(master): -80 80-\n" + "Partitions(replica): -80 80-\n" + "TabletTypes: master,replica",
    )

    vtgate_server, vtgate_port = utils.vtgate_start()
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port, "test_nj", 30.0)
    topology.read_topology(vtgate_client)
Beispiel #42
0
def setup_tablets():
  global vtgate_server
  global vtgate_port

  # Start up a master mysql and vttablet
  logging.debug("Setting up tablets")
  utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
  shard_0_master.init_tablet('master', keyspace='test_keyspace', shard='0')
  shard_0_replica.init_tablet('replica', keyspace='test_keyspace', shard='0')
  shard_1_master.init_tablet('master', keyspace='test_keyspace', shard='1')
  shard_1_replica.init_tablet('replica', keyspace='test_keyspace', shard='1')

  utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'], auto_log=True)
  utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/1'], auto_log=True)
  utils.validate_topology()
  shard_0_master.create_db(shard_0_master.dbname)
  shard_0_replica.create_db(shard_0_master.dbname)
  shard_1_master.create_db(shard_0_master.dbname)
  shard_1_replica.create_db(shard_0_master.dbname)

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.mquery(shard_0_master.dbname, create_vt_insert_test)
    t.mquery(shard_0_master.dbname, create_vt_a)

  utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'], auto_log=True)

  vtgate_server, vtgate_port = utils.vtgate_start()

  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.start_vttablet(wait_for_state=None)
  for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
    t.wait_for_vttablet_state('SERVING')

  utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                   shard_0_master.tablet_alias], auto_log=True)
  utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/1',
                   shard_1_master.tablet_alias], auto_log=True)
Beispiel #43
0
def setup_tablets():
    global vtgate_server
    global vtgate_port

    # Start up a master mysql and vttablet
    logging.debug("Setting up tablets")
    utils.run_vtctl(["CreateKeyspace", KEYSPACE_NAME])
    utils.run_vtctl(["SetKeyspaceShardingInfo", "-force", KEYSPACE_NAME, "keyspace_id", "uint64"])
    shard_0_master.init_tablet("master", keyspace=KEYSPACE_NAME, shard="-80")
    shard_0_replica.init_tablet("replica", keyspace=KEYSPACE_NAME, shard="-80")
    shard_1_master.init_tablet("master", keyspace=KEYSPACE_NAME, shard="80-")
    shard_1_replica.init_tablet("replica", keyspace=KEYSPACE_NAME, shard="80-")

    utils.run_vtctl(["RebuildKeyspaceGraph", KEYSPACE_NAME], auto_log=True)

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
        t.create_db("vt_test_keyspace")
        for create_table in create_tables:
            t.mquery(shard_0_master.dbname, create_table)
        t.start_vttablet(wait_for_state=None)

    for t in [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica]:
        t.wait_for_vttablet_state("SERVING")

    utils.run_vtctl(["ReparentShard", "-force", KEYSPACE_NAME + "/-80", shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(["ReparentShard", "-force", KEYSPACE_NAME + "/80-", shard_1_master.tablet_alias], auto_log=True)

    utils.run_vtctl(["RebuildKeyspaceGraph", KEYSPACE_NAME], auto_log=True)

    utils.check_srv_keyspace(
        "test_nj",
        KEYSPACE_NAME,
        "Partitions(master): -80 80-\n" + "Partitions(replica): -80 80-\n" + "TabletTypes: master,replica",
    )

    vtgate_server, vtgate_port = utils.vtgate_start()
Beispiel #44
0
def setUpModule():
  global vtgate_server, vtgate_port
  logging.debug("in setUpModule")
  try:
    environment.topo_server().setup()
    setup_topology()

    # start mysql instance external to the test
    global __tablets
    setup_procs = []
    for tablet in __tablets:
      setup_procs.append(tablet.init_mysql())
    utils.wait_procs(setup_procs)
    create_db()
    start_tablets()
    vtgate_server, vtgate_port = utils.vtgate_start()
    # FIXME(shrutip): this should be removed once vtgate_cursor's
    # dependency on topology goes away.
    vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                          "test_nj", 30.0)
    topology.read_topology(vtgate_client)
  except:
    tearDownModule()
    raise
Beispiel #45
0
def restart_vtgate(extra_args={}):
  global vtgate_server, vtgate_port
  utils.vtgate_kill(vtgate_server)
  vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port, extra_args=extra_args)
Beispiel #46
0
      self.fail("Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))

  def test_vtgate_restart_read(self):
    global vtgate_server, vtgate_port
    try:
      vtgate_conn = get_connection()
    except Exception, e:
      self.fail("Connection to vtgate failed with error %s" % (str(e)))
    utils.vtgate_kill(vtgate_server)
    with self.assertRaises(dbexceptions.OperationalError):
      vtgate_conn._execute(
          "select 1 from vt_insert_test", {},
          KEYSPACE_NAME, 'replica',
          keyranges=[self.keyrange])
    vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
    vtgate_conn = get_connection()
    try:
      results = vtgate_conn._execute(
          "select 1 from vt_insert_test", {},
          KEYSPACE_NAME, 'replica',
          keyranges=[self.keyrange])
    except Exception, e:
      self.fail("Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))

  def test_tablet_restart_stream_execute(self):
    try:
      vtgate_conn = get_connection()
    except Exception, e:
      self.fail("Connection to vtgate failed with error %s" % (str(e)))
Beispiel #47
0
    def test_vtgate(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        utils.run_vtctl(['CreateShard', 'test_keyspace/0'])
        tablet_62344.init_tablet('master', 'test_keyspace', '0', parent=False)
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
        utils.validate_topology()
        srvShard = utils.run_vtctl_json(
            ['GetSrvShard', 'test_nj', 'test_keyspace/0'])
        self.assertEqual(srvShard['MasterCell'], 'test_nj')

        # if these statements don't run before the tablet it will wedge
        # waiting for the db to become accessible. this is more a bug than
        # a feature.
        tablet_62344.mquery("", ["set global read_only = off"])
        tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
                              self._populate_vt_select_test)

        tablet_62344.start_vttablet()
        gate_proc, gate_port = utils.vtgate_start()

        conn = vtgate.connect("localhost:%s" % (gate_port), "master",
                              "test_keyspace", "0", 2.0)

        # _execute
        (result, count, lastrow,
         fields) = conn._execute("select * from vt_select_test", {})
        self.assertEqual(count, 4, "want 4, got %d" % (count))
        self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))

        # _execute_batch
        queries = [
            "select * from vt_select_test where id = :id",
            "select * from vt_select_test where id = :id",
        ]
        bindvars = [
            {
                "id": 1
            },
            {
                "id": 2
            },
        ]
        rowsets = conn._execute_batch(queries, bindvars)
        self.assertEqual(rowsets[0][0][0][0], 1)
        self.assertEqual(rowsets[1][0][0][0], 2)

        # _stream_execute
        (result, count, lastrow,
         fields) = conn._stream_execute("select * from vt_select_test", {})
        self.assertEqual(len(fields), 2, "want 2, got %d" % (len(fields)))
        count = 0
        while 1:
            r = conn._stream_next()
            if not r:
                break
            count += 1
        self.assertEqual(count, 4, "want 4, got %d" % (count))

        # begin-rollback
        conn.begin()
        conn._execute("insert into vt_select_test values(:id, :msg)", {
            "id": 5,
            "msg": "test4"
        })
        conn.rollback()
        (result, count, lastrow,
         fields) = conn._execute("select * from vt_select_test", {})
        self.assertEqual(count, 4, "want 4, got %d" % (count))

        # begin-commit
        conn.begin()
        conn._execute("insert into vt_select_test values(:id, :msg)", {
            "id": 5,
            "msg": "test4"
        })
        conn.commit()
        (result, count, lastrow,
         fields) = conn._execute("select * from vt_select_test", {})
        self.assertEqual(count, 5, "want 5, got %d" % (count))

        # error on dml. We still need to get a transaction id
        conn.begin()
        with self.assertRaises(dbexceptions.IntegrityError):
            conn._execute("insert into vt_select_test values(:id, :msg)", {
                "id": 5,
                "msg": "test4"
            })
        self.assertTrue(conn.session["ShardSessions"][0]["TransactionId"] != 0)
        conn.commit()

        # interleaving
        conn2 = vtgate.connect("localhost:%s" % (gate_port), "master",
                               "test_keyspace", "0", 2.0)
        thd = threading.Thread(target=self._query_lots, args=(conn2, ))
        thd.start()
        for i in xrange(250):
            (result, count, lastrow, fields) = conn._execute(
                "select id from vt_select_test where id = 2", {})
            self.assertEqual(result, [(2, )])
            if i % 10 == 0:
                conn._stream_execute(
                    "select id from vt_select_test where id = 3", {})
                while 1:
                    result = conn._stream_next()
                    if not result:
                        break
                    self.assertEqual(result, (3, ))
        thd.join()

        # close
        conn.close()

        utils.vtgate_kill(gate_proc)
        tablet_62344.kill_vttablet()
Beispiel #48
0
 def setUp(self):
   environment.topo_server().wipe()
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
Beispiel #49
0
    def test_sharding(self):

        shard_0_master.init_tablet('master', 'test_keyspace', '-80')
        shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
        shard_1_master.init_tablet('master', 'test_keyspace', '80-')
        shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')

        utils.run_vtctl('RebuildKeyspaceGraph test_keyspace', auto_log=True)

        # run checks now before we start the tablets
        utils.validate_topology()

        # create databases, start the tablets, wait for them to start
        for t in [
                shard_0_master, shard_0_replica, shard_1_master,
                shard_1_replica
        ]:
            t.create_db('vt_test_keyspace')
            t.start_vttablet(wait_for_state=None)
        for t in [
                shard_0_master, shard_0_replica, shard_1_master,
                shard_1_replica
        ]:
            t.wait_for_vttablet_state('SERVING')

        # apply the schema on the first shard through vtctl, so all tablets
        # are the same (replication is not enabled yet, so allow_replication=false
        # is just there to be tested)
        utils.run_vtctl([
            'ApplySchema', '-stop-replication',
            '-sql=' + create_vt_select_test.replace("\n", ""),
            shard_0_master.tablet_alias
        ])
        utils.run_vtctl([
            'ApplySchema', '-stop-replication',
            '-sql=' + create_vt_select_test.replace("\n", ""),
            shard_0_replica.tablet_alias
        ])

        if environment.topo_server_implementation == 'zookeeper':
            # start zkocc, we'll use it later, indirectly with the vtdb-zkocc driver
            zkocc_server = utils.zkocc_start()

        # start vtgate, we'll use it later
        vtgate_server, vtgate_port = utils.vtgate_start()

        for t in [
                shard_0_master, shard_0_replica, shard_1_master,
                shard_1_replica
        ]:
            t.reset_replication()
        utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
                        shard_0_master.tablet_alias,
                        auto_log=True)
        utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
                        shard_1_master.tablet_alias,
                        auto_log=True)

        # apply the schema on the second shard using a simple schema upgrade
        utils.run_vtctl([
            'ApplySchemaShard', '-simple',
            '-sql=' + create_vt_select_test_reverse.replace("\n", ""),
            'test_keyspace/80-'
        ])

        # insert some values directly (db is RO after minority reparent)
        # FIXME(alainjobart) these values don't match the shard map
        utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
        utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
        shard_0_master.mquery(
            'vt_test_keyspace',
            "insert into vt_select_test (id, msg) values (1, 'test 1')",
            write=True)
        shard_1_master.mquery(
            'vt_test_keyspace',
            "insert into vt_select_test (id, msg) values (10, 'test 10')",
            write=True)

        utils.validate_topology(ping_tablets=True)

        utils.pause("Before the sql scatter query")

        # note the order of the rows is not guaranteed, as the go routines
        # doing the work can go out of order
        self._check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])

        # write a value, re-read them all
        utils.vtclient2(
            3803,
            "/test_nj/test_keyspace/master",
            "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
            bindvars='{"keyspace_id": 2}',
            driver="vtdb",
            verbose=True)
        self._check_rows(
            ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])

        # make sure the '2' value was written on first shard
        rows = shard_0_master.mquery(
            'vt_test_keyspace',
            "select id, msg from vt_select_test order by id")
        self.assertEqual(rows, (
            (1, 'test 1'),
            (2, 'test 2'),
        ), 'wrong mysql_query output: %s' % str(rows))

        utils.pause("After db writes")

        # now use various topo servers and streaming or both for the same query
        self._check_rows(
            ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
            driver="vtdb-streaming")
        if environment.topo_server_implementation == 'zookeeper':
            self._check_rows(
                ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
                driver="vtdb-zk")
            self._check_rows(
                ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
                driver="vtdb-zk-streaming")
            self._check_rows(
                ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
                driver="vtdb-zkocc")
            self._check_rows(
                ["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
                driver="vtdb-zkocc-streaming")

        # make sure the schema checking works
        self._check_rows_schema_diff("vtdb")
        if environment.topo_server_implementation == 'zookeeper':
            self._check_rows_schema_diff("vtdb-zk")
            self._check_rows_schema_diff("vtdb-zkocc")

        # throw in some schema validation step
        # we created the schema differently, so it should show
        utils.run_vtctl('ValidateSchemaShard test_keyspace/-80')
        utils.run_vtctl('ValidateSchemaShard test_keyspace/80-')
        out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace',
                                   trap_output=True,
                                   raise_on_error=False)
        if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \
           "test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err:
            self.fail('wrong ValidateSchemaKeyspace output: ' + err)

        # validate versions
        utils.run_vtctl('ValidateVersionShard test_keyspace/-80',
                        auto_log=True)
        utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True)

        # show and validate permissions
        utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True)
        utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80',
                        auto_log=True)
        utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace',
                        auto_log=True)

        if environment.topo_server_implementation == 'zookeeper':
            # and create zkns on this complex keyspace, make sure a few files are created
            utils.run_vtctl('ExportZknsForKeyspace test_keyspace')
            out, err = utils.run(environment.binary_path('zk') +
                                 ' ls -R /zk/test_nj/zk?s/vt/test_keysp*',
                                 trap_output=True)
            lines = out.splitlines()
            for base in ['-80', '80-']:
                for db_type in ['master', 'replica']:
                    for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
                        expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
                        if expected not in lines:
                            self.fail('missing zkns part:\n%s\nin:%s' %
                                      (expected, out))

        # now try to connect using the python client and shard-aware connection
        # to both shards
        # first get the topology and check it
        vtgate_client = zkocc.ZkOccConnection("localhost:%u" % vtgate_port,
                                              "test_nj", 30.0)
        topology.read_keyspaces(vtgate_client)

        shard_0_master_addrs = topology.get_host_port_by_name(
            vtgate_client, "test_keyspace.-80.master:_vtocc")
        if len(shard_0_master_addrs) != 1:
            self.fail(
                'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
                % " ".join([
                    "%s:%u(%s)" % (h, p, str(e))
                    for (h, p, e) in shard_0_master_addrs
                ]))
        logging.debug(
            "shard 0 master addrs: %s", " ".join([
                "%s:%u(%s)" % (h, p, str(e))
                for (h, p, e) in shard_0_master_addrs
            ]))

        # connect to shard -80
        conn = tablet3.TabletConnection(
            "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
            "", "test_keyspace", "-80", 10.0)
        conn.dial()
        (results, rowcount, lastrowid, fields) = conn._execute(
            "select id, msg from vt_select_test order by id", {})
        self.assertEqual(results, [
            (1, 'test 1'),
            (2, 'test 2'),
        ], 'wrong conn._execute output: %s' % str(results))

        # connect to shard 80-
        shard_1_master_addrs = topology.get_host_port_by_name(
            vtgate_client, "test_keyspace.80-.master:_vtocc")
        conn = tablet3.TabletConnection(
            "%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]),
            "", "test_keyspace", "80-", 10.0)
        conn.dial()
        (results, rowcount, lastrowid, fields) = conn._execute(
            "select id, msg from vt_select_test order by id", {})
        self.assertEqual(results, [
            (10, 'test 10'),
        ], 'wrong conn._execute output: %s' % str(results))
        vtgate_client.close()

        # try to connect with bad shard
        try:
            conn = tablet3.TabletConnection(
                "localhost:%u" % shard_0_master.port, "", "test_keyspace",
                "-90", 10.0)
            conn.dial()
            self.fail('expected an exception')
        except Exception as e:
            if "fatal: Shard mismatch, expecting -80, received -90" not in str(
                    e):
                self.fail('unexpected exception: ' + str(e))

        utils.vtgate_kill(vtgate_server)
        if environment.topo_server_implementation == 'zookeeper':
            utils.kill_sub_process(zkocc_server)
        tablet.kill_tablets(
            [shard_0_master, shard_0_replica, shard_1_master, shard_1_replica])
Beispiel #50
0
def restart_vtgate(extra_args={}):
    global vtgate_server, vtgate_port
    utils.vtgate_kill(vtgate_server)
    vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port,
                                                    extra_args=extra_args)
Beispiel #51
0
 def setUp(self):
   environment.topo_server().wipe()
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
Beispiel #52
0
                "Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))

    def test_vtgate_restart_read(self):
        global vtgate_server, vtgate_port
        try:
            vtgate_conn = get_connection()
        except Exception, e:
            self.fail("Connection to vtgate failed with error %s" % (str(e)))
        utils.vtgate_kill(vtgate_server)
        with self.assertRaises(dbexceptions.OperationalError):
            vtgate_conn._execute("select 1 from vt_insert_test", {},
                                 KEYSPACE_NAME,
                                 'replica',
                                 keyranges=[self.keyrange])
        vtgate_server, vtgate_port = utils.vtgate_start(vtgate_port)
        vtgate_conn = get_connection()
        try:
            results = vtgate_conn._execute("select 1 from vt_insert_test", {},
                                           KEYSPACE_NAME,
                                           'replica',
                                           keyranges=[self.keyrange])
        except Exception, e:
            self.fail(
                "Communication with shard %s replica failed with error %s" %
                (shard_names[self.shard_index], str(e)))

    def test_tablet_restart_stream_execute(self):
        try:
            vtgate_conn = get_connection()
        except Exception, e:
Beispiel #53
0
    def test_secure(self):
        zkocc_server = utils.zkocc_start()

        # start the tablets
        shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
                                      key=cert_dir + "/vt-server-key.pem")
        shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
                                     key=cert_dir + "/vt-server-key.pem",
                                     repl_extra_flags={
                                         'flags': "2048",
                                         'ssl-ca': cert_dir + "/ca-cert.pem",
                                         'ssl-cert':
                                         cert_dir + "/client-cert.pem",
                                         'ssl-key':
                                         cert_dir + "/client-key.pem",
                                     })

        # Reparent using SSL
        for t in [shard_0_master, shard_0_slave]:
            t.reset_replication()
        utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
                        shard_0_master.tablet_alias,
                        auto_log=True)

        # then get the topology and check it
        zkocc_client = zkocc.ZkOccConnection(
            "localhost:%u" % environment.zkocc_port_base, "test_nj", 30.0)
        topology.read_keyspaces(zkocc_client)

        shard_0_master_addrs = topology.get_host_port_by_name(
            zkocc_client, "test_keyspace.0.master:_vts")
        if len(shard_0_master_addrs) != 1:
            self.fail(
                'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
                % " ".join([
                    "%s:%u(%s)" % (h, p, str(e))
                    for (h, p, e) in shard_0_master_addrs
                ]))
        if shard_0_master_addrs[0][2] != True:
            self.fail(
                'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
            )
        logging.debug(
            "shard 0 master addrs: %s", " ".join([
                "%s:%u(%s)" % (h, p, str(e))
                for (h, p, e) in shard_0_master_addrs
            ]))

        # make sure asking for optionally secure connections works too
        auto_addrs = topology.get_host_port_by_name(
            zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
        if auto_addrs != shard_0_master_addrs:
            self.fail(
                'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
                % (str(shard_0_master_addrs), str(auto_addrs)))

        # try to connect with regular client
        try:
            conn = tablet3.TabletConnection(
                "%s:%u" %
                (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "",
                "test_keyspace", "0", 10.0)
            conn.dial()
            self.fail("No exception raised to secure port")
        except dbexceptions.FatalError as e:
            if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
                self.fail("Unexpected exception: %s" % str(e))

        sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
        if sconn != 0:
            self.fail("unexpected conns %s" % sconn)

        # connect to encrypted port
        conn = tablet3.TabletConnection(
            "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
            "",
            "test_keyspace",
            "0",
            5.0,
            encrypted=True)
        conn.dial()
        (results, rowcount, lastrowid,
         fields) = conn._execute("select 1 from dual", {})
        self.assertEqual(results, [
            (1, ),
        ], 'wrong conn._execute output: %s' % str(results))

        sconn = utils.get_vars(shard_0_master.port)["SecureConnections"]
        if sconn != 1:
            self.fail("unexpected conns %s" % sconn)
        saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
        if saccept == 0:
            self.fail("unexpected accepts %s" % saccept)

        # trigger a time out on a secure connection, see what exception we get
        try:
            conn._execute("select sleep(100) from dual", {})
            self.fail("No timeout exception")
        except dbexceptions.TimeoutError as e:
            logging.debug("Got the right exception for SSL timeout: %s",
                          str(e))

        # start a vtgate to connect to that tablet
        gate_proc, gate_port, gate_secure_port = utils.vtgate_start(
            tablet_bson_encrypted=True,
            cert=cert_dir + "/vt-server-cert.pem",
            key=cert_dir + "/vt-server-key.pem")

        # try to connect to vtgate with regular client
        timeout = 2.0
        try:
            conn = vtgatev2.connect([
                "localhost:%s" % (gate_secure_port),
            ], timeout)
            self.fail("No exception raised to VTGate secure port")
        except dbexceptions.OperationalError as e:
            exception_type = e.args[2]
            exception_msg = str(e.args[2][0][0])
            self.assertIsInstance(exception_type, dbexceptions.FatalError,
                                  "unexpected exception type")
            if not exception_msg.startswith('Unexpected EOF in handshake to'):
                self.fail("Unexpected exception message: %s" % exception_msg)

        sconn = utils.get_vars(gate_port)["SecureConnections"]
        if sconn != 0:
            self.fail("unexpected conns %s" % sconn)

        # connect to vtgate with encrypted port
        conn = vtgatev2.connect([
            "localhost:%s" % (gate_secure_port),
        ],
                                timeout,
                                encrypted=True)
        (results, rowcount, lastrowid, fields) = conn._execute(
            "select 1 from dual", {},
            "test_keyspace",
            "master",
            keyranges=[
                keyrange.KeyRange(keyrange_constants.NON_PARTIAL_KEYRANGE),
            ])
        self.assertEqual(rowcount, 1, "want 1, got %d" % (rowcount))
        self.assertEqual(len(fields), 1, "want 1, got %d" % (len(fields)))
        self.assertEqual(results, [
            (1, ),
        ], 'wrong conn._execute output: %s' % str(results))

        sconn = utils.get_vars(gate_port)["SecureConnections"]
        if sconn != 1:
            self.fail("unexpected conns %s" % sconn)
        saccept = utils.get_vars(gate_port)["SecureAccepts"]
        if saccept == 0:
            self.fail("unexpected accepts %s" % saccept)

        # trigger a time out on a vtgate secure connection, see what exception we get
        try:
            conn._execute("select sleep(4) from dual", {},
                          "test_keyspace",
                          "master",
                          keyranges=[
                              keyrange.KeyRange(
                                  keyrange_constants.NON_PARTIAL_KEYRANGE),
                          ])
            self.fail("No timeout exception")
        except dbexceptions.TimeoutError as e:
            logging.debug("Got the right exception for SSL timeout: %s",
                          str(e))
        conn.close()
        utils.vtgate_kill(gate_proc)

        # kill everything
        utils.kill_sub_process(zkocc_server)