Esempio n. 1
0
def setUpModule():
  global vtgateclienttest_process
  global vtgateclienttest_port
  global vtgateclienttest_grpc_port

  try:
    environment.topo_server().setup()

    vtgateclienttest_port = environment.reserve_ports(1)
    args = environment.binary_args('vtgateclienttest') + [
        '-log_dir', environment.vtlogroot,
        '-port', str(vtgateclienttest_port),
        ]

    if protocols_flavor().vtgate_python_protocol() == 'grpc':
      vtgateclienttest_grpc_port = environment.reserve_ports(1)
      args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])

    vtgateclienttest_process = utils.run_bg(args)
    utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
  except:
    tearDownModule()
    raise
Esempio n. 2
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_replica1.teardown_mysql(),
      shard_0_replica2.teardown_mysql(),
      shard_0_rdonly.teardown_mysql(),
      shard_0_backup.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_replica1.teardown_mysql(),
      shard_2_master.teardown_mysql(),
      shard_2_replica1.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica1.remove_tree()
  shard_0_replica2.remove_tree()
  shard_0_rdonly.remove_tree()
  shard_0_backup.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica1.remove_tree()
  shard_2_master.remove_tree()
  shard_2_replica1.remove_tree()
Esempio n. 3
0
def setUpModule():
  try:
    environment.topo_server().setup()

    setup_procs = [
        shard_master.init_mysql(),
        shard_replica.init_mysql(),
        shard_rdonly1.init_mysql(),
        shard_0_master.init_mysql(),
        shard_0_replica.init_mysql(),
        shard_0_rdonly1.init_mysql(),
        shard_1_master.init_mysql(),
        shard_1_replica.init_mysql(),
        shard_1_rdonly1.init_mysql(),
    ]
    utils.wait_procs(setup_procs)
    init_keyspace()
    logging.debug('environment set up with the following shards and tablets:')
    logging.debug('=========================================================')
    logging.debug('TABLETS: test_keyspace/0:\n%s', all_shard_tablets)
    logging.debug('TABLETS: test_keyspace/-80:\n%s', shard_0_tablets)
    logging.debug('TABLETS: test_keyspace/80-:\n%s', shard_1_tablets)
  except:
    tearDownModule()
    raise
 def set_up(self):
   try:
     environment.topo_server().setup()
     utils.wait_procs([t.init_mysql() for t in self.tablets])
     utils.run_vtctl(['CreateKeyspace', self.keyspace])
     utils.run_vtctl(['SetKeyspaceShardingInfo', '-force', self.keyspace, 'keyspace_id', 'uint64'])
     for t in self.tablets:
       t.init_tablet(t.type, keyspace=self.keyspace, shard=t.shard)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     for t in self.tablets:
       t.create_db('vt_' + self.keyspace)
       t.start_vttablet(
         wait_for_state=None,
         extra_args=['-queryserver-config-schema-reload-time', '1'],
       )
     for t in self.tablets:
       t.wait_for_vttablet_state('SERVING')
     for t in self.tablets:
       if t.type == "master":
         utils.run_vtctl(['ReparentShard', '-force', self.keyspace+'/'+t.shard, t.tablet_alias], auto_log=True)
     utils.run_vtctl(['RebuildKeyspaceGraph', self.keyspace], auto_log=True)
     if self.schema:
       utils.run_vtctl(['ApplySchemaKeyspace', '-simple', '-sql', self.schema, self.keyspace])
     if self.vschema:
       if self.vschema[0] == '{':
         utils.run_vtctl(['ApplyVSchema', "-vschema", self.vschema])
       else:
         utils.run_vtctl(['ApplyVSchema', "-vschema_file", self.vschema])
     self.vtgate_server, self.vtgate_port = utils.vtgate_start(cache_ttl='500s', vtport=self.vtgate_port)
     vtgate_client = zkocc.ZkOccConnection("localhost:%u" % self.vtgate_port, "test_nj", 30.0)
     topology.read_topology(vtgate_client)
   except:
     self.shutdown()
     raise
Esempio n. 5
0
  def setUp(self):
    utils.wait_procs([self.tablet.init_mysql()])
    self.tablet.mquery(
        '', ['create database vt_test_keyspace', 'set global read_only = off'])

    self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
    with open(
        os.path.join(self.vttop, 'test', 'test_data', 'test_schema.sql')) as f:
      self.clean_sqls = []
      self.init_sqls = []
      clean_mode = False
      for line in f:
        line = line.rstrip()
        if line == '# clean':
          clean_mode = True
        if line=='' or line.startswith('#'):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
      try:
        for line in self.init_sqls:
          mcu.execute(line, {})
      finally:
        mcu.close()

    customrules = os.path.join(environment.tmproot, 'customrules.json')
    schema_override = os.path.join(environment.tmproot, 'schema_override.json')
    self.create_schema_override(schema_override)
    table_acl_config = os.path.join(
        environment.vttop, 'test', 'test_data', 'table_acl_config.json')

    environment.topo_server().setup()
    self.create_customrules(customrules);
    utils.run_vtctl('CreateKeyspace -force test_keyspace')
    self.tablet.init_tablet('master', 'test_keyspace', '0')
    if environment.topo_server().flavor() == 'zookeeper':
      self.tablet.start_vttablet(
          memcache=self.memcache,
          zkcustomrules='/zk/test_ca/config/customrules/testrules',
          schema_override=schema_override,
          table_acl_config=table_acl_config,
      )
    else:
      self.tablet.start_vttablet(
          memcache=self.memcache,
          filecustomrules=customrules,
          schema_override=schema_override,
          table_acl_config=table_acl_config,
      )
    self.conn = self.connect()
    self.txlogger = utils.curl(
        self.url('/debug/txlog'), background=True,
        stdout=open(self.txlog_file, 'w'))
    self.txlog = framework.Tailer(self.txlog_file, flush=self.tablet.flush)
    self.log = framework.Tailer(
        os.path.join(environment.vtlogroot, 'vttablet.INFO'),
        flush=self.tablet.flush)
    self.querylog = Querylog(self)
Esempio n. 6
0
def setUpModule():
  try:
    environment.topo_server().setup()

    _init_mysql(all_tablets)

    utils.run_vtctl(['CreateKeyspace', test_keyspace])

    shard_0_master.init_tablet('replica', test_keyspace, '0')
    shard_0_replica1.init_tablet('replica', test_keyspace, '0')
    shard_0_replica2.init_tablet('replica', test_keyspace, '0')
    shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0')
    shard_0_backup.init_tablet('backup', test_keyspace, '0')
    shard_1_master.init_tablet('replica', test_keyspace, '1')
    shard_1_replica1.init_tablet('replica', test_keyspace, '1')

    utils.Vtctld().start(enable_schema_change_dir=True)

    # create databases, start the tablets
    for t in initial_tablets:
      t.create_db(db_name)
      t.start_vttablet(wait_for_state=None)

    # wait for the tablets to start
    for t in initial_tablets:
      t.wait_for_vttablet_state('NOT_SERVING')
  except Exception as setup_exception:  # pylint: disable=broad-except
    try:
      tearDownModule()
    except Exception as e:  # pylint: disable=broad-except
      logging.exception('Tearing down a failed setUpModule() failed: %s', e)
    raise setup_exception
Esempio n. 7
0
def tearDownModule():
  global vtgate_server
  logging.debug("in tearDownModule")
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  utils.vtgate_kill(vtgate_server)
  tablet.kill_tablets([shard_0_master, shard_0_replica, shard_1_master,
                       shard_1_replica])
  teardown_procs = [shard_0_master.teardown_mysql(),
                    shard_0_replica.teardown_mysql(),
                    shard_1_master.teardown_mysql(),
                    shard_1_replica.teardown_mysql(),
                   ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
Esempio n. 8
0
def setUpModule():
    try:
        environment.topo_server().setup()

        setup_procs = [
            shard_0_master.init_mysql(),
            shard_0_replica.init_mysql(),
            shard_0_ny_rdonly.init_mysql(),
            shard_1_master.init_mysql(),
            shard_1_slave1.init_mysql(),
            shard_1_slave2.init_mysql(),
            shard_1_ny_rdonly.init_mysql(),
            shard_1_rdonly1.init_mysql(),
            shard_2_master.init_mysql(),
            shard_2_replica1.init_mysql(),
            shard_2_replica2.init_mysql(),
            shard_3_master.init_mysql(),
            shard_3_replica.init_mysql(),
            shard_3_rdonly1.init_mysql(),
        ]
        utils.Vtctld().start()
        utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise
Esempio n. 9
0
def setUpModule():
  try:
    environment.topo_server().setup()

  except:
    tearDownModule()
    raise
Esempio n. 10
0
def tearDownModule():
  utils.required_teardown()
  if utils.options.skip_teardown:
    return

  if use_mysqlctld:
    # Try to terminate mysqlctld gracefully, so it kills its mysqld.
    for proc in setup_procs:
      utils.kill_sub_process(proc, soft=True)
    teardown_procs = setup_procs
  else:
    teardown_procs = [
        tablet_master.teardown_mysql(),
        tablet_replica1.teardown_mysql(),
        tablet_replica2.teardown_mysql(),
    ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  tablet_master.remove_tree()
  tablet_replica1.remove_tree()
  tablet_replica2.remove_tree()
Esempio n. 11
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([src_master, src_replica, src_rdonly1, src_rdonly2,
                       dst_master, dst_replica])

  teardown_procs = [
      src_master.teardown_mysql(),
      src_replica.teardown_mysql(),
      src_rdonly1.teardown_mysql(),
      src_rdonly2.teardown_mysql(),
      dst_master.teardown_mysql(),
      dst_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  src_master.remove_tree()
  src_replica.remove_tree()
  src_rdonly1.remove_tree()
  src_rdonly2.remove_tree()
  dst_master.remove_tree()
  dst_replica.remove_tree()
Esempio n. 12
0
def tearDownModule():
  utils.required_teardown()
  if utils.options.skip_teardown:
    return

  if utils.vtgate:
    utils.vtgate.kill()
  teardown_procs = [
      source_master.teardown_mysql(),
      source_replica.teardown_mysql(),
      source_rdonly1.teardown_mysql(),
      source_rdonly2.teardown_mysql(),
      destination_master.teardown_mysql(),
      destination_replica.teardown_mysql(),
      destination_rdonly1.teardown_mysql(),
      destination_rdonly2.teardown_mysql(),
  ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  source_master.remove_tree()
  source_replica.remove_tree()
  source_rdonly1.remove_tree()
  source_rdonly2.remove_tree()
  destination_master.remove_tree()
  destination_replica.remove_tree()
  destination_rdonly1.remove_tree()
  destination_rdonly2.remove_tree()
Esempio n. 13
0
 def tearDown(self):
   tablet.Tablet.check_vttablet_count()
   environment.topo_server().wipe()
   for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
     t.reset_replication()
     t.clean_dbs()
   super(TestReparent, self).tearDown()
Esempio n. 14
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([shard_0_master, shard_0_replica,
                       shard_1_master, shard_1_replica])
  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_replica.teardown_mysql(),
      shard_1_master.teardown_mysql(),
      shard_1_replica.teardown_mysql(),
      unsharded_master.teardown_mysql(),
      unsharded_replica.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_replica.remove_tree()
  shard_1_master.remove_tree()
  shard_1_replica.remove_tree()
  unsharded_master.remove_tree()
  unsharded_replica.remove_tree()
Esempio n. 15
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  if use_mysqlctld:
    # Try to terminate mysqlctld gracefully, so it kills its mysqld.
    for proc in setup_procs:
      utils.kill_sub_process(proc, soft=True)
    teardown_procs = setup_procs
  else:
    teardown_procs = [
        tablet_62344.teardown_mysql(),
        tablet_31981.teardown_mysql(),
        ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  tablet_62344.remove_tree()
  tablet_31981.remove_tree()

  path = os.path.join(environment.vtdataroot, 'snapshot')
  try:
    shutil.rmtree(path)
  except OSError as e:
    logging.debug("removing snapshot %s: %s", path, str(e))
Esempio n. 16
0
 def tearDown(self):
   tablet.Tablet.check_vttablet_count()
   environment.topo_server().wipe()
   for t in [tablet_62344, tablet_62044]:
     t.reset_replication()
     t.set_semi_sync_enabled(master=False)
     t.clean_dbs()
Esempio n. 17
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()
Esempio n. 18
0
def setUpModule():
  try:
    environment.topo_server().setup()
    setup_procs = [t.init_mysql() for t in all_tablets]
    utils.wait_procs(setup_procs)
  except:
    tearDownModule()
    raise
Esempio n. 19
0
def zkocc_start(cells=['test_nj'], extra_params=[]):
  args = environment.binary_args('zkocc') + [
          '-port', str(environment.topo_server().zkocc_port_base),
          '-stderrthreshold=ERROR',
          ] + extra_params + cells
  sp = run_bg(args)
  wait_for_vars("zkocc", environment.topo_server().zkocc_port_base)
  return sp
Esempio n. 20
0
 def tearDown(self):
   tablet.Tablet.check_vttablet_count()
   environment.topo_server().wipe()
   for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
     t.reset_replication()
     t.set_semi_sync_enabled(master=False)
     t.clean_dbs(include_vt=True)
   super(TestReparent, self).tearDown()
Esempio n. 21
0
 def setUp(self):
   environment.topo_server().wipe()
   self.vtgate_zk, self.vtgate_zk_port = utils.vtgate_start()
   if environment.topo_server().flavor() == 'zookeeper':
     self.zkocc_server = utils.zkocc_start()
     self.vtgate_zkocc, self.vtgate_zkocc_port = utils.vtgate_start(topo_impl="zkocc")
     self.topo = zkocc.ZkOccConnection("localhost:%u" % environment.topo_server().zkocc_port_base, 'test_nj', 30)
     self.topo.dial()
Esempio n. 22
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
                                      'RefreshState', tablet_62344.tablet_alias],
                                     expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    if environment.topo_server().flavor() == 'zookeeper':
      # extra small test: we ran for a while, get the states we were in,
      # make sure they're accounted for properly
      # first the query engine States
      v = utils.get_vars(tablet_62344.port)
      logging.debug("vars: %s" % str(v))

      # then the Zookeeper connections
      if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
        self.fail('invalid zk test_nj state: %s' %
                  v['ZkMetaConn']['test_nj']['Current'])
      if v['ZkMetaConn']['global']['Current'] != 'Connected':
        self.fail('invalid zk global state: %s' %
                  v['ZkMetaConn']['global']['Current'])
      if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
        self.fail('not enough time in Connected state: %u',
                  v['ZkMetaConn']['test_nj']['DurationConnected'])
      if v['TabletType'] != 'master':
        self.fail('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Esempio n. 23
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(["CreateKeyspace", "test_keyspace"])

        tablet_62344.init_tablet("master", "test_keyspace", "0")
        utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"])
        utils.validate_topology()
        self._check_srv_shard()
        tablet_62344.create_db("vt_test_keyspace")
        tablet_62344.start_vttablet()

        utils.run_vtctl(["Ping", tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (
            environment.binary_args("vtctl")
            + environment.topo_server().flags()
            + [
                "-tablet_manager_protocol",
                protocols_flavor().tablet_manager_protocol(),
                "-tablet_protocol",
                protocols_flavor().tabletconn_protocol(),
                "-log_dir",
                environment.vtlogroot,
                "Sleep",
                tablet_62344.tablet_alias,
                "10s",
            ]
        )
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        _, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        if environment.topo_server().flavor() == "zookeeper":
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug("vars: %s", v)

            # then the Zookeeper connections
            if v["ZkCachedConn"]["test_nj"] != "Connected":
                self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"])
            if v["ZkCachedConn"]["global"] != "Connected":
                self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"])
            if v["TabletType"] != "master":
                self.fail("TabletType not exported correctly")

        tablet_62344.kill_vttablet()
Esempio n. 24
0
 def shutdown(self):
   tablet.kill_tablets(self.tablets)
   teardown_procs = [t.teardown_mysql() for t in self.tablets]
   utils.wait_procs(teardown_procs, raise_on_error=False)
   environment.topo_server().teardown()
   utils.kill_sub_processes()
   utils.remove_tmp_files()
   for t in self.tablets:
     t.remove_tree()
Esempio n. 25
0
  def setUp(self):
    utils.wait_procs([self.tablet.init_mysql()])
    self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])

    self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
    with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
      self.clean_sqls = []
      self.init_sqls = []
      clean_mode = False
      for line in f:
        line = line.rstrip()
        if line == "# clean":
          clean_mode = True
        if line=='' or line.startswith("#"):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
      try:
        for line in self.init_sqls:
          mcu.execute(line, {})
      finally:
        mcu.close()

    customrules = os.path.join(environment.tmproot, 'customrules.json')
    self.create_customrules(customrules)
    schema_override = os.path.join(environment.tmproot, 'schema_override.json')
    self.create_schema_override(schema_override)
    table_acl_config = os.path.join(environment.vttop, 'test', 'test_data', 'table_acl_config.json')

    if self.env == 'vttablet':
      environment.topo_server().setup()
      utils.run_vtctl('CreateKeyspace -force test_keyspace')
      self.tablet.init_tablet('master', 'test_keyspace', '0')
      self.tablet.start_vttablet(
              memcache=self.memcache,
              customrules=customrules,
              schema_override=schema_override,
              table_acl_config=table_acl_config,
              auth=True,
      )
    else:
      self.tablet.start_vtocc(
              memcache=self.memcache,
              customrules=customrules,
              schema_override=schema_override,
              table_acl_config=table_acl_config,
              auth=True,
              keyspace="test_keyspace", shard="0",
      )
    self.conn = self.connect()
    self.txlogger = utils.curl(self.url('/debug/txlog'), background=True, stdout=open(self.txlog_file, 'w'))
    self.txlog = framework.Tailer(self.txlog_file, flush=self.tablet.flush)
    self.log = framework.Tailer(os.path.join(environment.vtlogroot, '%s.INFO' % self.env), flush=self.tablet.flush)
    self.querylog = Querylog(self)
Esempio n. 26
0
def setUpModule():
  global vtgate_server
  global vtgate_port
  global vtgate_socket_file
  global master_start_position

  try:
    environment.topo_server().setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()]
    utils.wait_procs(setup_procs)

    # Start up a master mysql and vttablet
    logging.debug('Setting up tablets')
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    master_tablet.create_db('vt_test_keyspace')
    master_tablet.create_db('other_database')
    replica_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('other_database')

    utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])

    vtgate_socket_file = environment.tmproot + '/vtgate.sock'
    vtgate_server, vtgate_port = utils.vtgate_start(
        socket_file=vtgate_socket_file)

    master_tablet.start_vttablet()
    replica_tablet.start_vttablet()
    utils.run_vtctl(['SetReadWrite', master_tablet.tablet_alias])
    utils.check_db_read_write(master_tablet.tablet_uid)

    for t in [master_tablet, replica_tablet]:
      t.reset_replication()
    utils.run_vtctl(['ReparentShard', '-force', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)

    # reset counter so tests don't assert
    tablet.Tablet.tablets_running = 0

    master_start_position = _get_master_current_position()
    master_tablet.mquery('vt_test_keyspace', _create_vt_insert_test)
    master_tablet.mquery('vt_test_keyspace', _create_vt_a)
    master_tablet.mquery('vt_test_keyspace', _create_vt_b)

    utils.run_vtctl(['ReloadSchema', master_tablet.tablet_alias])
    utils.run_vtctl(['ReloadSchema', replica_tablet.tablet_alias])

  except:
    tearDownModule()
    raise
Esempio n. 27
0
def setUpModule():
  try:
    environment.topo_server().setup()
    setup_procs = [t.init_mysql(use_rbr=base_sharding.use_rbr)
                   for t in all_tablets]
    utils.Vtctld().start()
    utils.wait_procs(setup_procs)
  except:
    tearDownModule()
    raise
Esempio n. 28
0
def setUpModule():
  global keyspace_env
  global shard_0_master
  global shard_1_master
  global lookup_master
  logging.debug('in setUpModule')

  try:
    environment.topo_server().setup()
    logging.debug('Setting up tablets')
    keyspace_env = keyspace_util.TestEnv()
    keyspace_env.launch(
        'user',
        shards=['-80', '80-'],
        ddls=[
            create_vt_user,
            create_vt_user2,
            create_vt_user_extra,
            create_vt_music,
            create_vt_music_extra,
            create_join_user,
            create_join_user_extra,
            create_join_name_info,
            create_twopc_user,
            ],
        rdonly_count=1,  # to test SplitQuery
        twopc_coordinator_address='localhost:15028',  # enables 2pc
        )
    keyspace_env.launch(
        'lookup',
        ddls=[
            create_vt_user_seq,
            create_vt_music_seq,
            create_vt_main_seq,
            create_music_user_map,
            create_name_user2_map,
            create_main,
            create_twopc_lookup,
            ],
        twopc_coordinator_address='localhost:15028',  # enables 2pc
        )
    shard_0_master = keyspace_env.tablet_map['user.-80.master']
    shard_1_master = keyspace_env.tablet_map['user.80-.master']
    lookup_master = keyspace_env.tablet_map['lookup.0.master']

    utils.apply_vschema(vschema)
    utils.VtGate().start(
        tablets=[shard_0_master, shard_1_master, lookup_master],
        extra_args=['-transaction_mode', 'TWOPC'])
    utils.vtgate.wait_for_endpoints('user.-80.master', 1)
    utils.vtgate.wait_for_endpoints('user.80-.master', 1)
    utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
  except:
    tearDownModule()
    raise
Esempio n. 29
0
def setUpModule():
  try:
    environment.topo_server().setup()

    _init_mysql(tablets)

    utils.run_vtctl(['CreateKeyspace', test_keyspace])

    shard_0_master.init_tablet(  'master',  test_keyspace, '0')
    shard_0_replica1.init_tablet('replica', test_keyspace, '0')
    shard_0_replica2.init_tablet('replica', test_keyspace, '0')
    shard_0_rdonly.init_tablet(  'rdonly',  test_keyspace, '0')
    shard_0_backup.init_tablet(  'backup',  test_keyspace, '0')
    shard_1_master.init_tablet(  'master',  test_keyspace, '1')
    shard_1_replica1.init_tablet('replica', test_keyspace, '1')

    utils.run_vtctl(['RebuildKeyspaceGraph', test_keyspace], auto_log=True)

    # run checks now before we start the tablets
    utils.validate_topology()

    utils.Vtctld().start()

    # create databases, start the tablets
    for t in tablets:
      t.create_db(db_name)
      t.start_vttablet(wait_for_state=None)

    # wait for the tablets to start
    shard_0_master.wait_for_vttablet_state('SERVING')
    shard_0_replica1.wait_for_vttablet_state('SERVING')
    shard_0_replica2.wait_for_vttablet_state('SERVING')
    shard_0_rdonly.wait_for_vttablet_state('SERVING')
    shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
    shard_1_master.wait_for_vttablet_state('SERVING')
    shard_1_replica1.wait_for_vttablet_state('SERVING')

    # make sure all replication is good
    for t in tablets:
      t.reset_replication()

    utils.run_vtctl(['InitShardMaster', test_keyspace+'/0',
                     shard_0_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['InitShardMaster', test_keyspace+'/1',
                     shard_1_master.tablet_alias], auto_log=True)
    utils.run_vtctl(['ValidateKeyspace', '-ping-tablets', test_keyspace])

    # check after all tablets are here and replication is fixed
    utils.validate_topology(ping_tablets=True)
  except Exception as setup_exception:
    try:
      tearDownModule()
    except Exception as e:
      logging.exception("Tearing down a failed setUpModule() failed: %s", e)
    raise setup_exception
Esempio n. 30
0
def tearDownModule():
  logging.debug('in tearDownModule')
  if utils.options.skip_teardown:
    return
  logging.debug('Tearing down the servers and setup')
  keyspace_env.teardown()

  environment.topo_server().teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()
Esempio n. 31
0
def setUpModule():
    logging.debug('in setUpModule')
    try:
        environment.topo_server().setup()

        # start mysql instance external to the test
        setup_procs = [
            shard_0_master.init_mysql(),
            shard_0_replica1.init_mysql(),
        ]
        utils.wait_procs(setup_procs)
        setup_tablets()
        setup_vtgate()
        # After VTGate comes up, populate it with some initial data
        initial_writes(0, keyrange.KeyRange(''))
    except Exception, e:
        logging.exception('error during set up: %s', e)
        tearDownModule()
        raise
Esempio n. 32
0
def tearDownModule():
  if utils.options.skip_teardown:
    return

  shard_0_master.kill_vttablet()
  shard_0_slave.kill_vttablet()

  teardown_procs = [
      shard_0_master.teardown_mysql(),
      shard_0_slave.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_0_slave.remove_tree()
Esempio n. 33
0
def tearDownModule():
  utils.required_teardown()
  if utils.options.skip_teardown:
    return

  tablet.kill_tablets([master_tablet, replica_tablet])

  teardown_procs = [
      master_tablet.teardown_mysql(),
      replica_tablet.teardown_mysql(),
      ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()
  utils.kill_sub_processes()
  utils.remove_tmp_files()

  master_tablet.remove_tree()
  replica_tablet.remove_tree()
Esempio n. 34
0
def setUpModule():
    try:
        environment.topo_server().setup()

        setup_procs = [
            shard_master.init_mysql(),
            shard_replica.init_mysql(),
            shard_rdonly1.init_mysql(),
            shard_0_master.init_mysql(),
            shard_0_replica.init_mysql(),
            shard_0_rdonly1.init_mysql(),
            shard_1_master.init_mysql(),
            shard_1_replica.init_mysql(),
            shard_1_rdonly1.init_mysql(),
        ]
        utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise
Esempio n. 35
0
def tearDownModule():
    global __tablets
    if utils.options.skip_teardown:
        return
    if __tablets is not None:
        tablet.kill_tablets(__tablets)
        teardown_procs = []
        for t in __tablets:
            teardown_procs.append(t.teardown_mysql())
        utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()

    utils.kill_sub_processes()
    utils.remove_tmp_files()

    if __tablets is not None:
        for t in __tablets:
            t.remove_tree()
Esempio n. 36
0
def setUpModule():
  global vtgate_server, vtgate_port
  logging.debug("in setUpModule")
  try:
    environment.topo_server().setup()
    setup_topology()

    # start mysql instance external to the test
    global __tablets
    setup_procs = []
    for tablet in __tablets:
      setup_procs.append(tablet.init_mysql())
    utils.wait_procs(setup_procs)
    create_db()
    start_tablets()
    vtgate_server, vtgate_port = utils.vtgate_start()
  except:
    tearDownModule()
    raise
Esempio n. 37
0
def setUpModule():
    try:
        environment.topo_server().setup()

        setup_procs = [
            source_master.init_mysql(),
            source_replica.init_mysql(),
            source_rdonly1.init_mysql(),
            source_rdonly2.init_mysql(),
            destination_master.init_mysql(),
            destination_replica.init_mysql(),
            destination_rdonly1.init_mysql(),
            destination_rdonly2.init_mysql(),
        ]
        utils.Vtctld().start()
        utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise
Esempio n. 38
0
def setUpModule():
  try:
    if environment.topo_server().flavor() == 'zookeeper':
      # this is a one-off test to make sure our zookeeper implementation
      # behaves with a server that is not DNS-resolveable
      environment.topo_server().setup(add_bad_host=True)
    else:
      environment.topo_server().setup()

    # start mysql instance external to the test
    setup_procs = [
        tablet_62344.init_mysql(),
        tablet_62044.init_mysql(),
        ]
    utils.Vtctld().start()
    utils.wait_procs(setup_procs)
  except:
    tearDownModule()
    raise
Esempio n. 39
0
def setUpModule():
  global keyspace_env
  global shard_0_master
  global shard_1_master
  global lookup_master
  logging.debug('in setUpModule')

  try:
    environment.topo_server().setup()
    logging.debug('Setting up tablets')
    keyspace_env = keyspace_util.TestEnv()
    keyspace_env.launch(
        'user',
        shards=['-80', '80-'],
        ddls=[
            create_vt_user,
            create_vt_user2,
            create_vt_user_extra,
            create_vt_music,
            create_vt_music_extra,
            create_join_user,
            create_join_user_extra,
            create_join_name_info,
            ],
        )
    keyspace_env.launch(
        'lookup',
        ddls=[
            create_vt_user_seq,
            create_vt_music_seq,
            create_music_user_map,
            create_name_user2_map,
            ],
        )
    shard_0_master = keyspace_env.tablet_map['user.-80.master']
    shard_1_master = keyspace_env.tablet_map['user.80-.master']
    lookup_master = keyspace_env.tablet_map['lookup.0.master']

    utils.apply_vschema(vschema)
    utils.VtGate().start()
  except:
    tearDownModule()
    raise
Esempio n. 40
0
def setUpModule():
    global vtgate_server
    global vtgate_port

    try:
        environment.topo_server().setup()

        # start mysql instance external to the test
        setup_procs = [master_tablet.init_mysql(), replica_tablet.init_mysql()]
        utils.wait_procs(setup_procs)

        # Start up a master mysql and vttablet
        logging.debug("Setting up tablets")
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
        master_tablet.init_tablet('master', 'test_keyspace', '0')
        replica_tablet.init_tablet('replica', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildKeyspaceGraph', 'test_keyspace'])
        utils.validate_topology()

        master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
        replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)

        vtgate_server, vtgate_port = utils.vtgate_start()

        master_tablet.start_vttablet(memcache=True, wait_for_state=None)
        replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
        master_tablet.wait_for_vttablet_state('SERVING')
        replica_tablet.wait_for_vttablet_state('SERVING')

        utils.run_vtctl([
            'ReparentShard', '-force', 'test_keyspace/0',
            master_tablet.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        # restart the replica tablet so the stats are reset
        replica_tablet.kill_vttablet()
        replica_tablet.start_vttablet(memcache=True)
    except:
        tearDownModule()
        raise
Esempio n. 41
0
def tearDownModule():
    logging.debug('in tearDownModule')
    utils.required_teardown()
    if utils.options.skip_teardown:
        return
    logging.debug('Tearing down the servers and setup')
    tablet.kill_tablets([shard_0_master, shard_0_replica1])
    teardown_procs = [
        shard_0_master.teardown_mysql(),
        shard_0_replica1.teardown_mysql(),
    ]
    utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()

    utils.kill_sub_processes()
    utils.remove_tmp_files()

    shard_0_master.remove_tree()
    shard_0_replica1.remove_tree()
Esempio n. 42
0
def tearDownModule():
    if utils.options.skip_teardown:
        return

    teardown_procs = [
        tablet_62344.teardown_mysql(),
        tablet_62044.teardown_mysql(),
        tablet_41983.teardown_mysql(),
        tablet_31981.teardown_mysql(),
    ]
    utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()

    tablet_62344.remove_tree()
    tablet_62044.remove_tree()
    tablet_41983.remove_tree()
    tablet_31981.remove_tree()
Esempio n. 43
0
  def start(self, cell='test_nj', retry_count=2,
            topo_impl=None, cache_ttl='1s',
            extra_args=None, tablets=None,
            tablet_types_to_wait='MASTER,REPLICA',
            l2vtgates=None,
            cells_to_watch=None):
    """Start vtgate. Saves it into the global vtgate variable if not set yet."""

    args = environment.binary_args('vtgate') + [
        '-port', str(self.port),
        '-cell', cell,
        '-retry-count', str(retry_count),
        '-log_dir', environment.vtlogroot,
        '-srv_topo_cache_ttl', cache_ttl,
        '-srv_topo_cache_refresh', cache_ttl,
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
        '-stderrthreshold', get_log_level(),
        '-normalize_queries',
        '-gateway_implementation', vtgate_gateway_flavor().flavor(),
    ]

    if cells_to_watch:
      args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets))
    else:
      args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))

    if l2vtgates:
      args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)])
    if tablet_types_to_wait:
      args.extend(['-tablet_types_to_wait', tablet_types_to_wait])

    if protocols_flavor().vtgate_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
      args.extend(['-grpc_max_message_size',
                   str(environment.grpc_max_message_size)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if topo_impl:
      args.extend(['-topo_implementation', topo_impl])
    else:
      args.extend(environment.topo_server().flags())
    if extra_args:
      args.extend(extra_args)
    if self.mysql_port:
      args.extend(['-mysql_server_port', str(self.mysql_port)])

    self.proc = run_bg(args)
    # We use a longer timeout here, as we may be waiting for the initial
    # state of a few tablets.
    wait_for_vars('vtgate', self.port, timeout=20.0)

    global vtgate
    if not vtgate:
      vtgate = self
Esempio n. 44
0
def setUpModule():
    try:
        environment.topo_server().setup()

        # start mysql instance external to the test
        global setup_procs
        setup_procs = [
            tablet_master.init_mysql(),
            tablet_replica1.init_mysql(),
            tablet_replica2.init_mysql(),
        ]
        if use_mysqlctld:
            tablet_master.wait_for_mysqlctl_socket()
            tablet_replica1.wait_for_mysqlctl_socket()
            tablet_replica2.wait_for_mysqlctl_socket()
        else:
            utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise
Esempio n. 45
0
def setUpModule():
    global keyspace_env
    global shard_0_master
    global shard_1_master
    global lookup_master
    global vtgate_server
    global vtgate_port
    logging.debug("in setUpModule")

    try:
        environment.topo_server().setup()
        logging.debug("Setting up tablets")
        keyspace_env = keyspace_util.TestEnv()
        keyspace_env.launch(
            "user",
            shards=["-80", "80-"],
            ddls=[
                create_vt_user,
                create_vt_user2,
                create_vt_user_extra,
                create_vt_music,
                create_vt_music_extra,
            ],
        )
        keyspace_env.launch(
            "lookup",
            ddls=[
                create_vt_user_idx,
                create_music_user_map,
                create_name_user2_map,
            ],
        )
        shard_0_master = keyspace_env.tablet_map["user.-80.master"]
        shard_1_master = keyspace_env.tablet_map["user.80-.master"]
        lookup_master = keyspace_env.tablet_map["lookup.0.master"]

        utils.apply_vschema(schema)
        vtgate_server, vtgate_port = utils.vtgate_start()
    except:
        tearDownModule()
        raise
Esempio n. 46
0
def tearDownModule():
    utils.required_teardown()
    if utils.options.skip_teardown:
        return

    teardown_procs = [
        shard_0_master.teardown_mysql(),
        shard_0_replica.teardown_mysql(),
        shard_0_ny_rdonly.teardown_mysql(),
        shard_1_master.teardown_mysql(),
        shard_1_slave1.teardown_mysql(),
        shard_1_slave2.teardown_mysql(),
        shard_1_ny_rdonly.teardown_mysql(),
        shard_1_rdonly1.teardown_mysql(),
        shard_2_master.teardown_mysql(),
        shard_2_replica1.teardown_mysql(),
        shard_2_replica2.teardown_mysql(),
        shard_3_master.teardown_mysql(),
        shard_3_replica.teardown_mysql(),
        shard_3_rdonly1.teardown_mysql(),
    ]
    utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()

    shard_0_master.remove_tree()
    shard_0_replica.remove_tree()
    shard_0_ny_rdonly.remove_tree()
    shard_1_master.remove_tree()
    shard_1_slave1.remove_tree()
    shard_1_slave2.remove_tree()
    shard_1_ny_rdonly.remove_tree()
    shard_1_rdonly1.remove_tree()
    shard_2_master.remove_tree()
    shard_2_replica1.remove_tree()
    shard_2_replica2.remove_tree()
    shard_3_master.remove_tree()
    shard_3_replica.remove_tree()
    shard_3_rdonly1.remove_tree()
    def setUp(self):
        try:
            os.makedirs(environment.tmproot)
        except OSError:
            # directory already exists
            pass

        try:
            topo_flavor = environment.topo_server().flavor()
            environment.topo_server().setup()

            # start mysql instance external to the test
            setup_procs = [
                self.replica.init_mysql(),
                self.master.init_mysql(),
            ]
            utils.Vtctld().start()
            logging.debug(utils.vtctld_connection)
            utils.wait_procs(setup_procs)

            for t in self.master, self.replica:
                t.create_db('vt_test_keyspace')

            self.master.init_tablet('replica',
                                    'test_keyspace',
                                    '0',
                                    start=True)
            self.replica.init_tablet('replica',
                                     'test_keyspace',
                                     '0',
                                     start=True)
            utils.run_vtctl([
                'InitShardMaster', '-force', 'test_keyspace/0',
                self.master.tablet_alias
            ])
            self.master.mquery('vt_test_keyspace', _create_vt_insert_test)
            for t in [self.master, self.replica]:
                t.set_semi_sync_enabled(master=False, slave=False)
        except Exception as e:
            logging.exception(e)
            self.tearDown()
Esempio n. 48
0
def setUpModule():
    try:
        environment.topo_server().setup()

        _init_mysql(all_tablets)

        utils.run_vtctl(['CreateKeyspace', test_keyspace])

        shard_0_master.init_tablet('master', test_keyspace, '0')
        shard_0_replica1.init_tablet('replica', test_keyspace, '0')
        shard_0_replica2.init_tablet('replica', test_keyspace, '0')
        shard_0_rdonly.init_tablet('rdonly', test_keyspace, '0')
        shard_0_backup.init_tablet('backup', test_keyspace, '0')
        shard_1_master.init_tablet('master', test_keyspace, '1')
        shard_1_replica1.init_tablet('replica', test_keyspace, '1')

        # run checks now before we start the tablets
        utils.validate_topology()

        utils.Vtctld().start(enable_schema_change_dir=True)

        # create databases, start the tablets
        for t in initial_tablets:
            t.create_db(db_name)
            t.start_vttablet(wait_for_state=None)

        # wait for the tablets to start
        shard_0_master.wait_for_vttablet_state('SERVING')
        shard_0_replica1.wait_for_vttablet_state('NOT_SERVING')
        shard_0_replica2.wait_for_vttablet_state('NOT_SERVING')
        shard_0_rdonly.wait_for_vttablet_state('NOT_SERVING')
        shard_0_backup.wait_for_vttablet_state('NOT_SERVING')
        shard_1_master.wait_for_vttablet_state('SERVING')
        shard_1_replica1.wait_for_vttablet_state('NOT_SERVING')
    except Exception as setup_exception:  # pylint: disable=broad-except
        try:
            tearDownModule()
        except Exception as e:  # pylint: disable=broad-except
            logging.exception('Tearing down a failed setUpModule() failed: %s',
                              e)
        raise setup_exception
def setUpModule():
    global keyspace_env
    global shard_0_master
    global shard_0_replica
    global shard_1_master
    global lookup_master
    logging.debug('in setUpModule')

    try:
        environment.topo_server().setup()
        logging.debug('Setting up tablets')
        keyspace_env = keyspace_util.TestEnv()
        keyspace_env.launch(
            'user',
            shards=['-80', '80-'],
            ddls=[
                create_sharded_message,
            ],
        )
        keyspace_env.launch(
            'lookup',
            ddls=[
                create_unsharded_message,
            ],
        )
        shard_0_master = keyspace_env.tablet_map['user.-80.master']
        shard_0_replica = keyspace_env.tablet_map['user.-80.replica.0']
        shard_1_master = keyspace_env.tablet_map['user.80-.master']
        lookup_master = keyspace_env.tablet_map['lookup.0.master']

        utils.apply_vschema(vschema)
        utils.VtGate().start(tablets=[
            shard_0_master, shard_0_replica, shard_1_master, lookup_master
        ])
        utils.vtgate.wait_for_endpoints('user.-80.master', 1)
        utils.vtgate.wait_for_endpoints('user.-80.replica', 1)
        utils.vtgate.wait_for_endpoints('user.80-.master', 1)
        utils.vtgate.wait_for_endpoints('lookup.0.master', 1)
    except:
        tearDownModule()
        raise
Esempio n. 50
0
def setUpModule():
  try:
    environment.topo_server().setup()
    utils.Vtctld().start()

    setup_procs = [
        master_tablet.init_mysql(),
        replica_tablet.init_mysql(),
        ]
    utils.wait_procs(setup_procs)

    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')

    master_tablet.create_db('vt_test_keyspace')
    replica_tablet.create_db('vt_test_keyspace')
  except:
    tearDownModule()
    raise
Esempio n. 51
0
    def start(self):
        args = environment.binary_args('vtctld') + [
            '-debug',
            '-web_dir',
            environment.vttop + '/web/vtctld',
            '--templates',
            environment.vttop + '/go/cmd/vtctld/templates',
            '--log_dir',
            environment.vtlogroot,
            '--port',
            str(self.port),
            '--schema_change_dir',
            self.schema_change_dir,
            '--schema_change_controller',
            'local',
            '--schema_change_check_interval',
            '1',
            '-tablet_manager_protocol',
            protocols_flavor().tablet_manager_protocol(),
            '-vtgate_protocol',
            protocols_flavor().vtgate_protocol(),
            '-tablet_protocol',
            protocols_flavor().tabletconn_protocol(),
        ] + environment.topo_server().flags()
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ','.join(protocols_flavor().service_map())])
        if protocols_flavor().vtctl_client_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'),
                         'w')
        stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
                         'w')
        self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)

        # wait for the process to listen to RPC
        timeout = 30
        while True:
            v = get_vars(self.port)
            if v:
                break
            timeout = wait_step('waiting for vtctld to start',
                                timeout,
                                sleep_time=0.2)

        # save the running instance so vtctl commands can be remote executed now
        global vtctld, vtctld_connection
        if not vtctld:
            vtctld = self
            protocol, endpoint = self.rpc_endpoint(python=True)
            vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)

        return self.proc
Esempio n. 52
0
def setUpModule():
    global keyspace_env
    global shard_0_master

    try:
        environment.topo_server().setup()
        keyspace_env = keyspace_util.TestEnv()
        keyspace_env.launch('user', ddls=[
            create_vt_user,
            create_main,
        ])
        shard_0_master = keyspace_env.tablet_map['user.0.master']
        utils.VtGate().start(
            tablets=[shard_0_master],
            extra_args=['-vschema_ddl_authorized_users', '%'],
        )
        utils.vtgate.wait_for_endpoints('user.0.master', 1)

    except:
        tearDownModule()
        raise
Esempio n. 53
0
def main():
    parser = optparse.OptionParser(usage="usage: %prog [options]")
    utils.add_options(parser)
    (options, args) = parser.parse_args()
    options.debug = True
    utils.set_options(options)
    env = keyspace_util.TestEnv()
    vtgate_server = None
    try:
        environment.topo_server().setup()
        env.launch(
            "user",
            shards=["-80", "80-"],
            ddls=[
                'create table user(user_id bigint, name varchar(128), primary key(user_id))',
                'create table user_extra(user_id bigint, extra varchar(128), primary key(user_id))',
                'create table music(user_id bigint, music_id bigint, primary key(user_id, music_id))',
                'create table music_extra(music_id bigint, keyspace_id bigint unsigned, primary key(music_id))',
            ],
        )
        env.launch(
            "lookup",
            ddls=[
                'create table user_idx(user_id bigint not null auto_increment, primary key(user_id))',
                'create table name_user_idx(name varchar(128), user_id bigint, primary key(name, user_id))',
                'create table music_user_idx(music_id bigint not null auto_increment, user_id bigint, primary key(music_id))',
            ],
        )
        utils.apply_vschema(vschema)
        vtgate_server, vtgate_port = utils.vtgate_start(cache_ttl='500s')
        utils.Vtctld().start()
        print "vtgate:", vtgate_port
        print "vtctld:", utils.vtctld.port
        utils.pause("the cluster is up, press enter to shut it down...")
    finally:
        utils.vtgate_kill(vtgate_server)
        env.teardown()
        utils.kill_sub_processes()
        utils.remove_tmp_files()
        environment.topo_server().teardown()
Esempio n. 54
0
def setUpModule():
    try:
        if utils.options.xvfb:
            try:
                # This will be killed automatically by utils.kill_sub_processes()
                utils.run_bg(['Xvfb', ':15', '-ac'])
                os.environ['DISPLAY'] = ':15'
            except OSError as err:
                # Despite running in background, utils.run_bg() will throw immediately
                # if the Xvfb binary is not found.
                logging.error(
                    "Can't start Xvfb (will try local DISPLAY instead): %s",
                    err)

        environment.topo_server().setup()

        setup_procs = [t.init_mysql() for t in tablets]
        utils.Vtctld().start()
        utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise
Esempio n. 55
0
def setUpModule():
  try:
    environment.topo_server().setup()

    # start mysql instance external to the test
    setup_procs = [master_tablet.init_mysql(),
                   replica_tablet.init_mysql()]
    utils.wait_procs(setup_procs)

    # start a vtctld so the vtctl insert commands are just RPCs, not forks
    utils.Vtctld().start()

    # Start up a master mysql and vttablet
    logging.debug('Setting up tablets')
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])
    master_tablet.init_tablet('master', 'test_keyspace', '0')
    replica_tablet.init_tablet('replica', 'test_keyspace', '0')
    utils.validate_topology()

    master_tablet.populate('vt_test_keyspace', create_vt_insert_test)
    replica_tablet.populate('vt_test_keyspace', create_vt_insert_test)

    utils.VtGate().start()

    master_tablet.start_vttablet(memcache=True, wait_for_state=None)
    replica_tablet.start_vttablet(memcache=True, wait_for_state=None)
    master_tablet.wait_for_vttablet_state('SERVING')
    replica_tablet.wait_for_vttablet_state('SERVING')

    utils.run_vtctl(['InitShardMaster', 'test_keyspace/0',
                     master_tablet.tablet_alias], auto_log=True)
    utils.validate_topology()

    # restart the replica tablet so the stats are reset
    replica_tablet.kill_vttablet()
    replica_tablet.start_vttablet(memcache=True)
  except:
    tearDownModule()
    raise
Esempio n. 56
0
def tearDownModule():
    utils.required_teardown()
    if utils.options.skip_teardown:
        return

    teardown_procs = [
        tablet_master.teardown_mysql(
            extra_args=['-db-credentials-file', db_credentials_file]),
        tablet_replica1.teardown_mysql(
            extra_args=['-db-credentials-file', db_credentials_file]),
        tablet_replica2.teardown_mysql(
            extra_args=['-db-credentials-file', db_credentials_file]),
    ]
    utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()

    tablet_master.remove_tree()
    tablet_replica1.remove_tree()
    tablet_replica2.remove_tree()
Esempio n. 57
0
def tearDownModule():
  global vtgate_server
  logging.debug("in tearDownModule")
  if utils.options.skip_teardown:
    return
  logging.debug("Tearing down the servers and setup")
  utils.vtgate_kill(vtgate_server)
  tablet.kill_tablets([shard_0_master, shard_1_master, lookup_master])
  teardown_procs = [shard_0_master.teardown_mysql(),
                    shard_1_master.teardown_mysql(),
                    lookup_master.teardown_mysql(),
                   ]
  utils.wait_procs(teardown_procs, raise_on_error=False)

  environment.topo_server().teardown()

  utils.kill_sub_processes()
  utils.remove_tmp_files()

  shard_0_master.remove_tree()
  shard_1_master.remove_tree()
  lookup_master.remove_tree()
Esempio n. 58
0
def tearDownModule():
    if utils.options.skip_teardown:
        return

    tablet.kill_tablets([src_master, src_replica, dst_master, dst_replica])

    teardown_procs = [
        src_master.teardown_mysql(),
        src_replica.teardown_mysql(),
        dst_master.teardown_mysql(),
        dst_replica.teardown_mysql(),
    ]
    utils.wait_procs(teardown_procs, raise_on_error=False)

    environment.topo_server().teardown()
    utils.kill_sub_processes()
    utils.remove_tmp_files()

    src_master.remove_tree()
    src_replica.remove_tree()
    dst_master.remove_tree()
    dst_replica.remove_tree()
Esempio n. 59
0
  def setUp(self):
    environment.topo_server().wipe()
    if environment.topo_server().flavor() == 'zookeeper':
      utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc1')
      utils.run(environment.binary_argstr('zk')+' touch -p /zk/test_nj/vt/zkocc2')
      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename1 = fd.name
      fd.write("Test data 1")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename1+' /zk/test_nj/vt/zkocc1/data1')

      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename2 = fd.name
      fd.write("Test data 2")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename2+' /zk/test_nj/vt/zkocc1/data2')

      fd = tempfile.NamedTemporaryFile(dir=environment.tmproot, delete=False)
      filename3 = fd.name
      fd.write("Test data 3")
      fd.close()
      utils.run(environment.binary_argstr('zk')+' cp '+filename3+' /zk/test_nj/vt/zkocc1/data3')
Esempio n. 60
0
def setUpModule():
    try:
        environment.topo_server().setup()

        setup_procs = [
            shard_master.init_mysql(),
            shard_replica.init_mysql(),
            shard_rdonly1.init_mysql(),
            shard_0_master.init_mysql(),
            shard_0_replica.init_mysql(),
            shard_0_rdonly1.init_mysql(),
            shard_1_master.init_mysql(),
            shard_1_replica.init_mysql(),
            shard_1_rdonly1.init_mysql(),
        ]
        # we only use vtgate for testing SplitQuery works correctly.
        # we want cache_ttl at zero so we re-read the topology for every test query.
        utils.VtGate().start(cache_ttl='0')
        utils.wait_procs(setup_procs)
    except:
        tearDownModule()
        raise