Beispiel #1
0
def setUpModule():
  global vtgateclienttest_process
  global vtgateclienttest_port
  global vtgateclienttest_grpc_port

  try:
    environment.topo_server().setup()

    vtgateclienttest_port = environment.reserve_ports(1)
    args = environment.binary_args('vtgateclienttest') + [
        '-log_dir', environment.vtlogroot,
        '-port', str(vtgateclienttest_port),
        ]

    if protocols_flavor().vtgate_python_protocol() == 'grpc':
      vtgateclienttest_grpc_port = environment.reserve_ports(1)
      args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])

    vtgateclienttest_process = utils.run_bg(args)
    utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
  except:
    tearDownModule()
    raise
Beispiel #2
0
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s', auth=False, timeout="5s", cert=None, key=None, ca_cert=None):
  port = vtport or environment.reserve_ports(1)
  secure_port = None
  args = [environment.binary_path('vtgate'),
          '-port', str(port),
          '-cell', cell,
          '-retry-delay', '%ss' % (str(retry_delay)),
          '-retry-count', str(retry_count),
          '-log_dir', environment.vtlogroot,
          '-srv_topo_cache_ttl', cache_ttl,
          '-timeout', timeout,
          ] + environment.tabletconn_protocol_flags()
  if topo_impl:
    args.extend(['-topo_implementation', topo_impl])
  else:
    args.extend(environment.topo_server_flags())
  if tablet_bson_encrypted:
    args.append('-tablet-bson-encrypted')
  if auth:
    args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
  if cert:
    secure_port = environment.reserve_ports(1)
    args.extend(['-secure-port', '%s' % secure_port,
                 '-cert', cert,
                 '-key', key])
    if ca_cert:
      args.extend(['-ca-cert', ca_cert])

  sp = run_bg(args)
  if cert:
    wait_for_vars("vtgate", port, "SecureConnections")
    return sp, port, secure_port
  else:
    wait_for_vars("vtgate", port)
    return sp, port
Beispiel #3
0
  def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None,
               use_mysqlctld=False):
    self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
    self.port = port or (environment.reserve_ports(1))
    self.mysql_port = mysql_port or (environment.reserve_ports(1))
    self.grpc_port = environment.reserve_ports(1)
    self.use_mysqlctld = use_mysqlctld
    Tablet.seq += 1

    if cell:
      self.cell = cell
    else:
      self.cell = tablet_cell_map.get(tablet_uid, 'nj')
    self.proc = None

    # filled in during init_tablet
    self.keyspace = None
    self.shard = None
    self.index = None
    self.tablet_index = None

    # utility variables
    self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
    self.zk_tablet_path = (
        '/zk/test_%s/vt/tablets/%010d' % (self.cell, self.tablet_uid))
Beispiel #4
0
def _get_vtworker_cmd(clargs, auto_log=False):
  """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
  port = environment.reserve_ports(1)
  rpc_port = port
  args = environment.binary_args('vtworker') + [
          '-log_dir', environment.vtlogroot,
          '-min_healthy_rdonly_endpoints', '1',
          '-port', str(port),
          '-resolve_ttl', '2s',
          '-executefetch_retry_time', '1s',
          ]
  args.extend(environment.topo_server().flags())
  args.extend(['-tablet_manager_protocol',
               protocols_flavor().tablet_manager_protocol()])
  if protocols_flavor().service_map():
    args.extend(['-service_map',
                 ",".join(protocols_flavor().service_map())])
  if protocols_flavor().vtworker_client_protocol() == 'grpc':
    rpc_port = environment.reserve_ports(1)
    args.extend(['-grpc_port', str(rpc_port)])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  cmd = args + clargs
  return cmd, port, rpc_port
Beispiel #5
0
 def __init__(self, port=None):
   """Creates the Vtgate instance and reserve the ports if necessary."""
   self.port = port or environment.reserve_ports(1)
   if protocols_flavor().vtgate_protocol() == 'grpc':
     self.grpc_port = environment.reserve_ports(1)
   self.secure_port = None
   self.proc = None
Beispiel #6
0
 def __init__(self, port=None, mysql_server=False):
     """Creates the Vtgate instance and reserve the ports if necessary."""
     self.port = port or environment.reserve_ports(1)
     if protocols_flavor().vtgate_protocol() == 'grpc':
         self.grpc_port = environment.reserve_ports(1)
     self.proc = None
     self.mysql_port = None
     if mysql_server:
         self.mysql_port = environment.reserve_ports(1)
Beispiel #7
0
 def __init__(self, port=None, cert=None, key=None):
   """Creates the Vtgate instance and reserve the ports if necessary.
   """
   self.port = port or environment.reserve_ports(1)
   self.secure_port = None
   if cert:
     self.secure_port = environment.reserve_ports(1)
     self.cert = cert
     self.key = key
   self.proc = None
Beispiel #8
0
 def __init__(self, port=None, cert=None, key=None):
     """Creates the Vtgate instance and reserve the ports if necessary.
 """
     self.port = port or environment.reserve_ports(1)
     self.secure_port = None
     if cert:
         self.secure_port = environment.reserve_ports(1)
         self.cert = cert
         self.key = key
     self.proc = None
Beispiel #9
0
  def assign_ports(self):
    """Assign ports if not already assigned"""

    if self.ports_assigned:
      return

    from environment import reserve_ports

    self.zk_port_base = reserve_ports(3)
    self.zkocc_port_base = reserve_ports(3)
    self.hostname = socket.gethostname()
    self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
    self.zk_client_port = self.zk_port_base + 2
    self.ports_assigned = True
Beispiel #10
0
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1,
                 topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s',
                 auth=False, timeout="5s", cert=None, key=None, ca_cert=None,
                 socket_file=None, schema=None, extra_args=None):
  port = vtport or environment.reserve_ports(1)
  secure_port = None
  args = environment.binary_args('vtgate') + [
          '-port', str(port),
          '-cell', cell,
          '-retry-delay', '%ss' % (str(retry_delay)),
          '-retry-count', str(retry_count),
          '-log_dir', environment.vtlogroot,
          '-srv_topo_cache_ttl', cache_ttl,
          '-timeout', timeout,
          ] + protocols_flavor().tabletconn_protocol_flags()
  if topo_impl:
    args.extend(['-topo_implementation', topo_impl])
  else:
    args.extend(environment.topo_server().flags())
  if tablet_bson_encrypted:
    args.append('-tablet-bson-encrypted')
  if auth:
    args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
  if cert:
    secure_port = environment.reserve_ports(1)
    args.extend(['-secure-port', '%s' % secure_port,
                 '-cert', cert,
                 '-key', key])
    if ca_cert:
      args.extend(['-ca_cert', ca_cert])
  if socket_file:
    args.extend(['-socket_file', socket_file])
  if schema:
    fname = os.path.join(environment.tmproot, "vtgate_schema.json")
    with open(fname, "w") as f:
      f.write(schema)
    args.extend(['-schema-file', fname])

  if extra_args:
    args.extend(extra_args)

  sp = run_bg(args)
  if cert:
    wait_for_vars("vtgate", port, "SecureConnections")
    return sp, port, secure_port
  else:
    wait_for_vars("vtgate", port)
    return sp, port
Beispiel #11
0
  def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
    """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
    environment.prog_compile('vtaction')
    args = [environment.binary_path('vttablet'),
            '-port', '%s' % (port or self.port),
            '-tablet-path', self.tablet_alias,
            '-log_dir', environment.vtlogroot]
    args.extend(environment.topo_server_flags())
    args.extend(environment.binlog_player_protocol_flags())

    dbconfigs = self._get_db_configs_file(repl_extra_flags)
    for key1 in dbconfigs:
      for key2 in dbconfigs[key1]:
        args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])

    if memcache:
      if os.path.exists(environment.vtroot + "/bin/memcached"):
        args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
      else:
        args.extend(["-rowcache-bin", "memcached"])
      memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
      args.extend(["-rowcache-socket", memcache_socket])
      args.extend(["-enable-rowcache"])

    if auth:
      args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])

    if customrules:
      args.extend(['-customrules', customrules])

    if schema_override:
      args.extend(['-schema-override', schema_override])

    if cert:
      self.secure_port = environment.reserve_ports(1)
      args.extend(['-secure-port', '%s' % self.secure_port,
                   '-cert', cert,
                   '-key', key])
      if ca_cert:
        args.extend(['-ca-cert', ca_cert])

    stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
    # increment count only the first time
    if not self.proc:
      Tablet.tablets_running += 1
    self.proc = utils.run_bg(args, stderr=stderr_fd)
    stderr_fd.close()

    # wait for zookeeper PID just to be sure we have it
    if environment.topo_server_implementation == 'zookeeper':
      utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)

    # wait for query service to be in the right state
    if wait_for_state:
      self.wait_for_vttablet_state(wait_for_state, port=port)

    return self.proc
Beispiel #12
0
  def test_standalone(self):
    """Sample test for run_local_database.py as a standalone process.
    """

    # launch a backend database based on the provided topology and schema
    port = environment.reserve_ports(1)
    args = [environment.run_local_database,
            '--port', str(port),
            '--topology',
            'test_keyspace/-80:test_keyspace_0,'
            'test_keyspace/80-:test_keyspace_1',
            '--schema_dir', os.path.join(environment.vttop, 'test',
                                         'vttest_schema')]
    sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    config = json.loads(sp.stdout.readline())

    # gather the vars for the vtgate process
    url = 'http://localhost:%d/debug/vars' % config['port']
    f = urllib.urlopen(url)
    data = f.read()
    f.close()
    json_vars = json.loads(data)
    self.assertIn('vtgate', json_vars['cmdline'][0])

    # and we're done, clean-up
    sp.stdin.write('\n')
    sp.wait()
Beispiel #13
0
  def start_vttablet(self, port=None, auth=False, memcache=False, wait_for_state="SERVING", customrules=None, schema_override=None, cert=None, key=None, ca_cert=None, repl_extra_flags={}):
    """
    Starts a vttablet process, and returns it.
    The process is also saved in self.proc, so it's easy to kill as well.
    """
    environment.prog_compile('vtaction')
    args = [environment.binary_path('vttablet'),
            '-port', '%s' % (port or self.port),
            '-tablet-path', self.tablet_alias,
            '-log_dir', environment.vtlogroot]
    args.extend(environment.topo_server_flags())
    args.extend(utils.binlog_player_protocol_flags)

    dbconfigs = self._get_db_configs_file(repl_extra_flags)
    for key1 in dbconfigs:
      for key2 in dbconfigs[key1]:
        args.extend(["-db-config-"+key1+"-"+key2, dbconfigs[key1][key2]])

    if memcache:
      if os.path.exists(environment.vtroot + "/bin/memcached"):
        args.extend(["-rowcache-bin", environment.vtroot + "/bin/memcached"])
      else:
        args.extend(["-rowcache-bin", "memcached"])
      memcache_socket = os.path.join(self.tablet_dir, "memcache.sock")
      args.extend(["-rowcache-socket", memcache_socket])
      args.extend(["-enable-rowcache"])

    if auth:
      args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])

    if customrules:
      args.extend(['-customrules', customrules])

    if schema_override:
      args.extend(['-schema-override', schema_override])

    if cert:
      self.secure_port = environment.reserve_ports(1)
      args.extend(['-secure-port', '%s' % self.secure_port,
                   '-cert', cert,
                   '-key', key])
      if ca_cert:
        args.extend(['-ca-cert', ca_cert])

    stderr_fd = open(os.path.join(self.tablet_dir, "vttablet.stderr"), "w")
    # increment count only the first time
    if not self.proc:
      Tablet.tablets_running += 1
    self.proc = utils.run_bg(args, stderr=stderr_fd)
    stderr_fd.close()

    # wait for zookeeper PID just to be sure we have it
    if environment.topo_server_implementation == 'zookeeper':
      utils.run(environment.binary_path('zk')+' wait -e ' + self.zk_pid, stdout=utils.devnull)

    # wait for query service to be in the right state
    if wait_for_state:
      self.wait_for_vttablet_state(wait_for_state, port=port)

    return self.proc
Beispiel #14
0
def run_vtworker(clargs,
                 log_level='',
                 auto_log=False,
                 expect_fail=False,
                 **kwargs):
    args = [
        environment.binary_path('vtworker'), '-log_dir', environment.vtlogroot,
        '-port',
        str(environment.reserve_ports(1))
    ]

    if auto_log:
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'
    if log_level:
        args.append('--stderrthreshold=%s' % log_level)

    cmd = args + clargs
    if expect_fail:
        return run_fail(cmd, **kwargs)
    return run(cmd, **kwargs)
Beispiel #15
0
  def __init__(self, name):
    import environment  # pylint: disable=g-import-not-at-top
    import utils  # pylint: disable=g-import-not-at-top

    self.port_base = environment.reserve_ports(2)

    self.name = name
    self.hostname = 'localhost'
    self.client_port = self.port_base
    self.peer_port = self.port_base + 1
    self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
    self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
    self.api_url = self.client_addr + '/v2'

    dirname = 'etcd_' + self.name
    self.data_dir = os.path.join(environment.vtdataroot, dirname)

    self.proc = utils.run_bg([
        'etcd', '-name', self.name,
        '-advertise-client-urls', self.client_addr,
        '-initial-advertise-peer-urls', self.peer_addr,
        '-listen-client-urls', self.client_addr,
        '-listen-peer-urls', self.peer_addr,
        '-initial-cluster', '%s=%s' % (self.name, self.peer_addr),
        '-data-dir', self.data_dir],
                             stdout=open(os.path.join(
                                 environment.vtlogroot,
                                 dirname + '.stdout'),
                                         'w'),
                             stderr=open(os.path.join(
                                 environment.vtlogroot,
                                 dirname + '.stderr'),
                                         'w'),)
Beispiel #16
0
  def setUpClass(cls):
    """Set up two keyspaces: one unsharded, one with two shards."""
    topology = vttest_pb2.VTTestTopology()
    topology.cells.append('test')
    topology.cells.append('test2')
    keyspace = topology.keyspaces.add(name='test_keyspace')
    keyspace.replica_count = 2
    keyspace.rdonly_count = 2
    keyspace.shards.add(name='-80')
    keyspace.shards.add(name='80-')
    keyspace2 = topology.keyspaces.add(name='test_keyspace2')
    keyspace2.shards.add(name='0')
    keyspace2.replica_count = 2
    keyspace2.rdonly_count = 1

    cls.driver = environment.create_webdriver()

    port = environment.reserve_ports(1)
    vttest_environment.base_port = port

    environment.reset_mysql_flavor()

    cls.db = local_database.LocalDatabase(
        topology,
        os.path.join(environment.vttop, 'test/vttest_schema'),
        False, None,
        web_dir=os.path.join(environment.vttop, 'web/vtctld'),
        default_schema_dir=os.path.join(
            environment.vttop, 'test/vttest_schema/default'),
        web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
    cls.db.setup()
    cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
    utils.pause('Paused test after vtcombo was started.\n'
                'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
Beispiel #17
0
    def test_standalone(self):
        """Sample test for run_local_database.py as a standalone process.
    """

        # launch a backend database based on the provided topology and schema
        port = environment.reserve_ports(1)
        args = [
            environment.run_local_database, '--port',
            str(port), '--topology', 'test_keyspace/-80:test_keyspace_0,'
            'test_keyspace/80-:test_keyspace_1', '--schema_dir',
            os.path.join(environment.vttop, 'test', 'vttest_schema')
        ]
        sp = subprocess.Popen(args,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE)
        config = json.loads(sp.stdout.readline())

        # gather the vars for the vtgate process
        url = 'http://localhost:%d/debug/vars' % config['port']
        f = urllib.urlopen(url)
        data = f.read()
        f.close()
        vars = json.loads(data)
        self.assertIn('vtgate', vars['cmdline'][0])

        # and we're done, clean-up
        sp.stdin.write('\n')
        sp.wait()
Beispiel #18
0
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
  """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
  """
  port = environment.reserve_ports(1)
  args = environment.binary_args('vtworker') + [
          '-log_dir', environment.vtlogroot,
          '-min_healthy_rdonly_endpoints', '1',
          '-port', str(port),
          '-resolve_ttl', '2s',
          '-executefetch_retry_time', '1s',
          ]
  args.extend(environment.topo_server().flags())
  args.extend(protocols_flavor().tablet_manager_protocol_flags())

  if auto_log:
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'
  if log_level:
    args.append('--stderrthreshold=%s' % log_level)

  cmd = args + clargs
  return cmd, port
Beispiel #19
0
    def __init__(self,
                 tablet_uid=None,
                 cell=None,
                 vt_dba_passwd=None,
                 mysql_port=None):
        self.tablet_uid = tablet_uid or (Vtbackup.default_uid + Vtbackup.seq)
        self.mysql_port = mysql_port or (environment.reserve_ports(1))
        self.vt_dba_passwd = vt_dba_passwd
        Vtbackup.seq += 1

        if cell:
            self.cell = cell
        else:
            self.cell = tablet_cell_map.get(tablet_uid, 'nj')
        self.proc = None

        # filled in during init_mysql
        self.mysqlctld_process = None

        # filled in during init_tablet
        self.keyspace = None
        self.shard = None
        self.index = None
        self.tablet_index = None
        self.dbname = None
        # default to false
        self.external_mysql = False
        # utility variables
        self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
Beispiel #20
0
def run_vtworker(clargs,
                 log_level='',
                 auto_log=False,
                 expect_fail=False,
                 **kwargs):
    args = environment.binary_args('vtworker') + [
        '-log_dir', environment.vtlogroot, '-port',
        str(environment.reserve_ports(1))
    ]
    args.extend(environment.topo_server().flags())
    args.extend(protocols_flavor().tablet_manager_protocol_flags())

    if auto_log:
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'
    if log_level:
        args.append('--stderrthreshold=%s' % log_level)

    cmd = args + clargs
    if expect_fail:
        return run_fail(cmd, **kwargs)
    return run(cmd, **kwargs)
Beispiel #21
0
    def __init__(self, name):
        import environment
        import utils

        self.port_base = environment.reserve_ports(2)

        self.name = name
        self.hostname = 'localhost'
        self.client_port = self.port_base
        self.peer_port = self.port_base + 1
        self.client_addr = 'http://%s:%u' % (self.hostname, self.client_port)
        self.peer_addr = 'http://%s:%u' % (self.hostname, self.peer_port)
        self.api_url = self.client_addr + '/v2'

        dirname = 'etcd_' + self.name
        self.data_dir = os.path.join(environment.vtdataroot, dirname)

        self.proc = utils.run_bg(
            [
                'etcd', '-name', self.name, '-advertise-client-urls',
                self.client_addr, '-initial-advertise-peer-urls',
                self.peer_addr, '-listen-client-urls', self.client_addr,
                '-listen-peer-urls', self.peer_addr, '-initial-cluster',
                '%s=%s' %
                (self.name, self.peer_addr), '-data-dir', self.data_dir
            ],
            stdout=open(
                os.path.join(environment.vtlogroot, dirname + '.stdout'), 'w'),
            stderr=open(
                os.path.join(environment.vtlogroot, dirname + '.stderr'), 'w'),
        )
Beispiel #22
0
  def __init__(self, name):
    import environment
    import utils

    self.port_base = environment.reserve_ports(2)

    self.name = name
    self.hostname = 'localhost'
    self.client_port = self.port_base
    self.peer_port = self.port_base + 1
    self.client_addr = '%s:%u' % (self.hostname, self.client_port)
    self.peer_addr = '%s:%u' % (self.hostname, self.peer_port)
    self.api_url = 'http://%s/v2' % (self.client_addr)

    dirname = 'etcd_' + self.name
    self.data_dir = os.path.join(environment.vtdataroot, dirname)

    self.proc = utils.run_bg([
        'etcd', '-name', self.name, '-addr',
        self.client_addr, '-peer-addr',
        self.peer_addr, '-data-dir', self.data_dir],
                             stdout=open(os.path.join(
                                 environment.vtlogroot,
                                 dirname + '.stdout'),
                                         'w'),
                             stderr=open(os.path.join(
                                 environment.vtlogroot,
                                 dirname + '.stderr'),
                                         'w'),)
Beispiel #23
0
def _get_vtworker_cmd(clargs, auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Args:
    clargs: Command line arguments passed to vtworker.
    auto_log: If true, set --stderrthreshold according to the test log level.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
    port = environment.reserve_ports(1)
    rpc_port = port
    args = environment.binary_args('vtworker') + [
        '-log_dir',
        environment.vtlogroot,
        '-min_healthy_rdonly_endpoints',
        '1',
        '-port',
        str(port),
        # use a long resolve TTL because of potential race conditions with doing
        # an EmergencyReparent and resolving the master (as EmergencyReparent
        # will delete the old master before updating the shard record with the
        # new master)
        '-resolve_ttl',
        '10s',
        '-executefetch_retry_time',
        '1s',
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol(),
        '-tablet_protocol',
        protocols_flavor().tabletconn_protocol(),
    ]
    args.extend(environment.topo_server().flags())
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ','.join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == 'grpc':
        rpc_port = environment.reserve_ports(1)
        args.extend(['-grpc_port', str(rpc_port)])

    if auto_log:
        args.append('--stderrthreshold=%s' % get_log_level())

    cmd = args + clargs
    return cmd, port, rpc_port
Beispiel #24
0
def _get_vtworker_cmd(clargs, auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Args:
    clargs: Command line arguments passed to vtworker.
    auto_log: If true, set --stderrthreshold according to the test log level.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
    port = environment.reserve_ports(1)
    rpc_port = port
    args = environment.binary_args("vtworker") + [
        "-log_dir",
        environment.vtlogroot,
        "-min_healthy_rdonly_endpoints",
        "1",
        "-port",
        str(port),
        # use a long resolve TTL because of potential race conditions with doing
        # an EmergencyReparent and resolving the master (as EmergencyReparent
        # will delete the old master before updating the shard record with the
        # new master)
        "-resolve_ttl",
        "10s",
        "-executefetch_retry_time",
        "1s",
        "-tablet_manager_protocol",
        protocols_flavor().tablet_manager_protocol(),
        "-tablet_protocol",
        protocols_flavor().tabletconn_protocol(),
    ]
    args.extend(environment.topo_server().flags())
    if protocols_flavor().service_map():
        args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == "grpc":
        rpc_port = environment.reserve_ports(1)
        args.extend(["-grpc_port", str(rpc_port)])

    if auto_log:
        args.append("--stderrthreshold=%s" % get_log_level())

    cmd = args + clargs
    return cmd, port, rpc_port
Beispiel #25
0
    def setup(self, add_bad_host=False):
        from environment import reserve_ports, run, binary_args, vtlogroot, tmproot

        self.zk_port_base = reserve_ports(3)
        self.zkocc_port_base = reserve_ports(3)

        self.hostname = socket.gethostname()
        self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
        self.zk_client_port = self.zk_port_base + 2

        run(
            binary_args('zkctl') + [
                '-log_dir', vtlogroot, '-zk.cfg',
                '1@%s:%s' % (self.hostname, self.zk_ports), 'init'
            ])
        config = tmproot + '/test-zk-client-conf.json'
        with open(config, 'w') as f:
            ca_server = 'localhost:%u' % (self.zk_client_port)
            if add_bad_host:
                ca_server += ',does.not.exists:1234'
            zk_cell_mapping = {
                'test_nj':
                'localhost:%u' % (self.zk_client_port),
                'test_ny':
                'localhost:%u' % (self.zk_client_port),
                'test_ca':
                ca_server,
                'global':
                'localhost:%u' % (self.zk_client_port),
                'test_nj:_zkocc':
                'localhost:%u,localhost:%u,localhost:%u' %
                tuple(self.zkocc_port_base + i for i in range(3)),
                'test_ny:_zkocc':
                'localhost:%u' % (self.zkocc_port_base),
                'test_ca:_zkocc':
                'localhost:%u' % (self.zkocc_port_base),
                'global:_zkocc':
                'localhost:%u' % (self.zkocc_port_base),
            }
            json.dump(zk_cell_mapping, f)
        os.environ['ZK_CLIENT_CONFIG'] = config
        logging.debug('Using ZK_CLIENT_CONFIG=%s', str(config))
        run(binary_args('zk') + ['touch', '-p', '/zk/test_nj/vt'])
        run(binary_args('zk') + ['touch', '-p', '/zk/test_ny/vt'])
        run(binary_args('zk') + ['touch', '-p', '/zk/test_ca/vt'])
Beispiel #26
0
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
  """
    port = environment.reserve_ports(1)
    rpc_port = environment.reserve_ports(1)
    args = environment.binary_args('vtworker') + [
        '-log_dir',
        environment.vtlogroot,
        '-min_healthy_rdonly_endpoints',
        '1',
        '-port',
        str(port),
        '-resolve_ttl',
        '2s',
        '-executefetch_retry_time',
        '1s',
    ]
    args.extend(environment.topo_server().flags())
    args.extend([
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol()
    ])
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ",".join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == 'grpc':
        args.extend(['-grpc_port', str(rpc_port)])

    if auto_log:
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'
    if log_level:
        args.append('--stderrthreshold=%s' % log_level)

    cmd = args + clargs
    return cmd, port, rpc_port
Beispiel #27
0
 def setUpClass(cls):
   cls.driver = environment.create_webdriver()
   port = environment.reserve_ports(1)
   keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar')
   cls.flask_process = subprocess.Popen(
       [os.path.join(keytar_folder, 'keytar.py'),
        '--config_file=%s' % os.path.join(keytar_folder, 'test_config.yaml'),
        '--port=%d' % port, '--password=foo'],
       preexec_fn=os.setsid)
   cls.flask_addr = 'http://localhost:%d' % port
Beispiel #28
0
def vtgate_start(vtport=None, cell='test_nj', retry_delay=1, retry_count=1,
                 topo_impl=None, tablet_bson_encrypted=False, cache_ttl='1s',
                 auth=False, timeout="5s", cert=None, key=None, ca_cert=None,
                 socket_file=None, extra_args=None):
  port = vtport or environment.reserve_ports(1)
  secure_port = None
  args = environment.binary_args('vtgate') + [
          '-port', str(port),
          '-cell', cell,
          '-retry-delay', '%ss' % (str(retry_delay)),
          '-retry-count', str(retry_count),
          '-log_dir', environment.vtlogroot,
          '-srv_topo_cache_ttl', cache_ttl,
          '-timeout', timeout,
          ] + environment.tabletconn_protocol_flags()
  if topo_impl:
    args.extend(['-topo_implementation', topo_impl])
  else:
    args.extend(environment.topo_server_flags())
  if tablet_bson_encrypted:
    args.append('-tablet-bson-encrypted')
  if auth:
    args.extend(['-auth-credentials', os.path.join(environment.vttop, 'test', 'test_data', 'authcredentials_test.json')])
  if cert:
    secure_port = environment.reserve_ports(1)
    args.extend(['-secure-port', '%s' % secure_port,
                 '-cert', cert,
                 '-key', key])
    if ca_cert:
      args.extend(['-ca_cert', ca_cert])
  if socket_file:
    args.extend(['-socket_file', socket_file])
  if extra_args:
    args.extend(extra_args)

  sp = run_bg(args)
  if cert:
    wait_for_vars("vtgate", port, "SecureConnections")
    return sp, port, secure_port
  else:
    wait_for_vars("vtgate", port)
    return sp, port
Beispiel #29
0
  def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None):
    self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
    self.port = port or (environment.reserve_ports(1))
    self.mysql_port = mysql_port or (environment.reserve_ports(1))
    Tablet.seq += 1

    if cell:
      self.cell = cell
    else:
      self.cell = tablet_cell_map.get(tablet_uid, 'nj')
    self.proc = None

    # filled in during init_tablet
    self.keyspace = None
    self.shard = None

    # utility variables
    self.tablet_alias = 'test_%s-%010d' % (self.cell, self.tablet_uid)
    self.zk_tablet_path = '/zk/test_%s/vt/tablets/%010d' % (self.cell, self.tablet_uid)
    self.zk_pid = self.zk_tablet_path + '/pid'
Beispiel #30
0
    def __init__(self, tablet_uid=None, port=None, mysql_port=None, cell=None):
        self.tablet_uid = tablet_uid or (Tablet.default_uid + Tablet.seq)
        self.port = port or (environment.reserve_ports(1))
        self.mysql_port = mysql_port or (environment.reserve_ports(1))
        Tablet.seq += 1

        if cell:
            self.cell = cell
        else:
            self.cell = tablet_cell_map.get(tablet_uid, "nj")
        self.proc = None

        # filled in during init_tablet
        self.keyspace = None
        self.shard = None

        # utility variables
        self.tablet_alias = "test_%s-%010d" % (self.cell, self.tablet_uid)
        self.zk_tablet_path = "/zk/test_%s/vt/tablets/%010d" % (self.cell, self.tablet_uid)
        self.zk_pid = self.zk_tablet_path + "/pid"
Beispiel #31
0
    def setUpClass(cls):
        """Set up two keyspaces: one unsharded, one with two shards."""
        topology = vttest_pb2.VTTestTopology()
        topology.cells.append('test')
        topology.cells.append('test2')
        keyspace = topology.keyspaces.add(name='test_keyspace')
        keyspace.replica_count = 2
        keyspace.rdonly_count = 2
        keyspace.shards.add(name='-80')
        keyspace.shards.add(name='80-')
        keyspace2 = topology.keyspaces.add(name='test_keyspace2')
        keyspace2.shards.add(name='0')
        keyspace2.replica_count = 2
        keyspace2.rdonly_count = 1

        if os.environ.get('CI') == 'true' and os.environ.get(
                'TRAVIS') == 'true':
            username = os.environ['SAUCE_USERNAME']
            access_key = os.environ['SAUCE_ACCESS_KEY']
            capabilities = {}
            capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
            capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
            capabilities['platform'] = 'Linux'
            capabilities['browserName'] = 'chrome'
            hub_url = '%s:%s@localhost:4445' % (username, access_key)
            cls.driver = webdriver.Remote(desired_capabilities=capabilities,
                                          command_executor='http://%s/wd/hub' %
                                          hub_url)
        else:
            os.environ['webdriver.chrome.driver'] = os.path.join(
                environment.vtroot, 'dist')
            # Only testing against Chrome for now
            cls.driver = webdriver.Chrome()
            cls.driver.set_window_position(0, 0)
            cls.driver.set_window_size(1920, 1280)

        port = environment.reserve_ports(1)
        vttest_environment.base_port = port
        mysql_flavor.set_mysql_flavor(None)

        cls.db = local_database.LocalDatabase(
            topology,
            os.path.join(environment.vttop, 'test/vttest_schema'),
            False,
            None,
            web_dir=os.path.join(environment.vttop, 'web/vtctld'),
            default_schema_dir=os.path.join(environment.vttop,
                                            'test/vttest_schema/default'),
            web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app'))
        cls.db.setup()
        cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
        utils.pause('Paused test after vtcombo was started.\n'
                    'For manual testing, connect to vtctld: %s' %
                    cls.vtctld_addr)
Beispiel #32
0
 def setUpClass(cls):
     cls.driver = environment.create_webdriver()
     port = environment.reserve_ports(1)
     keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar')
     cls.flask_process = subprocess.Popen([
         os.path.join(keytar_folder, 'keytar.py'),
         '--config_file=%s' %
         os.path.join(keytar_folder, 'test_config.yaml'),
         '--port=%d' % port, '--password=foo'
     ],
                                          preexec_fn=os.setsid)
     cls.flask_addr = 'http://localhost:%d' % port
Beispiel #33
0
    def setUpClass(cls):
        """Set up two keyspaces: one unsharded, one with two shards."""
        topology = vttest_pb2.VTTestTopology()
        topology.cells.append("test")
        topology.cells.append("test2")
        keyspace = topology.keyspaces.add(name="test_keyspace")
        keyspace.replica_count = 2
        keyspace.rdonly_count = 2
        keyspace.shards.add(name="-80")
        keyspace.shards.add(name="80-")
        keyspace2 = topology.keyspaces.add(name="test_keyspace2")
        keyspace2.shards.add(name="0")
        keyspace2.replica_count = 2
        keyspace2.rdonly_count = 1

        if os.environ.get("CI") == "true" and os.environ.get("TRAVIS") == "true":
            username = os.environ["SAUCE_USERNAME"]
            access_key = os.environ["SAUCE_ACCESS_KEY"]
            capabilities = {}
            capabilities["tunnel-identifier"] = os.environ["TRAVIS_JOB_NUMBER"]
            capabilities["build"] = os.environ["TRAVIS_BUILD_NUMBER"]
            capabilities["platform"] = "Linux"
            capabilities["browserName"] = "chrome"
            hub_url = "%s:%s@localhost:4445" % (username, access_key)
            cls.driver = webdriver.Remote(
                desired_capabilities=capabilities, command_executor="http://%s/wd/hub" % hub_url
            )
        else:
            os.environ["webdriver.chrome.driver"] = os.path.join(environment.vtroot, "dist")
            # Only testing against Chrome for now
            cls.driver = webdriver.Chrome()
            cls.driver.set_window_position(0, 0)
            cls.driver.set_window_size(1920, 1280)

        port = environment.reserve_ports(1)
        vttest_environment.base_port = port
        mysql_flavor.set_mysql_flavor(None)

        cls.db = local_database.LocalDatabase(
            topology,
            os.path.join(environment.vttop, "test/vttest_schema"),
            False,
            None,
            web_dir=os.path.join(environment.vttop, "web/vtctld"),
            default_schema_dir=os.path.join(environment.vttop, "test/vttest_schema/default"),
            web_dir2=os.path.join(environment.vttop, "web/vtctld2/app"),
        )
        cls.db.setup()
        cls.vtctld_addr = "http://localhost:%d" % cls.db.config()["port"]
        utils.pause(
            "Paused test after vtcombo was started.\n" "For manual testing, connect to vtctld: %s" % cls.vtctld_addr
        )
Beispiel #34
0
def _get_vtworker_cmd(clargs, auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Args:
    clargs: Command line arguments passed to vtworker.
    auto_log: If true, set --stderrthreshold according to the test log level.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
    port = environment.reserve_ports(1)
    rpc_port = port
    args = environment.binary_args('vtworker') + [
        '-log_dir',
        environment.vtlogroot,
        '-port',
        str(port),
        '-executefetch_retry_time',
        '1s',
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol(),
        '-tablet_protocol',
        protocols_flavor().tabletconn_protocol(),
    ]
    args.extend(environment.topo_server().flags())
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ','.join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == 'grpc':
        rpc_port = environment.reserve_ports(1)
        args.extend(['-grpc_port', str(rpc_port)])

    if auto_log:
        args.append('--stderrthreshold=%s' % get_log_level())

    cmd = args + clargs
    return cmd, port, rpc_port
Beispiel #35
0
    def assign_ports(self):
        """Assign ports if not already assigned"""

        if self.ports_assigned:
            return

        from environment import reserve_ports

        self.zk_port_base = reserve_ports(3)
        self.hostname = socket.getfqdn()
        self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
        self.zk_client_port = self.zk_port_base + 2
        self.ports_assigned = True
    def assign_ports(self):
        """Assign ports if not already assigned."""

        if self.ports_assigned:
            return

        from environment import reserve_ports  # pylint: disable=g-import-not-at-top
        import utils  # pylint: disable=g-import-not-at-top

        self.zk_port_base = reserve_ports(3)
        self.hostname = utils.hostname
        self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
        self.zk_client_port = self.zk_port_base + 2
        self.ports_assigned = True
Beispiel #37
0
  def assign_ports(self):
    """Assign ports if not already assigned."""

    if self.ports_assigned:
      return

    from environment import reserve_ports  # pylint: disable=g-import-not-at-top
    import utils  # pylint: disable=g-import-not-at-top

    self.zk_port_base = reserve_ports(3)
    self.hostname = utils.hostname
    self.zk_ports = ':'.join(str(self.zk_port_base + i) for i in range(3))
    self.addr = 'localhost:%d' % (self.zk_port_base + 2)
    self.ports_assigned = True
Beispiel #38
0
def _get_vtworker_cmd(clargs, auto_log=False):
  """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
  port = environment.reserve_ports(1)
  rpc_port = port
  args = environment.binary_args('vtworker') + [
      '-log_dir', environment.vtlogroot,
      '-min_healthy_rdonly_endpoints', '1',
      '-port', str(port),
      # use a long resolve TTL because of potential race conditions with doing
      # an EmergencyReparent and resolving the master (as EmergencyReparent
      # will delete the old master before updating the shard record with the
      # new master)
      '-resolve_ttl', '10s',
      '-executefetch_retry_time', '1s',
      '-tablet_manager_protocol',
      protocols_flavor().tablet_manager_protocol(),
      '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
  ]
  args.extend(environment.topo_server().flags())
  if protocols_flavor().service_map():
    args.extend(['-service_map',
                 ','.join(protocols_flavor().service_map())])
  if protocols_flavor().vtworker_client_protocol() == 'grpc':
    rpc_port = environment.reserve_ports(1)
    args.extend(['-grpc_port', str(rpc_port)])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  cmd = args + clargs
  return cmd, port, rpc_port
Beispiel #39
0
def setUpModule():
    global vtgateclienttest_process
    global vtgateclienttest_port
    global vtgateclienttest_grpc_port

    environment.topo_server().setup()

    vtgateclienttest_port = environment.reserve_ports(1)
    args = environment.binary_args('vtgateclienttest') + [
        '-log_dir',
        environment.vtlogroot,
        '-port',
        str(vtgateclienttest_port),
    ]

    if protocols_flavor().vtgate_python_protocol() == 'grpc':
        vtgateclienttest_grpc_port = environment.reserve_ports(1)
        args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ','.join(protocols_flavor().service_map())])

    vtgateclienttest_process = utils.run_bg(args)
    utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
Beispiel #40
0
    def _test_standalone(self, use_vtcombo):
        """Sample test for run_local_database.py as a standalone process."""

        # launch a backend database based on the provided topology and schema
        port = environment.reserve_ports(1)
        args = [
            environment.run_local_database,
            '--port',
            str(port),
            '--topology',
            'test_keyspace/-80:test_keyspace_0,'
            'test_keyspace/80-:test_keyspace_1',
            '--schema_dir',
            os.path.join(environment.vttop, 'test', 'vttest_schema'),
        ]
        if use_vtcombo:
            args.append('--use_vtcombo')
        sp = subprocess.Popen(args,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE)
        config = json.loads(sp.stdout.readline())

        # gather the vars for the vtgate process
        url = 'http://localhost:%d/debug/vars' % config['port']
        f = urllib.urlopen(url)
        data = f.read()
        f.close()
        json_vars = json.loads(data)
        process_name = 'vtgate'
        if use_vtcombo:
            process_name = 'vtcombo'
        self.assertIn(process_name, json_vars['cmdline'][0])

        # to test vtcombo:
        # ./vttest_sample_test.py -v -d
        # go install && vtcombo -port 15010 -grpc_port 15011 -service_map grpc-vtgateservice -topology test_keyspace/-80:test_keyspace_0,test_keyspace/80-:test_keyspace_1 -mycnf_server_id 1 -mycnf_socket_file $VTDATAROOT/vttest*/vt_0000000001/mysql.sock -db-config-dba-uname vt_dba -db-config-dba-charset utf8 -db-config-app-uname vt_app -db-config-app-charset utf8 -alsologtostderr
        # vtctl -vtgate_protocol grpc VtGateExecuteShards -server localhost:15011 -keyspace test_keyspace -shards -80 -tablet_type master "select 1 from dual"
        # vtctl -vtgate_protocol grpc VtGateExecuteKeyspaceIds -server localhost:15011 -keyspace test_keyspace -keyspace_ids 20 -tablet_type master "show tables"
        if use_vtcombo:
            utils.pause('good time to test vtcombo with database running')
        else:
            utils.pause('good time to test vtgate with database running')

        # and we're done, clean-up
        sp.stdin.write('\n')
        sp.wait()
Beispiel #41
0
def run_automation_server(auto_log=False):
  """Starts a background automation_server process.

  Returns:
    rpc_port - int with the port number of the RPC interface
  """
  rpc_port = environment.reserve_ports(1)
  args = environment.binary_args('automation_server') + [
          '-log_dir', environment.vtlogroot,
          '-port', str(rpc_port),
          '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
          '-vtworker_client_protocol', protocols_flavor().vtworker_client_protocol(),
          ]
  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  return run_bg(args), rpc_port
Beispiel #42
0
def vtgate_start(cell='test_nj', retry_delay=1, retry_count=1, topo_impl=None, tablet_bson_encrypted=False):
  port = environment.reserve_ports(1)
  args = [environment.binary_path('vtgate'),
          '-port', str(port),
          '-cell', cell,
          '-retry-delay', '%ss' % (str(retry_delay)),
          '-retry-count', str(retry_count),
          '-log_dir', environment.vtlogroot,
          ] + environment.vtgate_protocol_flags()
  if topo_impl:
    args.extend(['-topo_implementation', topo_impl])
  else:
    args.extend(environment.topo_server_flags())
  if tablet_bson_encrypted:
    args.append('-tablet-bson-encrypted')
  sp = run_bg(args)
  wait_for_vars("vtgate", port)
  return sp, port
Beispiel #43
0
    def __init__(self, name):
        import environment  # pylint: disable=g-import-not-at-top

        self.port_base = environment.reserve_ports(2)

        self.name = name
        self.hostname = 'localhost'
        self.client_port = self.port_base
        self.peer_port = self.port_base + 1
        self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
        self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
        self.api_url = self.client_addr + '/v2'

        dirname = 'etcd_' + self.name
        self.data_dir = os.path.join(environment.vtdataroot, dirname)
        self.log_base = os.path.join(environment.vtlogroot, dirname)

        self.start()
Beispiel #44
0
def run_automation_server(auto_log=False):
  """Starts a background automation_server process.

  Returns:
    rpc_port - int with the port number of the RPC interface
  """
  rpc_port = environment.reserve_ports(1)
  args = environment.binary_args('automation_server') + [
      '-log_dir', environment.vtlogroot,
      '-port', str(rpc_port),
      '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
      '-vtworker_client_protocol',
      protocols_flavor().vtworker_client_protocol(),
  ]
  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  return run_bg(args), rpc_port
Beispiel #45
0
  def __init__(self, name):
    import environment  # pylint: disable=g-import-not-at-top

    self.port_base = environment.reserve_ports(2)

    self.name = name
    self.hostname = 'localhost'
    self.client_port = self.port_base
    self.peer_port = self.port_base + 1
    self.client_addr = 'http://%s:%d' % (self.hostname, self.client_port)
    self.peer_addr = 'http://%s:%d' % (self.hostname, self.peer_port)
    self.api_url = self.client_addr + '/v2'

    dirname = 'etcd_' + self.name
    self.data_dir = os.path.join(environment.vtdataroot, dirname)
    self.log_base = os.path.join(environment.vtlogroot, dirname)

    self.start()
Beispiel #46
0
  def setUpClass(cls):
    """Set up two keyspaces: one unsharded, one with two shards."""
    if os.environ.get('CI') == 'true' and os.environ.get('TRAVIS') == 'true':
      username = os.environ['SAUCE_USERNAME']
      access_key = os.environ['SAUCE_ACCESS_KEY']
      capabilities = {}
      capabilities['tunnel-identifier'] = os.environ['TRAVIS_JOB_NUMBER']
      capabilities['build'] = os.environ['TRAVIS_BUILD_NUMBER']
      capabilities['platform'] = 'Linux'
      capabilities['browserName'] = 'chrome'
      hub_url = '%s:%s@localhost:4445' % (username, access_key)
      cls.driver = webdriver.Remote(
          desired_capabilities=capabilities,
          command_executor='http://%s/wd/hub' % hub_url)
    else:
      os.environ['webdriver.chrome.driver'] = os.path.join(
          os.environ['VTROOT'], 'dist')
      # Only testing against Chrome for now
      cls.driver = webdriver.Chrome()

    topology = vttest_pb2.VTTestTopology()
    topology.cells.append('test')
    keyspace = topology.keyspaces.add(name='test_keyspace')
    keyspace.replica_count = 2
    keyspace.rdonly_count = 2
    keyspace.shards.add(name='-80')
    keyspace.shards.add(name='80-')
    keyspace2 = topology.keyspaces.add(name='test_keyspace2')
    keyspace2.shards.add(name='0')
    keyspace2.replica_count = 2
    keyspace2.rdonly_count = 1

    port = environment.reserve_ports(1)
    vttest_environment.base_port = port
    mysql_flavor.set_mysql_flavor(None)

    cls.db = local_database.LocalDatabase(
        topology, '', False, None,
        os.path.join(os.environ['VTTOP'], 'web/vtctld2/dist'),
        os.path.join(os.environ['VTTOP'], 'test/vttest_schema/default'))
    cls.db.setup()
    cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port']
    utils.pause('Paused test after vtcombo was started.\n'
                'For manual testing, connect to vtctld: %s' % cls.vtctld_addr)
Beispiel #47
0
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
  args = [environment.binary_path('vtworker'),
          '-log_dir', environment.vtlogroot,
          '-port', str(environment.reserve_ports(1))]

  if auto_log:
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'
  if log_level:
    args.append('--stderrthreshold=%s' % log_level)

  cmd = args + clargs
  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Beispiel #48
0
  def _test_standalone(self, use_vtcombo):
    """Sample test for run_local_database.py as a standalone process."""

    # launch a backend database based on the provided topology and schema
    port = environment.reserve_ports(1)
    args = [environment.run_local_database,
            '--port', str(port),
            '--topology',
            'test_keyspace/-80:test_keyspace_0,'
            'test_keyspace/80-:test_keyspace_1',
            '--schema_dir', os.path.join(environment.vttop, 'test',
                                         'vttest_schema'),
    ]
    if use_vtcombo:
      args.append('--use_vtcombo')
    sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    config = json.loads(sp.stdout.readline())

    # gather the vars for the vtgate process
    url = 'http://localhost:%d/debug/vars' % config['port']
    f = urllib.urlopen(url)
    data = f.read()
    f.close()
    json_vars = json.loads(data)
    process_name = 'vtgate'
    if use_vtcombo:
      process_name = 'vtcombo'
    self.assertIn(process_name, json_vars['cmdline'][0])

    # to test vtcombo:
    # ./vttest_sample_test.py -v -d
    # go install && vtcombo -port 15010 -grpc_port 15011 -service_map grpc-vtgateservice -topology test_keyspace/-80:test_keyspace_0,test_keyspace/80-:test_keyspace_1 -mycnf_server_id 1 -mycnf_socket_file $VTDATAROOT/vttest*/vt_0000000001/mysql.sock -db-config-dba-uname vt_dba -db-config-dba-charset utf8 -db-config-app-uname vt_app -db-config-app-charset utf8 -alsologtostderr
    # vtctl -vtgate_protocol grpc VtGateExecuteShards -server localhost:15011 -keyspace test_keyspace -shards -80 -tablet_type master "select 1 from dual"
    # vtctl -vtgate_protocol grpc VtGateExecuteKeyspaceIds -server localhost:15011 -keyspace test_keyspace -keyspace_ids 20 -tablet_type master "show tables"
    if use_vtcombo:
      utils.pause('good time to test vtcombo with database running')
    else:
      utils.pause('good time to test vtgate with database running')

    # and we're done, clean-up
    sp.stdin.write('\n')
    sp.wait()
Beispiel #49
0
def run_vtworker(clargs, log_level='', auto_log=False, expect_fail=False, **kwargs):
  args = environment.binary_args('vtworker') + [
          '-log_dir', environment.vtlogroot,
          '-port', str(environment.reserve_ports(1))]
  args.extend(environment.topo_server_flags())
  args.extend(environment.tablet_manager_protocol_flags())

  if auto_log:
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'
  if log_level:
    args.append('--stderrthreshold=%s' % log_level)

  cmd = args + clargs
  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Beispiel #50
0
  def setup(self):
    import environment  # pylint: disable=g-import-not-at-top
    import utils  # pylint: disable=g-import-not-at-top

    self.port_base = environment.reserve_ports(5)
    self.server_addr = 'localhost:%d' % (self.port_base + 1)

    # Write our config file.
    self.config_file = os.path.join(environment.vtdataroot, 'consul.json')
    config = {
        'ports': {
            'dns': self.port_base,
            'http': self.port_base + 1,
            'rpc': self.port_base + 2,
            'serf_lan': self.port_base + 3,
            'serf_wan': self.port_base + 4,
        },
    }
    with open(self.config_file, 'w') as fd:
      fd.write(json.dumps(config))

    log_base = os.path.join(environment.vtlogroot, 'consul')
    self.proc = utils.run_bg([
        'consul', 'agent',
        '-dev',
        '-config-file', self.config_file],
                             stdout=open(log_base + '.stdout', 'a'),
                             stderr=open(log_base + '.stderr', 'a'))

    # Wait until the daemon is ready.
    utils.curl(
        'http://' + self.server_addr + '/v1/kv/?keys', retry_timeout=10)

    # Create the cell configurations using 'vtctl AddCellInfo'
    for cell in ['test_nj', 'test_ny', 'test_ca']:
      utils.run_vtctl_vtctl(['AddCellInfo',
                             '-root', cell,
                             '-server_address', self.server_addr,
                             cell])
Beispiel #51
0
  def setup(self):
    import environment  # pylint: disable=g-import-not-at-top
    import utils  # pylint: disable=g-import-not-at-top

    self.port_base = environment.reserve_ports(5)
    self.server_addr = 'localhost:%d' % (self.port_base + 1)

    # Write our config file.
    self.config_file = os.path.join(environment.vtdataroot, 'consul.json')
    config = {
        'ports': {
            'dns': self.port_base,
            'http': self.port_base + 1,
            'rpc': self.port_base + 2,
            'serf_lan': self.port_base + 3,
            'serf_wan': self.port_base + 4,
        },
    }
    with open(self.config_file, 'w') as fd:
      fd.write(json.dumps(config))

    log_base = os.path.join(environment.vtlogroot, 'consul')
    self.proc = utils.run_bg([
        'consul', 'agent',
        '-dev',
        '-config-file', self.config_file],
                             stdout=open(log_base + '.stdout', 'a'),
                             stderr=open(log_base + '.stderr', 'a'))

    # Wait until the daemon is ready.
    utils.curl(
        'http://' + self.server_addr + '/v1/kv/?keys', retry_timeout=10)

    # Create the cell configurations using 'vtctl AddCellInfo'
    for cell in ['test_nj', 'test_ny', 'test_ca']:
      utils.run_vtctl_vtctl(['AddCellInfo',
                             '-root', cell,
                             '-server_address', self.server_addr,
                             cell])
Beispiel #52
0
def run_automation_server(auto_log=False):
    """Starts a background automation_server process.

  Args:
    auto_log: True to log.

  Returns:
    rpc_port - int with the port number of the RPC interface
  """
    rpc_port = environment.reserve_ports(1)
    args = environment.binary_args("automation_server") + [
        "-log_dir",
        environment.vtlogroot,
        "-port",
        str(rpc_port),
        "-vtctl_client_protocol",
        protocols_flavor().vtctl_client_protocol(),
        "-vtworker_client_protocol",
        protocols_flavor().vtworker_client_protocol(),
    ]
    if auto_log:
        args.append("--stderrthreshold=%s" % get_log_level())

    return run_bg(args), rpc_port
Beispiel #53
0
  for proc in proc_list:
    if proc.returncode:
      if options.verbose >= 1 and proc.returncode not in (-9,):
        sys.stderr.write("proc failed: %s %s\n" % (proc.returncode, proc.args))
      if raise_on_error:
        raise CalledProcessError(proc.returncode, ' '.join(proc.args))

def run_procs(cmds, raise_on_error=True):
  procs = []
  for cmd in cmds:
    procs.append(run_bg(cmd))
  wait_procs(procs, raise_on_error=raise_on_error)

# background zk process
# (note the zkocc addresses will only work with an extra zkocc process)
zk_port_base = environment.reserve_ports(3)
def zk_setup(add_bad_host=False):
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_path('zkctl'), environment.vtlogroot, hostname, zk_ports))
  config = environment.tmproot+'/test-zk-client-conf.json'
  with open(config, 'w') as f:
    ca_server = 'localhost:%u' % (zk_port_base+2)
    if add_bad_host:
      ca_server += ',does.not.exists:1234'
    zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
                       'test_ny': 'localhost:%u'%(zk_port_base+2),
                       'test_ca': ca_server,
                       'global': 'localhost:%u'%(zk_port_base+2),
                       'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
                       'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
Beispiel #54
0
 def __init__(self):
     self.port = environment.reserve_ports(1)
     self.schema_change_dir = os.path.join(environment.tmproot,
                                           'schema_change_test')
     if protocols_flavor().vtctl_client_protocol() == 'grpc':
         self.grpc_port = environment.reserve_ports(1)
Beispiel #55
0
 def __init__(self):
     self.port = environment.reserve_ports(1)
Beispiel #56
0
    def _test_reparent_graceful(self, shard_id):
        utils.run_vtctl('CreateKeyspace test_keyspace')

        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet('master',
                                 'test_keyspace',
                                 shard_id,
                                 start=True)
        if environment.topo_server_implementation == 'zookeeper':
            shard = utils.run_vtctl_json(
                ['GetShard', 'test_keyspace/' + shard_id])
            self.assertEqual(
                shard['Cells'], ['test_nj'],
                'wrong list of cell in Shard: %s' % str(shard['Cells']))

        # Create a few slaves for testing reparenting.
        tablet_62044.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_41983.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_31981.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        for t in [tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state("SERVING")
        if environment.topo_server_implementation == 'zookeeper':
            shard = utils.run_vtctl_json(
                ['GetShard', 'test_keyspace/' + shard_id])
            self.assertEqual(
                shard['Cells'], ['test_nj', 'test_ny'],
                'wrong list of cell in Shard: %s' % str(shard['Cells']))

        # Recompute the shard layout node - until you do that, it might not be valid.
        utils.run_vtctl('RebuildShardGraph test_keyspace/' + shard_id)
        utils.validate_topology()

        # Force the slaves to reparent assuming that all the datasets are identical.
        for t in [tablet_62344, tablet_62044, tablet_41983, tablet_31981]:
            t.reset_replication()
        utils.pause("force ReparentShard?")
        utils.run_vtctl('ReparentShard -force test_keyspace/%s %s' %
                        (shard_id, tablet_62344.tablet_alias))
        utils.validate_topology(ping_tablets=True)

        self._check_db_addr(shard_id, 'master', tablet_62344.port)

        # Convert two replica to spare. That should leave only one node serving traffic,
        # but still needs to appear in the replication graph.
        utils.run_vtctl(
            ['ChangeSlaveType', tablet_41983.tablet_alias, 'spare'])
        utils.run_vtctl(
            ['ChangeSlaveType', tablet_31981.tablet_alias, 'spare'])
        utils.validate_topology()
        self._check_db_addr(shard_id, 'replica', tablet_62044.port)

        # Run this to make sure it succeeds.
        utils.run_vtctl('ShardReplicationPositions test_keyspace/%s' %
                        shard_id,
                        stdout=utils.devnull)

        # Perform a graceful reparent operation.
        utils.pause("graceful ReparentShard?")
        utils.run_vtctl('ReparentShard test_keyspace/%s %s' %
                        (shard_id, tablet_62044.tablet_alias),
                        auto_log=True)
        utils.validate_topology()

        self._check_db_addr(shard_id, 'master', tablet_62044.port)

        tablet.kill_tablets(
            [tablet_62344, tablet_62044, tablet_41983, tablet_31981])

        # Test address correction.
        new_port = environment.reserve_ports(1)
        tablet_62044.start_vttablet(port=new_port)
        # Wait a moment for address to reregister.
        time.sleep(1.0)

        self._check_db_addr(shard_id, 'master', new_port)

        tablet_62044.kill_vttablet()
Beispiel #57
0
    def test_standalone(self):
        """Sample test for run_local_database.py as a standalone process."""

        topology = vttest_pb2.VTTestTopology()
        keyspace = topology.keyspaces.add(name='test_keyspace')
        keyspace.replica_count = 2
        keyspace.rdonly_count = 1
        keyspace.shards.add(name='-80')
        keyspace.shards.add(name='80-')
        topology.keyspaces.add(name='redirect', served_from='test_keyspace')

        # launch a backend database based on the provided topology and schema
        port = environment.reserve_ports(1)
        args = [
            environment.run_local_database,
            '--port',
            str(port),
            '--proto_topo',
            text_format.MessageToString(topology, as_one_line=True),
            '--schema_dir',
            os.path.join(environment.vttop, 'test', 'vttest_schema'),
            '--web_dir',
            environment.vttop + '/web/vtctld',
        ]
        sp = subprocess.Popen(args,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE)
        config = json.loads(sp.stdout.readline())

        # gather the vars for the vtgate process
        url = 'http://localhost:%d/debug/vars' % config['port']
        f = urllib.urlopen(url)
        data = f.read()
        f.close()
        json_vars = json.loads(data)
        self.assertIn('vtcombo', json_vars['cmdline'][0])

        # build the vtcombo address and protocol
        protocol = protocols_flavor().vttest_protocol()
        if protocol == 'grpc':
            vtgate_addr = 'localhost:%d' % config['grpc_port']
        else:
            vtgate_addr = 'localhost:%d' % config['port']
        conn_timeout = 30.0
        utils.pause('Paused test after vtcombo was started.\n'
                    'For manual testing, connect to vtgate at: %s '
                    'using protocol: %s.\n'
                    'Press enter to continue.' % (vtgate_addr, protocol))

        # Remember the current timestamp after we sleep for a bit, so we
        # can use it for UpdateStream later.
        time.sleep(2)
        before_insert = long(time.time())

        # Connect to vtgate.
        conn = vtgate_client.connect(protocol, vtgate_addr, conn_timeout)

        # Insert a row.
        row_id = 123
        keyspace_id = get_keyspace_id(row_id)
        cursor = conn.cursor(tablet_type='master',
                             keyspace='test_keyspace',
                             keyspace_ids=[pack_kid(keyspace_id)],
                             writable=True)
        cursor.begin()
        insert = ('insert into test_table (id, msg, keyspace_id) values (:id, '
                  ':msg, :keyspace_id)')
        bind_variables = {
            'id': row_id,
            'msg': 'test %s' % row_id,
            'keyspace_id': keyspace_id,
        }
        cursor.execute(insert, bind_variables)
        cursor.commit()

        # Read the row back.
        cursor.execute('select * from test_table where id=:id', {'id': row_id})
        result = cursor.fetchall()
        self.assertEqual(result[0][1], 'test 123')

        # try to insert again, see if we get the right integrity error exception
        # (this is meant to test vtcombo properly returns exceptions, and to a
        # lesser extent that the python client converts it properly)
        cursor.begin()
        with self.assertRaises(dbexceptions.IntegrityError):
            cursor.execute(insert, bind_variables)
        cursor.rollback()

        # Insert a bunch of rows with long msg values.
        bind_variables['msg'] = 'x' * 64
        id_start = 1000
        rowcount = 500
        cursor.begin()
        for i in xrange(id_start, id_start + rowcount):
            bind_variables['id'] = i
            bind_variables['keyspace_id'] = get_keyspace_id(i)
            cursor.execute(insert, bind_variables)
        cursor.commit()
        cursor.close()

        # Try to fetch a large number of rows, from a rdonly
        # (more than one streaming result packet).
        stream_cursor = conn.cursor(
            tablet_type='rdonly',
            keyspace='test_keyspace',
            keyspace_ids=[pack_kid(keyspace_id)],
            cursorclass=vtgate_cursor.StreamVTGateCursor)
        stream_cursor.execute('select * from test_table where id >= :id_start',
                              {'id_start': id_start})
        self.assertEqual(rowcount, len(list(stream_cursor.fetchall())))
        stream_cursor.close()

        # try to read a row using the redirected keyspace, to a replica this time
        row_id = 123
        keyspace_id = get_keyspace_id(row_id)
        cursor = conn.cursor(tablet_type='replica',
                             keyspace='redirect',
                             keyspace_ids=[pack_kid(keyspace_id)])
        cursor.execute('select * from test_table where id=:id', {'id': row_id})
        result = cursor.fetchall()
        self.assertEqual(result[0][1], 'test 123')
        cursor.close()

        # Try to get the update stream from the connection. This makes
        # sure that part works as well.
        count = 0
        for (event, _) in conn.update_stream('test_keyspace',
                                             topodata_pb2.MASTER,
                                             timestamp=before_insert,
                                             shard='-80'):
            for statement in event.statements:
                if statement.table_name == 'test_table':
                    count += 1
            if count == rowcount + 1:
                # We're getting the initial value, plus the 500 updates.
                break

        # Insert a sentinel value into the second shard.
        row_id = 0x8100000000000000
        keyspace_id = get_keyspace_id(row_id)
        cursor = conn.cursor(tablet_type='master',
                             keyspace='test_keyspace',
                             keyspace_ids=[pack_kid(keyspace_id)],
                             writable=True)
        cursor.begin()
        bind_variables = {
            'id': row_id,
            'msg': 'test %s' % row_id,
            'keyspace_id': keyspace_id,
        }
        cursor.execute(insert, bind_variables)
        cursor.commit()
        cursor.close()

        # Try to connect to an update stream on the other shard.
        # We may get some random update stream events, but we should not get any
        # event that's related to the first shard. Only events related to
        # the Insert we just did.
        found = False
        for (event, _) in conn.update_stream('test_keyspace',
                                             topodata_pb2.MASTER,
                                             timestamp=before_insert,
                                             shard='80-'):
            for statement in event.statements:
                self.assertEqual(statement.table_name, 'test_table')
                fields, rows = proto3_encoding.convert_stream_event_statement(
                    statement)
                self.assertEqual(fields[0], 'id')
                self.assertEqual(rows[0][0], row_id)
                found = True
            if found:
                break

        # Clean up the connection
        conn.close()

        # Test we can connect to vtcombo for vtctl actions
        protocol = protocols_flavor().vtctl_python_client_protocol()
        if protocol == 'grpc':
            vtgate_addr = 'localhost:%d' % config['grpc_port']
        else:
            vtgate_addr = 'localhost:%d' % config['port']
        out, _ = utils.run(environment.binary_args('vtctlclient') + [
            '-vtctl_client_protocol',
            protocol,
            '-server',
            vtgate_addr,
            '-stderrthreshold',
            '0',
            'ListAllTablets',
            'test',
        ],
                           trap_output=True)
        num_master = 0
        num_replica = 0
        num_rdonly = 0
        num_dash_80 = 0
        num_80_dash = 0
        for line in out.splitlines():
            parts = line.split()
            self.assertEqual(parts[1], 'test_keyspace',
                             'invalid keyspace in line: %s' % line)
            if parts[3] == 'master':
                num_master += 1
            elif parts[3] == 'replica':
                num_replica += 1
            elif parts[3] == 'rdonly':
                num_rdonly += 1
            else:
                self.fail('invalid tablet type in line: %s' % line)
            if parts[2] == '-80':
                num_dash_80 += 1
            elif parts[2] == '80-':
                num_80_dash += 1
            else:
                self.fail('invalid shard name in line: %s' % line)
        self.assertEqual(num_master, 2)
        self.assertEqual(num_replica, 2)
        self.assertEqual(num_rdonly, 2)
        self.assertEqual(num_dash_80, 3)
        self.assertEqual(num_80_dash, 3)

        # and we're done, clean-up process
        sp.stdin.write('\n')
        sp.wait()
Beispiel #58
0
  for proc in proc_list:
    if proc.returncode:
      if options.verbose >= 1 and proc.returncode not in (-9,):
        sys.stderr.write("proc failed: %s %s\n" % (proc.returncode, proc.args))
      if raise_on_error:
        raise CalledProcessError(proc.returncode, ' '.join(proc.args))

def run_procs(cmds, raise_on_error=True):
  procs = []
  for cmd in cmds:
    procs.append(run_bg(cmd))
  wait_procs(procs, raise_on_error=raise_on_error)

# background zk process
# (note the zkocc addresses will only work with an extra zkocc process)
zk_port_base = environment.reserve_ports(3)
def zk_setup(add_bad_host=False):
  global zk_port_base
  zk_ports = ":".join([str(zk_port_base), str(zk_port_base+1), str(zk_port_base+2)])
  run('%s -log_dir %s -zk.cfg 1@%s:%s init' % (environment.binary_argstr('zkctl'), environment.vtlogroot, hostname, zk_ports))
  config = environment.tmproot+'/test-zk-client-conf.json'
  with open(config, 'w') as f:
    ca_server = 'localhost:%u' % (zk_port_base+2)
    if add_bad_host:
      ca_server += ',does.not.exists:1234'
    zk_cell_mapping = {'test_nj': 'localhost:%u'%(zk_port_base+2),
                       'test_ny': 'localhost:%u'%(zk_port_base+2),
                       'test_ca': ca_server,
                       'global': 'localhost:%u'%(zk_port_base+2),
                       'test_nj:_zkocc': 'localhost:%u,localhost:%u,localhost:%u'%(environment.zkocc_port_base,environment.zkocc_port_base+1,environment.zkocc_port_base+2),
                       'test_ny:_zkocc': 'localhost:%u'%(environment.zkocc_port_base),
Beispiel #59
0
    def _test_reparent_graceful(self, shard_id):
        # create the database so vttablets start, as they are serving
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62044.create_db('vt_test_keyspace')
        tablet_41983.create_db('vt_test_keyspace')
        tablet_31981.create_db('vt_test_keyspace')

        # Start up a master mysql and vttablet
        tablet_62344.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True)
        shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
        self.assertEqual(
            shard['cells'], ['test_nj'],
            'wrong list of cell in Shard: %s' % str(shard['cells']))

        # Create a few slaves for testing reparenting.
        tablet_62044.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_41983.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        tablet_31981.init_tablet('replica',
                                 'test_keyspace',
                                 shard_id,
                                 start=True,
                                 wait_for_start=False)
        for t in [tablet_62044, tablet_41983, tablet_31981]:
            t.wait_for_vttablet_state('NOT_SERVING')
        shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/' + shard_id])
        self.assertEqual(
            shard['cells'], ['test_nj', 'test_ny'],
            'wrong list of cell in Shard: %s' % str(shard['cells']))

        # Force the slaves to reparent assuming that all the datasets are
        # identical.
        utils.run_vtctl([
            'InitShardMaster', '-force', 'test_keyspace/' + shard_id,
            tablet_62344.tablet_alias
        ])
        utils.validate_topology(ping_tablets=True)
        tablet_62344.mquery('vt_test_keyspace', self._create_vt_insert_test)

        self._check_master_tablet(tablet_62344)

        utils.validate_topology()

        # Run this to make sure it succeeds.
        stdout, _ = utils.run_vtctl(
            ['ShardReplicationPositions', 'test_keyspace/' + shard_id],
            trap_output=True)
        lines = stdout.splitlines()
        self.assertEqual(len(lines), 4)  # one master, three slaves
        self.assertIn('master', lines[0])  # master first

        # Perform a graceful reparent operation.
        utils.run_vtctl([
            'PlannedReparentShard', '-keyspace_shard', 'test_keyspace/' +
            shard_id, '-new_master', tablet_62044.tablet_alias
        ],
                        auto_log=True)
        utils.validate_topology()

        self._check_master_tablet(tablet_62044)

        # insert data into the new master, check the connected slaves work
        self._populate_vt_insert_test(tablet_62044, 1)
        self._check_vt_insert_test(tablet_41983, 1)
        self._check_vt_insert_test(tablet_62344, 1)

        tablet.kill_tablets(
            [tablet_62344, tablet_62044, tablet_41983, tablet_31981])

        # Test address correction.
        new_port = environment.reserve_ports(1)
        tablet_62044.start_vttablet(port=new_port)

        # Wait until the new address registers.
        timeout = 30.0
        while True:
            try:
                self._check_master_tablet(tablet_62044, port=new_port)
                break
            except protocols_flavor().client_error_exception_type():
                timeout = utils.wait_step('waiting for new port to register',
                                          timeout,
                                          sleep_time=0.1)

        tablet_62044.kill_vttablet()