Exemple #1
0
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
                    **kwargs):
  args = environment.binary_args('vtctl') + [
      '-log_dir', environment.vtlogroot,
      '-enable_queries',
  ]
  args.extend(environment.topo_server().flags())
  args.extend(['-tablet_manager_protocol',
               protocols_flavor().tablet_manager_protocol()])
  args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
  args.extend(['-throttler_client_protocol',
               protocols_flavor().throttler_client_protocol()])
  args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])
  # TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is
  #                   integrated in the vtctl MigrateServed* commands.
  args.extend(['--wait_for_drain_sleep_rdonly', '0s'])
  args.extend(['--wait_for_drain_sleep_replica', '0s'])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  if isinstance(clargs, str):
    cmd = ' '.join(args) + ' ' + clargs
  else:
    cmd = args + clargs

  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Exemple #2
0
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
                    **kwargs):
  args = environment.binary_args('vtctl') + [
      '-log_dir', environment.vtlogroot,
      '-enable_queries',
  ]
  args.extend(environment.topo_server().flags())
  args.extend(['-tablet_manager_protocol',
               protocols_flavor().tablet_manager_protocol()])
  args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
  args.extend(['-throttler_client_protocol',
               protocols_flavor().throttler_client_protocol()])
  args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  if isinstance(clargs, str):
    cmd = ' '.join(args) + ' ' + clargs
  else:
    cmd = args + clargs

  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Exemple #3
0
  def rpc_endpoint(self, python=False):
    """RPC endpoint to vtctld.

    The RPC endpoint may differ from the webinterface URL e.g. because gRPC
    requires a dedicated port.

    Args:
      python: boolean, True iff this is for access with Python (as opposed to
              Go).

    Returns:
      protocol - string e.g. 'grpc'
      endpoint - string e.g. 'localhost:15001'
    """
    if python:
      protocol = protocols_flavor().vtctl_python_client_protocol()
    else:
      protocol = protocols_flavor().vtctl_client_protocol()
    rpc_port = self.port
    if protocol == 'grpc':
      # import the grpc vtctl client implementation, change the port
      if python:
        from vtctl import grpc_vtctl_client  # pylint: disable=g-import-not-at-top,unused-variable
      rpc_port = self.grpc_port
    return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))
Exemple #4
0
def _get_vtworker_cmd(clargs, auto_log=False):
  """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
  port = environment.reserve_ports(1)
  rpc_port = port
  args = environment.binary_args('vtworker') + [
          '-log_dir', environment.vtlogroot,
          '-min_healthy_rdonly_endpoints', '1',
          '-port', str(port),
          '-resolve_ttl', '2s',
          '-executefetch_retry_time', '1s',
          ]
  args.extend(environment.topo_server().flags())
  args.extend(['-tablet_manager_protocol',
               protocols_flavor().tablet_manager_protocol()])
  if protocols_flavor().service_map():
    args.extend(['-service_map',
                 ",".join(protocols_flavor().service_map())])
  if protocols_flavor().vtworker_client_protocol() == 'grpc':
    rpc_port = environment.reserve_ports(1)
    args.extend(['-grpc_port', str(rpc_port)])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  cmd = args + clargs
  return cmd, port, rpc_port
Exemple #5
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.validate_topology()
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    _, stderr = utils.run_vtctl(
        ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
        expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    tablet_62344.kill_vttablet()
Exemple #6
0
def run_vtctl_vtctl(clargs, log_level='', auto_log=False, expect_fail=False,
                    **kwargs):
  args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
  args.extend(environment.topo_server().flags())
  args.extend(protocols_flavor().tablet_manager_protocol_flags())
  args.extend(protocols_flavor().tabletconn_protocol_flags())

  if auto_log:
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'

  if log_level:
    args.append('--stderrthreshold=%s' % log_level)

  if isinstance(clargs, str):
    cmd = " ".join(args) + ' ' + clargs
  else:
    cmd = args + clargs

  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Exemple #7
0
  def start(self):
    args = environment.binary_args('vtctld') + [
            '-debug',
            '-templates', environment.vttop + '/go/cmd/vtctld/templates',
            '-log_dir', environment.vtlogroot,
            '-port', str(self.port),
            ] + \
            environment.topo_server().flags() + \
            protocols_flavor().tablet_manager_protocol_flags()
    stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w")
    self.proc = run_bg(args, stderr=stderr_fd)

    # wait for the process to listen to RPC
    timeout = 30
    while True:
      v = get_vars(self.port)
      if v:
        break
      timeout = wait_step('waiting for vtctld to start', timeout,
                          sleep_time=0.2)

    # save the running instance so vtctl commands can be remote executed now
    global vtctld, vtctld_connection
    if not vtctld:
      vtctld = self
      vtctld_connection = vtctl_client.connect(
          protocols_flavor().vtctl_client_protocol(), 'localhost:%u' % self.port, 30)

    return self.proc
def setUpModule():
  global vtgateclienttest_process
  global vtgateclienttest_port
  global vtgateclienttest_grpc_port

  try:
    environment.topo_server().setup()

    vtgateclienttest_port = environment.reserve_ports(1)
    args = environment.binary_args('vtgateclienttest') + [
        '-log_dir', environment.vtlogroot,
        '-port', str(vtgateclienttest_port),
        ]

    if protocols_flavor().vtgate_python_protocol() == 'grpc':
      vtgateclienttest_grpc_port = environment.reserve_ports(1)
      args.extend(['-grpc_port', str(vtgateclienttest_grpc_port)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])

    vtgateclienttest_process = utils.run_bg(args)
    utils.wait_for_vars('vtgateclienttest', vtgateclienttest_port)
  except:
    tearDownModule()
    raise
Exemple #9
0
  def test_actions_and_timeouts(self):
    # Start up a master mysql and vttablet
    utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

    tablet_62344.init_tablet('master', 'test_keyspace', '0')
    utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
    utils.validate_topology()
    srvShard = utils.run_vtctl_json(['GetSrvShard', 'test_nj',
                                     'test_keyspace/0'])
    self.assertEqual(srvShard['MasterCell'], 'test_nj')
    tablet_62344.create_db('vt_test_keyspace')
    tablet_62344.start_vttablet()

    utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

    # schedule long action in the background, sleep a little bit to make sure
    # it started to run
    args = (environment.binary_args('vtctl') +
            environment.topo_server().flags() +
            ['-tablet_manager_protocol',
             protocols_flavor().tablet_manager_protocol(),
             '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
             '-log_dir', environment.vtlogroot,
             'Sleep', tablet_62344.tablet_alias, '10s'])
    bg = utils.run_bg(args)
    time.sleep(3)

    # try a frontend RefreshState that should timeout as the tablet is busy
    # running the other one
    stdout, stderr = utils.run_vtctl(['-wait-time', '3s',
                                      'RefreshState', tablet_62344.tablet_alias],
                                     expect_fail=True)
    self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

    # wait for the background vtctl
    bg.wait()

    if environment.topo_server().flavor() == 'zookeeper':
      # extra small test: we ran for a while, get the states we were in,
      # make sure they're accounted for properly
      # first the query engine States
      v = utils.get_vars(tablet_62344.port)
      logging.debug("vars: %s" % str(v))

      # then the Zookeeper connections
      if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
        self.fail('invalid zk test_nj state: %s' %
                  v['ZkMetaConn']['test_nj']['Current'])
      if v['ZkMetaConn']['global']['Current'] != 'Connected':
        self.fail('invalid zk global state: %s' %
                  v['ZkMetaConn']['global']['Current'])
      if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
        self.fail('not enough time in Connected state: %u',
                  v['ZkMetaConn']['test_nj']['DurationConnected'])
      if v['TabletType'] != 'master':
        self.fail('TabletType not exported correctly')

    tablet_62344.kill_vttablet()
Exemple #10
0
    def start(
        self,
        cell="test_nj",
        retry_delay=1,
        retry_count=2,
        topo_impl=None,
        cache_ttl="1s",
        timeout_total="2s",
        timeout_per_conn="1s",
        extra_args=None,
        tablets=None,
    ):
        """Start vtgate. Saves it into the global vtgate variable if not set yet."""

        args = environment.binary_args("vtgate") + [
            "-port",
            str(self.port),
            "-cell",
            cell,
            "-retry-delay",
            "%ss" % (str(retry_delay)),
            "-retry-count",
            str(retry_count),
            "-log_dir",
            environment.vtlogroot,
            "-srv_topo_cache_ttl",
            cache_ttl,
            "-conn-timeout-total",
            timeout_total,
            "-conn-timeout-per-conn",
            timeout_per_conn,
            "-bsonrpc_timeout",
            "5s",
            "-tablet_protocol",
            protocols_flavor().tabletconn_protocol(),
            "-gateway_implementation",
            vtgate_gateway_flavor().flavor(),
            "-tablet_types_to_wait",
            "MASTER,REPLICA",
        ]
        args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
        if protocols_flavor().vtgate_protocol() == "grpc":
            args.extend(["-grpc_port", str(self.grpc_port)])
        if protocols_flavor().service_map():
            args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
        if topo_impl:
            args.extend(["-topo_implementation", topo_impl])
        else:
            args.extend(environment.topo_server().flags())
        if extra_args:
            args.extend(extra_args)

        self.proc = run_bg(args)
        wait_for_vars("vtgate", self.port)

        global vtgate
        if not vtgate:
            vtgate = self
Exemple #11
0
    def start(
        self,
        cell="test_nj",
        retry_delay=1,
        retry_count=2,
        topo_impl=None,
        cache_ttl="1s",
        timeout_total="4s",
        timeout_per_conn="2s",
        extra_args=None,
    ):
        """Starts the process for this vtgate instance.

    If no other instance has been started, saves it into the global
    vtgate variable.
    """
        args = environment.binary_args("vtgate") + [
            "-port",
            str(self.port),
            "-cell",
            cell,
            "-retry-delay",
            "%ss" % (str(retry_delay)),
            "-retry-count",
            str(retry_count),
            "-log_dir",
            environment.vtlogroot,
            "-srv_topo_cache_ttl",
            cache_ttl,
            "-conn-timeout-total",
            timeout_total,
            "-conn-timeout-per-conn",
            timeout_per_conn,
            "-bsonrpc_timeout",
            "5s",
            "-tablet_protocol",
            protocols_flavor().tabletconn_protocol(),
        ]
        if protocols_flavor().vtgate_protocol() == "grpc":
            args.extend(["-grpc_port", str(self.grpc_port)])
        if protocols_flavor().service_map():
            args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
        if topo_impl:
            args.extend(["-topo_implementation", topo_impl])
        else:
            args.extend(environment.topo_server().flags())
        if extra_args:
            args.extend(extra_args)

        self.proc = run_bg(args)
        if self.secure_port:
            wait_for_vars("vtgate", self.port, "SecureConnections")
        else:
            wait_for_vars("vtgate", self.port)

        global vtgate
        if not vtgate:
            vtgate = self
Exemple #12
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(["CreateKeyspace", "test_keyspace"])

        tablet_62344.init_tablet("master", "test_keyspace", "0")
        utils.run_vtctl(["RebuildShardGraph", "test_keyspace/0"])
        utils.validate_topology()
        self._check_srv_shard()
        tablet_62344.create_db("vt_test_keyspace")
        tablet_62344.start_vttablet()

        utils.run_vtctl(["Ping", tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (
            environment.binary_args("vtctl")
            + environment.topo_server().flags()
            + [
                "-tablet_manager_protocol",
                protocols_flavor().tablet_manager_protocol(),
                "-tablet_protocol",
                protocols_flavor().tabletconn_protocol(),
                "-log_dir",
                environment.vtlogroot,
                "Sleep",
                tablet_62344.tablet_alias,
                "10s",
            ]
        )
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        _, stderr = utils.run_vtctl(["-wait-time", "3s", "RefreshState", tablet_62344.tablet_alias], expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        if environment.topo_server().flavor() == "zookeeper":
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug("vars: %s", v)

            # then the Zookeeper connections
            if v["ZkCachedConn"]["test_nj"] != "Connected":
                self.fail("invalid zk test_nj state: %s" % v["ZkCachedConn"]["test_nj"])
            if v["ZkCachedConn"]["global"] != "Connected":
                self.fail("invalid zk global state: %s" % v["ZkCachedConn"]["global"])
            if v["TabletType"] != "master":
                self.fail("TabletType not exported correctly")

        tablet_62344.kill_vttablet()
Exemple #13
0
    def start(self,
              cell='test_nj',
              retry_delay=1,
              retry_count=2,
              topo_impl=None,
              cache_ttl='1s',
              timeout_total='2s',
              timeout_per_conn='1s',
              extra_args=None,
              tablets=None):
        """Start vtgate. Saves it into the global vtgate variable if not set yet."""

        args = environment.binary_args('vtgate') + [
            '-port',
            str(self.port),
            '-cell',
            cell,
            '-retry-delay',
            '%ss' % (str(retry_delay)),
            '-retry-count',
            str(retry_count),
            '-log_dir',
            environment.vtlogroot,
            '-srv_topo_cache_ttl',
            cache_ttl,
            '-conn-timeout-total',
            timeout_total,
            '-conn-timeout-per-conn',
            timeout_per_conn,
            '-bsonrpc_timeout',
            '5s',
            '-tablet_protocol',
            protocols_flavor().tabletconn_protocol(),
            '-gateway_implementation',
            vtgate_gateway_flavor().flavor(),
            '-tablet_types_to_wait',
            'MASTER,REPLICA',
        ]
        args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
        if protocols_flavor().vtgate_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ','.join(protocols_flavor().service_map())])
        if topo_impl:
            args.extend(['-topo_implementation', topo_impl])
        else:
            args.extend(environment.topo_server().flags())
        if extra_args:
            args.extend(extra_args)

        self.proc = run_bg(args)
        wait_for_vars('vtgate', self.port)

        global vtgate
        if not vtgate:
            vtgate = self
Exemple #14
0
 def rpc_endpoint(self, python=False):
     """Returns the protocol and endpoint to use for RPCs."""
     if python:
         protocol = protocols_flavor().vtgate_python_protocol()
     else:
         protocol = protocols_flavor().vtgate_protocol()
     if protocol == 'grpc':
         return protocol, 'localhost:%d' % self.grpc_port
     return protocol, self.addr()
Exemple #15
0
 def rpc_endpoint(self, python=False):
   """Returns the protocol and endpoint to use for RPCs."""
   if python:
     protocol = protocols_flavor().vtgate_python_protocol()
   else:
     protocol = protocols_flavor().vtgate_protocol()
   if protocol == 'grpc':
     return protocol, 'localhost:%d' % self.grpc_port
   return protocol, self.addr()
Exemple #16
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        tablet_62344.init_tablet('master', 'test_keyspace', '0')
        utils.run_vtctl(['RebuildShardGraph', 'test_keyspace/0'])
        utils.validate_topology()
        self._check_srv_shard()
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62344.start_vttablet()

        utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (environment.binary_args('vtctl') +
                environment.topo_server().flags() + [
                    '-tablet_manager_protocol',
                    protocols_flavor().tablet_manager_protocol(),
                    '-tablet_protocol',
                    protocols_flavor().tabletconn_protocol(), '-log_dir',
                    environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias,
                    '10s'
                ])
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        _, stderr = utils.run_vtctl(
            ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
            expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        if environment.topo_server().flavor() == 'zookeeper':
            # extra small test: we ran for a while, get the states we were in,
            # make sure they're accounted for properly
            # first the query engine States
            v = utils.get_vars(tablet_62344.port)
            logging.debug('vars: %s', v)

            # then the Zookeeper connections
            if v['ZkCachedConn']['test_nj'] != 'Connected':
                self.fail('invalid zk test_nj state: %s' %
                          v['ZkCachedConn']['test_nj'])
            if v['ZkCachedConn']['global'] != 'Connected':
                self.fail('invalid zk global state: %s' %
                          v['ZkCachedConn']['global'])
            if v['TabletType'] != 'master':
                self.fail('TabletType not exported correctly')

        tablet_62344.kill_vttablet()
Exemple #17
0
  def start(self, cell='test_nj', retry_count=2,
            topo_impl=None, cache_ttl='1s',
            extra_args=None, tablets=None,
            tablet_types_to_wait='MASTER,REPLICA',
            l2vtgates=None,
            cells_to_watch=None):
    """Start vtgate. Saves it into the global vtgate variable if not set yet."""

    args = environment.binary_args('vtgate') + [
        '-port', str(self.port),
        '-cell', cell,
        '-retry-count', str(retry_count),
        '-log_dir', environment.vtlogroot,
        '-srv_topo_cache_ttl', cache_ttl,
        '-srv_topo_cache_refresh', cache_ttl,
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
        '-stderrthreshold', get_log_level(),
        '-normalize_queries',
        '-gateway_implementation', vtgate_gateway_flavor().flavor(),
    ]

    if cells_to_watch:
      args.extend(vtgate_gateway_flavor().flags(cell=cells_to_watch, tablets=tablets))
    else:
      args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))

    if l2vtgates:
      args.extend(['-l2vtgate_addrs', ','.join(l2vtgates)])
    if tablet_types_to_wait:
      args.extend(['-tablet_types_to_wait', tablet_types_to_wait])

    if protocols_flavor().vtgate_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
      args.extend(['-grpc_max_message_size',
                   str(environment.grpc_max_message_size)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if topo_impl:
      args.extend(['-topo_implementation', topo_impl])
    else:
      args.extend(environment.topo_server().flags())
    if extra_args:
      args.extend(extra_args)
    if self.mysql_port:
      args.extend(['-mysql_server_port', str(self.mysql_port)])

    self.proc = run_bg(args)
    # We use a longer timeout here, as we may be waiting for the initial
    # state of a few tablets.
    wait_for_vars('vtgate', self.port, timeout=20.0)

    global vtgate
    if not vtgate:
      vtgate = self
Exemple #18
0
  def start(self, cell='test_nj', retry_count=2,
            topo_impl=None, cache_ttl='1s',
            extra_args=None, tablets=None,
            tablet_types_to_wait='MASTER,REPLICA',
            l2vtgates=None):
    """Start vtgate. Saves it into the global vtgate variable if not set yet."""

    args = environment.binary_args('vtgate') + [
        '-port', str(self.port),
        '-cell', cell,
        '-retry-count', str(retry_count),
        '-log_dir', environment.vtlogroot,
        '-srv_topo_cache_ttl', cache_ttl,
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
        '-stderrthreshold', get_log_level(),
        '-normalize_queries',
    ]
    if l2vtgates:
      args.extend([
          '-gateway_implementation', 'l2vtgategateway',
          '-l2vtgategateway_addrs', ','.join(l2vtgates),
      ])
    else:
      args.extend([
          '-gateway_implementation', vtgate_gateway_flavor().flavor(),
      ])
      args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
      if tablet_types_to_wait:
        args.extend(['-tablet_types_to_wait', tablet_types_to_wait])

    if protocols_flavor().vtgate_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
      args.extend(['-grpc_max_message_size',
                   str(environment.grpc_max_message_size)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if topo_impl:
      args.extend(['-topo_implementation', topo_impl])
    else:
      args.extend(environment.topo_server().flags())
    if extra_args:
      args.extend(extra_args)
    if self.mysql_port:
      args.extend(['-mysql_server_port', str(self.mysql_port)])

    self.proc = run_bg(args)
    # We use a longer timeout here, as we may be waiting for the initial
    # state of a few tablets.
    wait_for_vars('vtgate', self.port, timeout=20.0)

    global vtgate
    if not vtgate:
      vtgate = self
Exemple #19
0
    def start(self,
              cell='test_nj',
              retry_count=2,
              topo_impl=None,
              cache_ttl='1s',
              healthcheck_conn_timeout='2s',
              extra_args=None,
              tablets=None,
              tablet_types_to_wait='MASTER,REPLICA',
              tablet_filters=None):
        """Start l2vtgate."""

        args = environment.binary_args('l2vtgate') + [
            '-port',
            str(self.port),
            '-cell',
            cell,
            '-retry-count',
            str(retry_count),
            '-log_dir',
            environment.vtlogroot,
            '-srv_topo_cache_ttl',
            cache_ttl,
            '-healthcheck_conn_timeout',
            healthcheck_conn_timeout,
            '-tablet_protocol',
            protocols_flavor().tabletconn_protocol(),
            '-gateway_implementation',
            vtgate_gateway_flavor().flavor(),
            '-tablet_grpc_combine_begin_execute',
        ]
        args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
        if tablet_types_to_wait:
            args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
        if tablet_filters:
            args.extend(['-tablet_filters', tablet_filters])
        if protocols_flavor().vtgate_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ','.join(protocols_flavor().service_map())])
        if topo_impl:
            args.extend(['-topo_implementation', topo_impl])
        else:
            args.extend(environment.topo_server().flags())
        if extra_args:
            args.extend(extra_args)

        self.proc = run_bg(args)
        wait_for_vars('l2vtgate', self.port)
Exemple #20
0
    def start(self,
              cell='test_nj',
              retry_count=2,
              topo_impl=None,
              cache_ttl='1s',
              extra_args=None,
              tablets=None,
              tablet_types_to_wait='MASTER,REPLICA',
              tablet_filters=None):
        """Start l2vtgate."""

        args = environment.binary_args('l2vtgate') + [
            '-port',
            str(self.port),
            '-cell',
            cell,
            '-retry-count',
            str(retry_count),
            '-log_dir',
            environment.vtlogroot,
            '-srv_topo_cache_ttl',
            cache_ttl,
            '-srv_topo_cache_refresh',
            cache_ttl,
            '-tablet_protocol',
            protocols_flavor().tabletconn_protocol(),
            '-gateway_implementation',
            vtgate_gateway_flavor().flavor(),
        ]
        args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
        if tablet_types_to_wait:
            args.extend(['-tablet_types_to_wait', tablet_types_to_wait])
        if tablet_filters:
            args.extend(['-tablet_filters', tablet_filters])
        if protocols_flavor().vtgate_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ','.join(protocols_flavor().service_map())])
        if topo_impl:
            args.extend(['-topo_implementation', topo_impl])
        else:
            args.extend(environment.topo_server().flags())
        if extra_args:
            args.extend(extra_args)

        self.proc = run_bg(args)
        # We use a longer timeout here, as we may be waiting for the initial
        # state of a few tablets.
        wait_for_vars('l2vtgate', self.port, timeout=20.0)
Exemple #21
0
    def start(self):
        args = (
            environment.binary_args("vtctld")
            + [
                "-debug",
                "-web_dir",
                environment.vttop + "/web/vtctld",
                "--templates",
                environment.vttop + "/go/cmd/vtctld/templates",
                "--log_dir",
                environment.vtlogroot,
                "--port",
                str(self.port),
                "--schema_change_dir",
                self.schema_change_dir,
                "--schema_change_controller",
                "local",
                "--schema_change_check_interval",
                "1",
                "-tablet_manager_protocol",
                protocols_flavor().tablet_manager_protocol(),
                "-vtgate_protocol",
                protocols_flavor().vtgate_protocol(),
            ]
            + environment.topo_server().flags()
        )
        if protocols_flavor().service_map():
            args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
        if protocols_flavor().vtctl_client_protocol() == "grpc":
            args.extend(["-grpc_port", str(self.grpc_port)])
        stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"), "w")
        self.proc = run_bg(args, stderr=stderr_fd)

        # wait for the process to listen to RPC
        timeout = 30
        while True:
            v = get_vars(self.port)
            if v:
                break
            timeout = wait_step("waiting for vtctld to start", timeout, sleep_time=0.2)

        # save the running instance so vtctl commands can be remote executed now
        global vtctld, vtctld_connection
        if not vtctld:
            vtctld = self
            protocol, endpoint = self.rpc_endpoint(python=True)
            vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)

        return self.proc
Exemple #22
0
  def start(self, cell='test_nj', retry_count=2,
            topo_impl=None, cache_ttl='1s',
            healthcheck_conn_timeout='2s',
            extra_args=None, tablets=None,
            tablet_types_to_wait='MASTER,REPLICA',
            l2vtgates=None):
    """Start vtgate. Saves it into the global vtgate variable if not set yet."""

    args = environment.binary_args('vtgate') + [
        '-port', str(self.port),
        '-cell', cell,
        '-retry-count', str(retry_count),
        '-log_dir', environment.vtlogroot,
        '-srv_topo_cache_ttl', cache_ttl,
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
        '-tablet_grpc_combine_begin_execute',
    ]
    if l2vtgates:
      args.extend([
          '-gateway_implementation', 'l2vtgategateway',
          '-l2vtgategateway_addrs', ','.join(l2vtgates),
      ])
    else:
      args.extend([
          '-healthcheck_conn_timeout', healthcheck_conn_timeout,
          '-gateway_implementation', vtgate_gateway_flavor().flavor(),
      ])
      args.extend(vtgate_gateway_flavor().flags(cell=cell, tablets=tablets))
      if tablet_types_to_wait:
        args.extend(['-tablet_types_to_wait', tablet_types_to_wait])

    if protocols_flavor().vtgate_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if topo_impl:
      args.extend(['-topo_implementation', topo_impl])
    else:
      args.extend(environment.topo_server().flags())
    if extra_args:
      args.extend(extra_args)

    self.proc = run_bg(args)
    wait_for_vars('vtgate', self.port)

    global vtgate
    if not vtgate:
      vtgate = self
Exemple #23
0
 def __init__(self, port=None):
   """Creates the Vtgate instance and reserve the ports if necessary."""
   self.port = port or environment.reserve_ports(1)
   if protocols_flavor().vtgate_protocol() == 'grpc':
     self.grpc_port = environment.reserve_ports(1)
   self.secure_port = None
   self.proc = None
Exemple #24
0
 def setUp(self):
   addr = 'localhost:%d' % vtgateclienttest_port
   protocol = protocols_flavor().vtgate_python_protocol()
   self.conn = vtgate_client.connect(protocol, addr, 30.0)
   logging.info(
       'Start: %s, protocol %s.',
       '.'.join(self.id().split('.')[-2:]), protocol)
Exemple #25
0
 def setUp(self):
     super(TestPythonClientBase, self).setUp()
     addr = 'localhost:%d' % vtgateclienttest_port
     protocol = protocols_flavor().vtgate_python_protocol()
     self.conn = vtgate_client.connect(protocol, addr, 30.0)
     logging.info('Start: %s, protocol %s.',
                  '.'.join(self.id().split('.')[-2:]), protocol)
Exemple #26
0
def vtclient2(uid, path, query, bindvars=None, user=None, password=None, driver=None,
              verbose=False, raise_on_error=True):
  if (user is None) != (password is None):
    raise TypeError("you should provide either both or none of user and password")

  # for ZK paths to not have // in the path, that confuses things
  if path.startswith('/'):
    path = path[1:]
  server = "localhost:%u/%s" % (uid, path)

  cmdline = environment.binary_args('vtclient2') + ['-server', server]
  cmdline += environment.topo_server().flags()
  cmdline += protocols_flavor().tabletconn_protocol_flags()
  if user is not None:
    cmdline.extend(['-tablet-bson-username', user,
                    '-tablet-bson-password', password])
  if bindvars:
    cmdline.extend(['-bindvars', bindvars])
  if driver:
    cmdline.extend(['-driver', driver])
  if verbose:
    cmdline.extend(['-alsologtostderr', '-verbose'])
  cmdline.append(query)

  return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
Exemple #27
0
def run_vtworker_client(args, rpc_port):
    """Runs vtworkerclient to execute a command on a remote vtworker.

  Args:
    args: Atr string to send to binary.
    rpc_port: Port number.

  Returns:
    out: stdout of the vtworkerclient invocation
    err: stderr of the vtworkerclient invocation
  """
    out, err = run(
        environment.binary_args("vtworkerclient")
        + [
            "-vtworker_client_protocol",
            protocols_flavor().vtworker_client_protocol(),
            "-server",
            "localhost:%d" % rpc_port,
            "-stderrthreshold",
            get_log_level(),
        ]
        + args,
        trap_output=True,
    )
    return out, err
Exemple #28
0
    def vtclient(self,
                 sql,
                 tablet_type='master',
                 bindvars=None,
                 streaming=False,
                 verbose=False,
                 raise_on_error=False):
        """Uses the vtclient binary to send a query to vtgate."""
        args = environment.binary_args('vtclient') + [
            '-server',
            self.rpc_endpoint(), '-tablet_type', tablet_type,
            '-vtgate_protocol',
            protocols_flavor().vtgate_protocol()
        ]
        if bindvars:
            args.extend(['-bind_variables', json.dumps(bindvars)])
        if streaming:
            args.append('-streaming')
        if verbose:
            args.append('-alsologtostderr')
        args.append(sql)

        out, err = run(args, raise_on_error=raise_on_error, trap_output=True)
        out = out.splitlines()
        return out, err
Exemple #29
0
 def _read_srv_keyspace(self, keyspace_name):
   addr = utils.vtgate.rpc_endpoint()
   protocol = protocols_flavor().vtgate_python_protocol()
   conn = vtgate_client.connect(protocol, addr, 30.0)
   result = conn.get_srv_keyspace(keyspace_name)
   conn.close()
   return result
Exemple #30
0
 def test_streaming_transient_error(self):
     """Test we raise dbexceptions.IntegrityError for StreamExecute calls."""
     # TODO(aaijazi): this test doesn't work for all clients yet.
     if protocols_flavor().vtgate_python_protocol() != 'gorpc':
         return
     self._verify_exception_for_stream_execute('error://transient error',
                                               dbexceptions.TransientError)
Exemple #31
0
def run_vtworker(clargs,
                 log_level='',
                 auto_log=False,
                 expect_fail=False,
                 **kwargs):
    args = environment.binary_args('vtworker') + [
        '-log_dir', environment.vtlogroot, '-port',
        str(environment.reserve_ports(1))
    ]
    args.extend(environment.topo_server().flags())
    args.extend(protocols_flavor().tablet_manager_protocol_flags())

    if auto_log:
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'
    if log_level:
        args.append('--stderrthreshold=%s' % log_level)

    cmd = args + clargs
    if expect_fail:
        return run_fail(cmd, **kwargs)
    return run(cmd, **kwargs)
Exemple #32
0
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
  """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
  """
  port = environment.reserve_ports(1)
  args = environment.binary_args('vtworker') + [
          '-log_dir', environment.vtlogroot,
          '-min_healthy_rdonly_endpoints', '1',
          '-port', str(port),
          '-resolve_ttl', '2s',
          '-executefetch_retry_time', '1s',
          ]
  args.extend(environment.topo_server().flags())
  args.extend(protocols_flavor().tablet_manager_protocol_flags())

  if auto_log:
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'
  if log_level:
    args.append('--stderrthreshold=%s' % log_level)

  cmd = args + clargs
  return cmd, port
Exemple #33
0
def vtclient2(uid,
              path,
              query,
              bindvars=None,
              user=None,
              password=None,
              driver=None,
              verbose=False,
              raise_on_error=True):
    if (user is None) != (password is None):
        raise TypeError(
            "you should provide either both or none of user and password")

    # for ZK paths to not have // in the path, that confuses things
    if path.startswith('/'):
        path = path[1:]
    server = "localhost:%u/%s" % (uid, path)

    cmdline = environment.binary_args('vtclient2') + ['-server', server]
    cmdline += environment.topo_server().flags()
    cmdline += protocols_flavor().tabletconn_protocol_flags()
    if user is not None:
        cmdline.extend(
            ['-tablet-bson-username', user, '-tablet-bson-password', password])
    if bindvars:
        cmdline.extend(['-bindvars', bindvars])
    if driver:
        cmdline.extend(['-driver', driver])
    if verbose:
        cmdline.extend(['-alsologtostderr', '-verbose'])
    cmdline.append(query)

    return run(cmdline, raise_on_error=raise_on_error, trap_output=True)
Exemple #34
0
 def __init__(self, port=None):
     """Creates the Vtgate instance and reserve the ports if necessary."""
     self.port = port or environment.reserve_ports(1)
     if protocols_flavor().vtgate_protocol() == 'grpc':
         self.grpc_port = environment.reserve_ports(1)
     self.secure_port = None
     self.proc = None
Exemple #35
0
def get_connection(timeout=10.0):
    protocol = protocols_flavor().vtgate_python_protocol()
    try:
        return vtgate_client.connect(protocol, utils.vtgate.addr(), timeout)
    except Exception:
        logging.exception('Connection to vtgate (timeout=%s) failed.', timeout)
        raise
 def test_streaming_transient_error(self):
   """Test we raise dbexceptions.IntegrityError for StreamExecute calls."""
   # TODO(aaijazi): this test doesn't work for all clients yet.
   if protocols_flavor().vtgate_python_protocol() != 'gorpc':
     return
   self._verify_exception_for_stream_execute('error://transient error',
     dbexceptions.TransientError)
Exemple #37
0
def get_connection(timeout=10.0):
  protocol = protocols_flavor().vtgate_python_protocol()
  try:
    return vtgate_client.connect(protocol, utils.vtgate.addr(), timeout)
  except Exception:
    logging.exception('Connection to vtgate (timeout=%s) failed.', timeout)
    raise
Exemple #38
0
 def _read_srv_keyspace(self, keyspace_name):
   addr = utils.vtgate.rpc_endpoint()
   protocol = protocols_flavor().vtgate_python_protocol()
   conn = vtgate_client.connect(protocol, addr, 30.0)
   result = conn.get_srv_keyspace(keyspace_name)
   conn.close()
   return result
Exemple #39
0
def _get_vtworker_cmd(clargs, auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Args:
    clargs: Command line arguments passed to vtworker.
    auto_log: If true, set --stderrthreshold according to the test log level.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
    port = environment.reserve_ports(1)
    rpc_port = port
    args = environment.binary_args('vtworker') + [
        '-log_dir',
        environment.vtlogroot,
        '-min_healthy_rdonly_endpoints',
        '1',
        '-port',
        str(port),
        # use a long resolve TTL because of potential race conditions with doing
        # an EmergencyReparent and resolving the master (as EmergencyReparent
        # will delete the old master before updating the shard record with the
        # new master)
        '-resolve_ttl',
        '10s',
        '-executefetch_retry_time',
        '1s',
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol(),
        '-tablet_protocol',
        protocols_flavor().tabletconn_protocol(),
    ]
    args.extend(environment.topo_server().flags())
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ','.join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == 'grpc':
        rpc_port = environment.reserve_ports(1)
        args.extend(['-grpc_port', str(rpc_port)])

    if auto_log:
        args.append('--stderrthreshold=%s' % get_log_level())

    cmd = args + clargs
    return cmd, port, rpc_port
Exemple #40
0
 def setUp(self):
   self.master_tablet = shard_1_master
   if protocols_flavor().vtgate_python_types() == 'proto3':
     self.int_type = 265
     self.string_type = 6165
   else:
     self.int_type = 8L
     self.string_type = 253L
 def test_transient_error(self):
   """Test we raise dbexceptions.TransientError for Execute calls."""
   # TODO(aaijazi): this test doesn't work for all clients yet.
   if protocols_flavor().vtgate_python_protocol() != 'gorpc':
     return
   # Special query that makes vtgateclienttest return a TransientError.
   self._verify_exception_for_execute('error://transient error',
       dbexceptions.TransientError)
Exemple #42
0
 def test_transient_error(self):
     """Test we raise dbexceptions.TransientError for Execute calls."""
     # TODO(aaijazi): this test doesn't work for all clients yet.
     if protocols_flavor().vtgate_python_protocol() != 'gorpc':
         return
     # Special query that makes vtgateclienttest return a TransientError.
     self._verify_exception_for_execute('error://transient error',
                                        dbexceptions.TransientError)
Exemple #43
0
def _get_vtworker_cmd(clargs, auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Args:
    clargs: Command line arguments passed to vtworker.
    auto_log: If true, set --stderrthreshold according to the test log level.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
    rpc_port - int with the port number of the RPC interface
  """
    port = environment.reserve_ports(1)
    rpc_port = port
    args = environment.binary_args("vtworker") + [
        "-log_dir",
        environment.vtlogroot,
        "-min_healthy_rdonly_endpoints",
        "1",
        "-port",
        str(port),
        # use a long resolve TTL because of potential race conditions with doing
        # an EmergencyReparent and resolving the master (as EmergencyReparent
        # will delete the old master before updating the shard record with the
        # new master)
        "-resolve_ttl",
        "10s",
        "-executefetch_retry_time",
        "1s",
        "-tablet_manager_protocol",
        protocols_flavor().tablet_manager_protocol(),
        "-tablet_protocol",
        protocols_flavor().tabletconn_protocol(),
    ]
    args.extend(environment.topo_server().flags())
    if protocols_flavor().service_map():
        args.extend(["-service_map", ",".join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == "grpc":
        rpc_port = environment.reserve_ports(1)
        args.extend(["-grpc_port", str(rpc_port)])

    if auto_log:
        args.append("--stderrthreshold=%s" % get_log_level())

    cmd = args + clargs
    return cmd, port, rpc_port
Exemple #44
0
 def setUp(self):
   self.master_tablet = shard_1_master
   if protocols_flavor().vtgate_python_types() == 'proto3':
     self.int_type = 265
     self.string_type = 6165
   else:
     self.int_type = 8L
     self.string_type = 253L
Exemple #45
0
def run_automation_server(auto_log=False):
  """Starts a background automation_server process.

  Returns:
    rpc_port - int with the port number of the RPC interface
  """
  rpc_port = environment.reserve_ports(1)
  args = environment.binary_args('automation_server') + [
          '-log_dir', environment.vtlogroot,
          '-port', str(rpc_port),
          '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
          '-vtworker_client_protocol', protocols_flavor().vtworker_client_protocol(),
          ]
  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  return run_bg(args), rpc_port
Exemple #46
0
  def start(self, cell='test_nj', retry_delay=1, retry_count=2,
            topo_impl=None, cache_ttl='1s',
            auth=False, timeout_total='4s', timeout_per_conn='2s',
            extra_args=None):
    """Starts the process for this vtgate instance.

    If no other instance has been started, saves it into the global
    vtgate variable.
    """
    args = environment.binary_args('vtgate') + [
        '-port', str(self.port),
        '-cell', cell,
        '-retry-delay', '%ss' % (str(retry_delay)),
        '-retry-count', str(retry_count),
        '-log_dir', environment.vtlogroot,
        '-srv_topo_cache_ttl', cache_ttl,
        '-conn-timeout-total', timeout_total,
        '-conn-timeout-per-conn', timeout_per_conn,
        '-bsonrpc_timeout', '5s',
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
    ]
    if protocols_flavor().vtgate_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if topo_impl:
      args.extend(['-topo_implementation', topo_impl])
    else:
      args.extend(environment.topo_server().flags())
    if auth:
      args.extend(['-auth-credentials',
                   os.path.join(environment.vttop, 'test', 'test_data',
                                'authcredentials_test.json')])
    if extra_args:
      args.extend(extra_args)

    self.proc = run_bg(args)
    if self.secure_port:
      wait_for_vars('vtgate', self.port, 'SecureConnections')
    else:
      wait_for_vars('vtgate', self.port)

    global vtgate
    if not vtgate:
      vtgate = self
Exemple #47
0
    def test_actions_and_timeouts(self):
        # Start up a master mysql and vttablet
        utils.run_vtctl(['CreateKeyspace', 'test_keyspace'])

        tablet_62344.init_tablet('master', 'test_keyspace', '0')
        tablet_62344.create_db('vt_test_keyspace')
        tablet_62344.start_vttablet()
        # validate topology after starting tablet so that tablet has a chance
        # to update shard master_alias
        timeout = 10
        while True:
            shard = utils.run_vtctl_json(['GetShard', 'test_keyspace/0'])
            if shard['master_alias']['uid'] == 62344:
                break
            wait_step('master_alias has been set', timeout)

        utils.validate_topology()

        utils.run_vtctl(['Ping', tablet_62344.tablet_alias])

        # schedule long action in the background, sleep a little bit to make sure
        # it started to run
        args = (environment.binary_args('vtctl') +
                environment.topo_server().flags() + [
                    '-tablet_manager_protocol',
                    protocols_flavor().tablet_manager_protocol(),
                    '-tablet_protocol',
                    protocols_flavor().tabletconn_protocol(), '-log_dir',
                    environment.vtlogroot, 'Sleep', tablet_62344.tablet_alias,
                    '10s'
                ])
        bg = utils.run_bg(args)
        time.sleep(3)

        # try a frontend RefreshState that should timeout as the tablet is busy
        # running the other one
        _, stderr = utils.run_vtctl(
            ['-wait-time', '3s', 'RefreshState', tablet_62344.tablet_alias],
            expect_fail=True)
        self.assertIn(protocols_flavor().rpc_timeout_message(), stderr)

        # wait for the background vtctl
        bg.wait()

        tablet_62344.kill_vttablet()
Exemple #48
0
    def start(self, enable_schema_change_dir=False):
        args = environment.binary_args('vtctld') + [
            '-enable_queries',
            '-cell',
            'test_nj',
            '-web_dir',
            environment.vttop + '/web/vtctld',
            '--log_dir',
            environment.vtlogroot,
            '--port',
            str(self.port),
            '-tablet_manager_protocol',
            protocols_flavor().tablet_manager_protocol(),
            '-tablet_protocol',
            protocols_flavor().tabletconn_protocol(),
            '-throttler_client_protocol',
            protocols_flavor().throttler_client_protocol(),
            '-vtgate_protocol',
            protocols_flavor().vtgate_protocol(),
        ] + environment.topo_server().flags()
        if enable_schema_change_dir:
            args += [
                '--schema_change_dir',
                self.schema_change_dir,
                '--schema_change_controller',
                'local',
                '--schema_change_check_interval',
                '1',
            ]
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ','.join(protocols_flavor().service_map())])
        if protocols_flavor().vtctl_client_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'),
                         'w')
        stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
                         'w')
        self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)

        # wait for the process to listen to RPC
        timeout = 30
        while True:
            v = get_vars(self.port)
            if v:
                break
            if self.proc.poll() is not None:
                raise TestError('vtctld died while starting')
            timeout = wait_step('waiting for vtctld to start',
                                timeout,
                                sleep_time=0.2)

        # save the running instance so vtctl commands can be remote executed now
        global vtctld, vtctld_connection
        if not vtctld:
            vtctld = self
            protocol, endpoint = self.rpc_endpoint(python=True)
            vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)

        return self.proc
Exemple #49
0
  def vtctl_client(self, args):
    if options.verbose == 2:
      log_level='INFO'
    elif options.verbose == 1:
      log_level='WARNING'
    else:
      log_level='ERROR'

    port = self.port
    if protocols_flavor().vtctl_client_protocol() == 'grpc':
      port = self.grpc_port
    out, err = run(environment.binary_args('vtctlclient') +
                   ['-vtctl_client_protocol',
                    protocols_flavor().vtctl_client_protocol(),
                    '-server', 'localhost:%u' % port,
                    '-stderrthreshold', log_level] + args,
                   trap_output=True)
    return out
Exemple #50
0
def run_automation_server(auto_log=False):
  """Starts a background automation_server process.

  Returns:
    rpc_port - int with the port number of the RPC interface
  """
  rpc_port = environment.reserve_ports(1)
  args = environment.binary_args('automation_server') + [
      '-log_dir', environment.vtlogroot,
      '-port', str(rpc_port),
      '-vtctl_client_protocol', protocols_flavor().vtctl_client_protocol(),
      '-vtworker_client_protocol',
      protocols_flavor().vtworker_client_protocol(),
  ]
  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  return run_bg(args), rpc_port
Exemple #51
0
    def vtctl_client(self, args):
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'

        port = self.port
        if protocols_flavor().vtctl_client_protocol() == 'grpc':
            port = self.grpc_port
        out, err = run(environment.binary_args('vtctlclient') + [
            '-vtctl_client_protocol',
            protocols_flavor().vtctl_client_protocol(), '-server',
            'localhost:%u' % port, '-stderrthreshold', log_level
        ] + args,
                       trap_output=True)
        return out
Exemple #52
0
def _get_vtworker_cmd(clargs, log_level='', auto_log=False):
    """Assembles the command that is needed to run a vtworker.

  Returns:
    cmd - list of cmd arguments, can be passed to any `run`-like functions
    port - int with the port number that the vtworker is running with
  """
    port = environment.reserve_ports(1)
    rpc_port = environment.reserve_ports(1)
    args = environment.binary_args('vtworker') + [
        '-log_dir',
        environment.vtlogroot,
        '-min_healthy_rdonly_endpoints',
        '1',
        '-port',
        str(port),
        '-resolve_ttl',
        '2s',
        '-executefetch_retry_time',
        '1s',
    ]
    args.extend(environment.topo_server().flags())
    args.extend([
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol()
    ])
    if protocols_flavor().service_map():
        args.extend(
            ['-service_map', ",".join(protocols_flavor().service_map())])
    if protocols_flavor().vtworker_client_protocol() == 'grpc':
        args.extend(['-grpc_port', str(rpc_port)])

    if auto_log:
        if options.verbose == 2:
            log_level = 'INFO'
        elif options.verbose == 1:
            log_level = 'WARNING'
        else:
            log_level = 'ERROR'
    if log_level:
        args.append('--stderrthreshold=%s' % log_level)

    cmd = args + clargs
    return cmd, port, rpc_port
Exemple #53
0
 def update_stream_python_endpoint(self):
     protocol = protocols_flavor().binlog_player_python_protocol()
     port = self.port
     if protocol == "gorpc":
         from vtdb import gorpc_update_stream
     elif protocol == "grpc":
         # import the grpc update stream client implementation, change the port
         from vtdb import grpc_update_stream
         port = self.grpc_port
     return (protocol, 'localhost:%u' % port)
Exemple #54
0
  def start(self, enable_schema_change_dir=False):
    # Note the vtctld2 web dir is set to 'dist', which is populated
    # when a toplevel 'make build_web' is run. This is meant to test
    # the development version of the UI. The real checked-in app is in
    # app/.
    args = environment.binary_args('vtctld') + [
        '-enable_queries',
        '-cell', 'test_nj',
        '-web_dir', environment.vttop + '/web/vtctld',
        '-web_dir2', environment.vttop + '/web/vtctld2/dist',
        '--log_dir', environment.vtlogroot,
        '--port', str(self.port),
        '-tablet_manager_protocol',
        protocols_flavor().tablet_manager_protocol(),
        '-tablet_protocol', protocols_flavor().tabletconn_protocol(),
        '-throttler_client_protocol',
        protocols_flavor().throttler_client_protocol(),
        '-vtgate_protocol', protocols_flavor().vtgate_protocol(),
        '-workflow_manager_init',
        '-workflow_manager_use_election',
    ] + environment.topo_server().flags()
    # TODO(b/26388813): Remove the next two lines once vtctl WaitForDrain is
    #                   integrated in the vtctl MigrateServed* commands.
    args.extend(['--wait_for_drain_sleep_rdonly', '0s'])
    args.extend(['--wait_for_drain_sleep_replica', '0s'])
    if enable_schema_change_dir:
      args += [
          '--schema_change_dir', self.schema_change_dir,
          '--schema_change_controller', 'local',
          '--schema_change_check_interval', '1',
      ]
    if protocols_flavor().service_map():
      args.extend(['-service_map', ','.join(protocols_flavor().service_map())])
    if protocols_flavor().vtctl_client_protocol() == 'grpc':
      args.extend(['-grpc_port', str(self.grpc_port)])
    stdout_fd = open(os.path.join(environment.tmproot, 'vtctld.stdout'), 'w')
    stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'), 'w')
    self.proc = run_bg(args, stdout=stdout_fd, stderr=stderr_fd)

    # wait for the process to listen to RPC
    timeout = 30
    while True:
      v = get_vars(self.port)
      if v:
        break
      if self.proc.poll() is not None:
        raise TestError('vtctld died while starting')
      timeout = wait_step('waiting for vtctld to start', timeout,
                          sleep_time=0.2)

    # save the running instance so vtctl commands can be remote executed now
    global vtctld, vtctld_connection
    if not vtctld:
      vtctld = self
      protocol, endpoint = self.rpc_endpoint(python=True)
      vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)

    return self.proc
Exemple #55
0
    def start(self):
        args = environment.binary_args('vtctld') + [
                '-debug',
                '-templates', environment.vttop + '/go/cmd/vtctld/templates',
                '-log_dir', environment.vtlogroot,
                '-port', str(self.port),
                ] + \
                environment.topo_server().flags() + \
                protocols_flavor().tablet_manager_protocol_flags()
        if protocols_flavor().vtctl_client_protocol() == "grpc":
            args += [
                '-grpc_port',
                str(self.grpc_port), '-service_map', 'grpc-vtctl'
            ]
        stderr_fd = open(os.path.join(environment.tmproot, "vtctld.stderr"),
                         "w")
        self.proc = run_bg(args, stderr=stderr_fd)

        # wait for the process to listen to RPC
        timeout = 30
        while True:
            v = get_vars(self.port)
            if v:
                break
            timeout = wait_step('waiting for vtctld to start',
                                timeout,
                                sleep_time=0.2)

        # save the running instance so vtctl commands can be remote executed now
        protocol = protocols_flavor().vtctl_client_protocol()
        if protocol == "grpc":
            # import the grpc vtctl client implementation, disabled for now:
            # from vtctl import grpc_vtctl_client
            # temporary protocol override until python client support works
            protocol = "gorpc"
        global vtctld, vtctld_connection
        if not vtctld:
            vtctld = self
            vtctld_connection = vtctl_client.connect(
                protocol, 'localhost:%u' % self.port, 30)

        return self.proc
Exemple #56
0
def run_vtctl_vtctl(clargs, auto_log=False, expect_fail=False,
                    **kwargs):
  args = environment.binary_args('vtctl') + ['-log_dir', environment.vtlogroot]
  args.extend(environment.topo_server().flags())
  args.extend(['-tablet_manager_protocol',
               protocols_flavor().tablet_manager_protocol()])
  args.extend(['-tablet_protocol', protocols_flavor().tabletconn_protocol()])
  args.extend(['-vtgate_protocol', protocols_flavor().vtgate_protocol()])

  if auto_log:
    args.append('--stderrthreshold=%s' % get_log_level())

  if isinstance(clargs, str):
    cmd = ' '.join(args) + ' ' + clargs
  else:
    cmd = args + clargs

  if expect_fail:
    return run_fail(cmd, **kwargs)
  return run(cmd, **kwargs)
Exemple #57
0
    def start(self):
        args = environment.binary_args('vtctld') + [
                '-debug',
                '--templates', environment.vttop + '/go/cmd/vtctld/templates',
                '--log_dir', environment.vtlogroot,
                '--port', str(self.port),
                '--schema_change_dir', self.schema_change_dir,
                '--schema_change_controller', 'local',
                '--schema_change_check_interval', '1',
                '-tablet_manager_protocol',
                protocols_flavor().tablet_manager_protocol(),
                ] + \
                environment.topo_server().flags() + \
                protocols_flavor().vtgate_protocol_flags()
        if protocols_flavor().service_map():
            args.extend(
                ['-service_map', ",".join(protocols_flavor().service_map())])
        if protocols_flavor().vtctl_client_protocol() == 'grpc':
            args.extend(['-grpc_port', str(self.grpc_port)])
        stderr_fd = open(os.path.join(environment.tmproot, 'vtctld.stderr'),
                         'w')
        self.proc = run_bg(args, stderr=stderr_fd)

        # wait for the process to listen to RPC
        timeout = 30
        while True:
            v = get_vars(self.port)
            if v:
                break
            timeout = wait_step('waiting for vtctld to start',
                                timeout,
                                sleep_time=0.2)

        # save the running instance so vtctl commands can be remote executed now
        global vtctld, vtctld_connection
        if not vtctld:
            vtctld = self
            protocol, endpoint = self.rpc_endpoint(python=True)
            vtctld_connection = vtctl_client.connect(protocol, endpoint, 30)

        return self.proc
Exemple #58
0
    def rpc_endpoint(self, python=False):
        """RPC endpoint to vtctld.

    The RPC endpoint may differ from the webinterface URL e.g. because gRPC
    requires a dedicated port.

    Returns:
      protocol - string e.g. 'grpc'
      endpoint - string e.g. 'localhost:15001'
    """
        if python:
            protocol = protocols_flavor().vtctl_python_client_protocol()
        else:
            protocol = protocols_flavor().vtctl_client_protocol()
        rpc_port = self.port
        if protocol == 'grpc':
            # import the grpc vtctl client implementation, change the port
            if python:
                from vtctl import grpc_vtctl_client
            rpc_port = self.grpc_port
        return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))
Exemple #59
0
    def rpc_endpoint(self, python=False):
        """RPC endpoint to vtctld.

    The RPC endpoint may differ from the webinterface URL e.g. because gRPC
    requires a dedicated port.

    Args:
      python: boolean, True iff this is for access with Python (as opposed to
              Go).

    Returns:
      protocol - string e.g. 'grpc'
      endpoint - string e.g. 'localhost:15001'
    """
        if python:
            protocol = protocols_flavor().vtctl_python_client_protocol()
        else:
            protocol = protocols_flavor().vtctl_client_protocol()
        rpc_port = self.port
        if protocol == 'grpc':
            rpc_port = self.grpc_port
        return (protocol, '%s:%d' % (socket.getfqdn(), rpc_port))