def fix_served_types(keyspaces, namespace):
"""Ensures the smallest set of non-overlapping shards is serving.
Args:
keyspaces: [string], list of keyspaces to fix.
namespace: string, Kubernetes namespace for vtctld calls.
Returns:
0 for success, 1 for failure
"""
for keyspace in keyspaces:
output, success = vtctl_sandbox.execute_vtctl_command(
['FindAllShardsInKeyspace', keyspace], namespace=namespace)
if not success:
logging.error('Failed to execute FindAllShardsInKeyspace: %s',
output)
return 1
for cmd in get_vtctl_commands(keyspace, json.loads(output)):
logging.info('Executing %s', ' '.join(cmd))
vtctl_sandbox.execute_vtctl_command(cmd, namespace=namespace)
logging.info('Rebuilding keyspace %s', keyspace)
vtctl_sandbox.execute_vtctl_command(['RebuildKeyspaceGraph', keyspace],
namespace=namespace)
return 0
def fix_served_types(keyspaces, namespace):
"""Ensures the smallest set of non-overlapping shards is serving.
Args:
keyspaces: [string], list of keyspaces to fix.
namespace: string, Kubernetes namespace for vtctld calls.
Returns:
0 for success, 1 for failure
"""
for keyspace in keyspaces:
output, success = vtctl_sandbox.execute_vtctl_command(
['FindAllShardsInKeyspace', keyspace], namespace=namespace)
if not success:
logging.error('Failed to execute FindAllShardsInKeyspace: %s', output)
return 1
for cmd in get_vtctl_commands(keyspace, json.loads(output)):
logging.info('Executing %s', ' '.join(cmd))
vtctl_sandbox.execute_vtctl_command(cmd, namespace=namespace)
logging.info('Rebuilding keyspace %s', keyspace)
vtctl_sandbox.execute_vtctl_command(
['RebuildKeyspaceGraph', keyspace], namespace=namespace)
return 0
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option('-n',
'--namespace',
help='Kubernetes namespace',
default='vitess')
parser.add_option('-k',
'--keyspace',
help='Keyspace name',
default='test_keyspace')
parser.add_option('-d',
'--drop_table',
help='An optional table name to drop')
parser.add_option('-s',
'--sql_file',
help='File containing sql schema',
default=os.path.join(
os.environ['VTTOP'],
'examples/kubernetes/create_test_table.sql'))
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
with open(options.sql_file, 'r') as sql_file:
sql = sql_file.read()
if options.drop_table:
vtctl_sandbox.execute_vtctl_command([
'ApplySchema', '-sql',
'drop table if exists %s' % options.drop_table, options.keyspace
],
namespace=options.namespace)
vtctl_sandbox.execute_vtctl_command(
['ApplySchema', '-sql', sql, options.keyspace],
namespace=options.namespace)
def initial_reparent(keyspace, master_cell, num_shards, namespace):
"""Performs the first reparent."""
successfully_reparented = []
master_tablets = {}
while len(master_tablets) < num_shards:
for shard_name in sharding_utils.get_shard_names(num_shards):
shard_name = sandbox_utils.fix_shard_name(shard_name)
tablets = vtctl_sandbox.execute_vtctl_command(
['ListShardTablets', '%s/%s' % (
keyspace, sandbox_utils.fix_shard_name(shard_name))],
namespace=namespace)[0].split('\n')
tablets = [x.split(' ') for x in tablets if x]
potential_masters = [
x[0] for x in tablets if x[3] == 'replica'
and x[0].split('-')[0] == master_cell]
if potential_masters:
master_tablets[shard_name] = potential_masters[0]
while len(successfully_reparented) < num_shards:
for shard_name in sharding_utils.get_shard_names(num_shards):
shard_name = sandbox_utils.fix_shard_name(shard_name)
master_tablet_id = master_tablets[shard_name]
if is_master(master_tablet_id, namespace):
logging.info('Tablet %s is the master of %s/%s.',
master_tablet_id, keyspace, shard_name)
successfully_reparented.append(shard_name)
if shard_name in successfully_reparented:
continue
logging.info('Setting tablet %s as master for %s/%s.',
master_tablet_id, keyspace, shard_name)
vtctl_sandbox.execute_vtctl_command(
['InitShardMaster', '-force', '%s/%s' % (keyspace, shard_name),
master_tablet_id], namespace=namespace, timeout_s=5)
logging.info('Done with initial reparent.')
def initial_reparent(keyspace, master_cell, num_shards, namespace, timeout_s):
"""Performs the first reparent."""
successfully_reparented = []
master_tablets = {}
start_time = time.time()
logging.info('Finding tablets to reparent to.')
while len(master_tablets) < num_shards:
if time.time() - start_time > timeout_s:
logging.error('Timed out waiting to find a replica tablet')
return 1
for shard_name in sharding_utils.get_shard_names(num_shards):
if shard_name in master_tablets:
continue
tablets = vtctl_sandbox.execute_vtctl_command(
['ListShardTablets',
'%s/%s' % (keyspace, shard_name)],
namespace=namespace)[0].split('\n')
tablets = [x.split(' ') for x in tablets if x]
potential_masters = [
x[0] for x in tablets
if x[3] == 'replica' and x[0].split('-')[0] == master_cell
]
if potential_masters:
master_tablets[shard_name] = potential_masters[0]
logging.info('%s selected for shard %s', potential_masters[0],
shard_name)
while time.time() - start_time < timeout_s:
for shard_name in sharding_utils.get_shard_names(num_shards):
master_tablet_id = master_tablets[shard_name]
if is_master(master_tablet_id, namespace):
logging.info('Tablet %s is the master of %s/%s.',
master_tablet_id, keyspace, shard_name)
successfully_reparented.append(shard_name)
if shard_name in successfully_reparented:
continue
logging.info('Setting tablet %s as master for %s/%s.',
master_tablet_id, keyspace, shard_name)
vtctl_sandbox.execute_vtctl_command([
'InitShardMaster', '-force',
'%s/%s' % (keyspace, shard_name), master_tablet_id
],
namespace=namespace,
timeout_s=5)
if len(successfully_reparented) == num_shards:
logging.info('Done with initial reparent.')
return 0
logging.error('Timed out waiting for initial reparent.')
return 1
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option('-n',
'--namespace',
help='Kubernetes namespace',
default='vitess')
parser.add_option('-c', '--cells', help='Comma separated list of cells')
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
logging.info('Waiting for mysql to become healthy.')
start_time = time.time()
good_tablets = []
while time.time() - start_time < 300:
if not good_tablets:
tablets = get_all_tablets(options.cells, options.namespace)
for tablet in [t for t in tablets if t not in good_tablets]:
_, success = vtctl_sandbox.execute_vtctl_command(
['ExecuteFetchAsDba', tablet, 'show databases'],
namespace=options.namespace)
if success:
good_tablets.append(tablet)
logging.info('%d of %d tablets healthy.', len(good_tablets),
len(tablets))
if len(good_tablets) == len(tablets):
logging.info('All tablets healthy in %f seconds.',
time.time() - start_time)
break
else:
logging.warn('Timed out waiting for tablets to be ready.')
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option('-n', '--namespace', help='Kubernetes namespace',
default='vitess')
parser.add_option('-c', '--cells', help='Comma separated list of cells')
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
logging.info('Waiting for mysql to become healthy.')
start_time = time.time()
good_tablets = []
while time.time() - start_time < 300:
if not good_tablets:
tablets = get_all_tablets(options.cells, options.namespace)
for tablet in [t for t in tablets if t not in good_tablets]:
_, success = vtctl_sandbox.execute_vtctl_command(
['ExecuteFetchAsDba', tablet, 'show databases'],
namespace=options.namespace)
if success:
good_tablets.append(tablet)
logging.info('%d of %d tablets healthy.', len(good_tablets), len(tablets))
if len(good_tablets) == len(tablets):
logging.info('All tablets healthy in %f seconds.',
time.time() - start_time)
break
else:
logging.warn('Timed out waiting for tablets to be ready.')
def initial_reparent(keyspace, master_cell, num_shards, namespace, timeout_s):
"""Performs the first reparent."""
successfully_reparented = []
master_tablets = {}
start_time = time.time()
logging.info('Finding tablets to reparent to.')
while len(master_tablets) < num_shards:
if time.time() - start_time > timeout_s:
logging.error('Timed out waiting to find a replica tablet')
return 1
for shard_name in sharding_utils.get_shard_names(num_shards):
if shard_name in master_tablets:
continue
tablets = vtctl_sandbox.execute_vtctl_command(
['ListShardTablets', '%s/%s' % (keyspace, shard_name)],
namespace=namespace)[0].split('\n')
tablets = [x.split(' ') for x in tablets if x]
potential_masters = [
x[0] for x in tablets if x[3] == 'replica'
and x[0].split('-')[0] == master_cell]
if potential_masters:
master_tablets[shard_name] = potential_masters[0]
logging.info(
'%s selected for shard %s', potential_masters[0], shard_name)
while time.time() - start_time < timeout_s:
for shard_name in sharding_utils.get_shard_names(num_shards):
master_tablet_id = master_tablets[shard_name]
if is_master(master_tablet_id, namespace):
logging.info('Tablet %s is the master of %s/%s.',
master_tablet_id, keyspace, shard_name)
successfully_reparented.append(shard_name)
if shard_name in successfully_reparented:
continue
logging.info('Setting tablet %s as master for %s/%s.',
master_tablet_id, keyspace, shard_name)
vtctl_sandbox.execute_vtctl_command(
['InitShardMaster', '-force', '%s/%s' % (keyspace, shard_name),
master_tablet_id], namespace=namespace, timeout_s=5)
if len(successfully_reparented) == num_shards:
logging.info('Done with initial reparent.')
return 0
logging.error('Timed out waiting for initial reparent.')
return 1
def initial_reparent(keyspace, master_cell, num_shards, namespace):
"""Performs the first reparent."""
successfully_reparented = []
master_tablets = {}
while len(master_tablets) < num_shards:
for shard_name in sharding_utils.get_shard_names(num_shards):
shard_name = sandbox_utils.fix_shard_name(shard_name)
tablets = vtctl_sandbox.execute_vtctl_command(
[
'ListShardTablets',
'%s/%s' %
(keyspace, sandbox_utils.fix_shard_name(shard_name))
],
namespace=namespace)[0].split('\n')
tablets = [x.split(' ') for x in tablets if x]
potential_masters = [
x[0] for x in tablets
if x[3] == 'replica' and x[0].split('-')[0] == master_cell
]
if potential_masters:
master_tablets[shard_name] = potential_masters[0]
while len(successfully_reparented) < num_shards:
for shard_name in sharding_utils.get_shard_names(num_shards):
shard_name = sandbox_utils.fix_shard_name(shard_name)
master_tablet_id = master_tablets[shard_name]
if is_master(master_tablet_id, namespace):
logging.info('Tablet %s is the master of %s/%s.',
master_tablet_id, keyspace, shard_name)
successfully_reparented.append(shard_name)
if shard_name in successfully_reparented:
continue
logging.info('Setting tablet %s as master for %s/%s.',
master_tablet_id, keyspace, shard_name)
vtctl_sandbox.execute_vtctl_command([
'InitShardMaster', '-force',
'%s/%s' % (keyspace, shard_name), master_tablet_id
],
namespace=namespace,
timeout_s=5)
logging.info('Done with initial reparent.')
def get_all_tablets(cells, namespace):
"""Returns a list of all tablet names."""
tablets = []
cells = cells.split(',')
for cell in cells:
cell_tablets = vtctl_sandbox.execute_vtctl_command(
['ListAllTablets', cell], namespace=namespace)[0].split('\n')
for t in cell_tablets:
tablets.append(t.split(' ')[0])
tablets = filter(None, tablets)
logging.info('Tablets: %s.', ', '.join(tablets))
return tablets
def get_all_tablets(cells, namespace):
"""Returns a list of all tablet names."""
tablets = []
cells = cells.split(',')
for cell in cells:
cell_tablets = vtctl_sandbox.execute_vtctl_command(
['ListAllTablets', cell], namespace=namespace)[0].split('\n')
for t in cell_tablets:
tablets.append(t.split(' ')[0])
tablets = filter(None, tablets)
logging.info('Tablets: %s.', ', '.join(tablets))
return tablets
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option('-n',
'--namespace',
help='Kubernetes namespace',
default='vitess')
parser.add_option('-c', '--cells', help='Comma separated list of cells')
parser.add_option('-t',
'--tablet_count',
help='Total number of expected tablets',
type=int)
parser.add_option('-w',
'--wait',
help='Max wait time (s)',
type=int,
default=300)
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
logging.info('Waiting for mysql to become healthy.')
start_time = time.time()
good_tablets = []
tablets = []
# Do this in a loop as the output of ListAllTablets may not be parseable
# until all tablets have been started.
while (time.time() - start_time < options.wait
and len(tablets) < options.tablet_count):
tablets = get_all_tablets(options.cells, options.namespace)
logging.info('Expecting %d tablets, found %d tablets',
options.tablet_count, len(tablets))
start_time = time.time()
while time.time() - start_time < options.wait:
for tablet in [t for t in tablets if t not in good_tablets]:
_, success = vtctl_sandbox.execute_vtctl_command(
['ExecuteFetchAsDba', tablet, 'show databases'],
namespace=options.namespace,
timeout_s=1)
if success:
good_tablets.append(tablet)
logging.info('%d of %d tablets healthy.', len(good_tablets),
len(tablets))
if len(good_tablets) == len(tablets):
logging.info('All tablets healthy in %f seconds.',
time.time() - start_time)
break
else:
logging.warn('Timed out waiting for tablets to be ready.')
sys.exit(1)
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option(
'-n', '--namespace', help='Kubernetes namespace', default='vitess')
parser.add_option(
'-k', '--keyspace', help='Keyspace name', default='test_keyspace')
parser.add_option(
'-d', '--drop_table', help='An optional table name to drop')
parser.add_option(
'-s', '--sql_file', help='File containing sql schema',
default=os.path.join(
os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql'))
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
with open(options.sql_file, 'r') as sql_file:
sql = sql_file.read()
if options.drop_table:
vtctl_sandbox.execute_vtctl_command(
['ApplySchema', '-sql', 'drop table if exists %s' % options.drop_table,
options.keyspace], namespace=options.namespace)
vtctl_sandbox.execute_vtctl_command(
['ApplySchema', '-sql', sql, options.keyspace],
namespace=options.namespace)
def main():
parser = optparse.OptionParser(usage='usage: %prog [options] [test_names]')
parser.add_option('-n', '--namespace', help='Kubernetes namespace',
default='vitess')
parser.add_option('-c', '--cells', help='Comma separated list of cells')
parser.add_option('-t', '--tablet_count',
help='Total number of expected tablets', type=int)
parser.add_option('-w', '--wait', help='Max wait time (s)', type=int,
default=300)
logging.getLogger().setLevel(logging.INFO)
options, _ = parser.parse_args()
logging.info('Waiting for mysql to become healthy.')
start_time = time.time()
good_tablets = []
tablets = []
# Do this in a loop as the output of ListAllTablets may not be parseable
# until all tablets have been started.
while (time.time() - start_time < options.wait and
len(tablets) < options.tablet_count):
tablets = get_all_tablets(options.cells, options.namespace)
logging.info('Expecting %d tablets, found %d tablets',
options.tablet_count, len(tablets))
start_time = time.time()
while time.time() - start_time < options.wait:
for tablet in [t for t in tablets if t not in good_tablets]:
_, success = vtctl_sandbox.execute_vtctl_command(
['ExecuteFetchAsDba', tablet, 'show databases'],
namespace=options.namespace, timeout_s=1)
if success:
good_tablets.append(tablet)
logging.info('%d of %d tablets healthy.', len(good_tablets), len(tablets))
if len(good_tablets) == len(tablets):
logging.info('All tablets healthy in %f seconds.',
time.time() - start_time)
break
else:
logging.warn('Timed out waiting for tablets to be ready.')
sys.exit(1)
def is_master(tablet, namespace):
tablet_info = (
vtctl_sandbox.execute_vtctl_command(
['GetTablet', tablet], namespace=namespace))
if json.loads(tablet_info[0])['type'] == topodata_pb2.MASTER:
return True
