def check_rows(to_look_for, driver="vtdb"):
out, err = utils.vtclient2(0,
"/test_nj/test_keyspace/master",
"select id, msg from vt_select_test",
driver=driver,
verbose=True)
for pattern in to_look_for:
if pattern not in err:
print "vtclient2 returned:"
print out
print err
raise utils.TestError('wrong vtclient2 output, missing: ' +
pattern)
if utils.options.verbose:
print out, err
def run_test_log_rotation():
start_position = _get_master_current_position()
master_tablet.mquery('vt_test_keyspace', "flush logs")
_exec_vt_txn(master_host, populate_vt_a(15))
_exec_vt_txn(master_host, ['delete from vt_a',])
master_conn = _get_master_stream_conn()
master_conn.dial()
binlog_pos, data, err = master_conn.stream_start(start_position)
if err:
raise utils.TestError("Update stream returned error '%s'", err)
master_txn_count = 0
logs_correct = False
while master_txn_count <=2:
binlog_pos, data, err = master_conn.stream_next()
if err:
raise utils.TestError("Update stream returned error '%s'", err)
if start_position['Position']['MasterFilename'] < binlog_pos['Position']['MasterFilename']:
logs_correct = True
utils.debug("Log rotation correctly interpreted")
break
if data['SqlType'] == 'COMMIT':
master_txn_count +=1
if not logs_correct:
raise utils.TestError("Flush logs didn't get properly interpreted")
def run_test_restart():
utils.zk_wipe()
utils.run_vtctl('CreateKeyspace -force test_keyspace')
# create the database so vttablets start, as it is serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0')
proc1 = tablet_62344.start_vttablet()
proc2 = tablet_62344.start_vttablet()
time.sleep(2.0)
proc1.poll()
if proc1.returncode is None:
raise utils.TestError("proc1 still running")
tablet_62344.kill_vttablet()
def check_rows_schema_diff(driver):
out, err = utils.vtclient2(0,
"/test_nj/test_keyspace/master",
"select * from vt_select_test",
driver=driver,
verbose=False,
raise_on_error=False)
if "column[0] name mismatch: id != msg" not in err and \
"column[0] name mismatch: msg != id" not in err:
print "vtclient2 returned:"
print out
print err
raise utils.TestError(
'wrong vtclient2 output, missing "name mismatch" of some kind')
if utils.options.verbose:
print out, err
def check_srv_keyspace(cell, keyspace, expected):
ks = utils.zk_cat_json('/zk/%s/vt/ns/%s' % (cell, keyspace))
result = ""
for tablet_type in sorted(ks['Partitions'].keys()):
result += "Partitions(%s):" % tablet_type
partition = ks['Partitions'][tablet_type]
for shard in partition['Shards']:
result = result + " %s-%s" % (shard['KeyRange']['Start'],
shard['KeyRange']['End'])
result += "\n"
result += "TabletTypes: " + ",".join(sorted(ks['TabletTypes']))
utils.debug("Cell %s keyspace %s has data:\n%s" % (cell, keyspace, result))
if result != expected:
raise utils.TestError(
"***** Expected srv keyspace:\n%s\nbut got:\n%s\n" %
(expected, result))
def run_test_restart():
zkocc_server = utils.zkocc_start()
shard_0_master.create_db('vt_test_keyspace')
proc1 = shard_0_master.start_vttablet(cert=cert_dir +
"/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
proc2 = shard_0_master.start_vttablet(cert=cert_dir +
"/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
time.sleep(2.0)
proc1.poll()
if proc1.returncode is None:
raise utils.TestError("proc1 still running")
shard_0_master.kill_vttablet()
utils.kill_sub_process(zkocc_server)
def test_vttablet_authenticated(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test,
self._populate_vt_select_test)
tablet_62344.start_vttablet(auth=True)
utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias)
err, out = tablet_62344.vquery('select * from vt_select_test', path='test_keyspace/0', user='******', password=r'ma kota')
logging.debug("Got rows: " + out)
if 'Row count: ' not in out:
raise utils.TestError("query didn't go through: %s, %s" % (err, out))
tablet_62344.kill_vttablet()
def _run_test_vtctl_partial_clone(create, populate,
start, end):
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force snapshot_test')
tablet_62344.init_tablet('master', 'snapshot_test', '0')
utils.run_vtctl('RebuildShardGraph snapshot_test/0')
utils.validate_topology()
tablet_62344.populate('vt_snapshot_test', create, populate)
tablet_62344.start_vttablet()
tablet_62044.init_tablet('idle', start=True)
# FIXME(alainjobart): not sure where the right place for this is,
# but it doesn't seem it should right here. It should be either in
# InitTablet (running an action on the vttablet), or in PartialClone
# (instead of doing a 'USE dbname' it could do a 'CREATE DATABASE
# dbname').
tablet_62044.mquery('', 'stop slave')
tablet_62044.create_db('vt_snapshot_test')
call(["touch", "/tmp/vtSimulateFetchFailures"])
utils.run_vtctl('PartialClone -force %s %s id %s %s' %
(tablet_62344.tablet_alias, tablet_62044.tablet_alias,
start, end))
utils.pause("after PartialClone")
# grab the new tablet definition from zk, make sure the start and
# end keys are set properly
out = utils.zk_cat(tablet_62044.zk_tablet_path)
if '"Start": "%s"' % start not in out or '"End": "%s"' % end not in out:
print "Tablet output:"
print "out"
raise utils.TestError('wrong Start or End')
tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 2)
utils.validate_topology()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
def run_test_sanity():
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.run_vtctl('RebuildKeyspaceGraph test_keyspace')
utils.validate_topology()
# if these statements don't run before the tablet it will wedge waiting for the
# db to become accessible. this is more a bug than a feature.
tablet_62344.populate('vt_test_keyspace', create_vt_select_test,
populate_vt_select_test)
tablet_62344.start_vttablet()
# make sure the query service is started right away
result, _ = utils.run_vtctl('Query test_nj test_keyspace "select * from vt_select_test"', trap_output=True)
rows = result.splitlines()
if len(rows) != 5:
raise utils.TestError("expected 5 rows in vt_select_test", rows, result)
# check Pings
utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)
utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias)
# Quickly check basic actions.
utils.run_vtctl('SetReadOnly ' + tablet_62344.tablet_alias)
utils.wait_db_read_only(62344)
utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias)
utils.check_db_read_write(62344)
utils.run_vtctl('DemoteMaster ' + tablet_62344.tablet_alias)
utils.wait_db_read_only(62344)
utils.validate_topology()
utils.run_vtctl('ValidateKeyspace test_keyspace')
# not pinging tablets, as it enables replication checks, and they
# break because we only have a single master, no slaves
utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0')
tablet_62344.kill_vttablet()
tablet_62344.init_tablet('idle')
tablet_62344.scrap(force=True)
def run_test_rebuild():
utils.run_vtctl('CreateKeyspace -force test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0')
tablet_62044.init_tablet('replica', 'test_keyspace', '0')
tablet_31981.init_tablet('experimental', 'test_keyspace', '0') # in ny by default
utils.run_vtctl('RebuildKeyspaceGraph -cells=test_nj test_keyspace',
auto_log=True)
utils.run_fail(utils.vtroot+'/bin/zk cat /zk/test_ny/vt/ns/test_keyspace/0/master')
utils.run_vtctl('RebuildKeyspaceGraph -cells=test_ny test_keyspace',
auto_log=True)
real_master = utils.zk_cat('/zk/test_nj/vt/ns/test_keyspace/0/master')
master_alias = utils.zk_cat('/zk/test_ny/vt/ns/test_keyspace/0/master')
if real_master != master_alias:
raise utils.TestError('master serving graph in all cells failed:\n%s!=\n%s'
% (real_master, master_alias))
def wait_for_vttablet_state(self, expected, timeout=5.0, port=None):
while True:
v = utils.get_vars(port or self.port)
if v == None:
logging.debug(" vttablet %s not answering at /debug/vars, waiting...", self.tablet_alias)
else:
if 'Voltron' not in v:
logging.debug(" vttablet %s not exporting Voltron, waiting...", self.tablet_alias)
else:
s = v['Voltron']['States']['Current']
if s != expected:
logging.debug(" vttablet %s in state %s != %s", self.tablet_alias, s, expected)
else:
break
logging.debug("sleeping a bit while we wait")
time.sleep(0.1)
timeout -= 0.1
if timeout <= 0:
raise utils.TestError("timeout waiting for state %s" % expected)
def wait_for_binlog_player_count(self, expected, timeout=30.0):
while True:
v = utils.get_vars(self.port)
if v == None:
logging.debug(" vttablet not answering at /debug/vars, waiting...")
else:
if 'BinlogPlayerMapSize' not in v:
logging.debug(" vttablet not exporting BinlogPlayerMapSize, waiting...")
else:
s = v['BinlogPlayerMapSize']
if s != expected:
logging.debug(" vttablet's binlog player map has count %u != %u",
s, expected)
else:
break
logging.debug("sleeping a bit while we wait")
time.sleep(0.5)
timeout -= 0.5
if timeout <= 0:
raise utils.TestError("timeout waiting for binlog player count %d" % expected)
logging.debug("tablet %s binlog player has %d players",
self.tablet_alias, expected)
def wait_for_binlog_server_state(self, expected, timeout=30.0):
while True:
v = utils.get_vars(self.port)
if v == None:
logging.debug(" vttablet not answering at /debug/vars, waiting...")
else:
if 'UpdateStreamState' not in v:
logging.debug(" vttablet not exporting BinlogServerState, waiting...")
else:
s = v['UpdateStreamState']
if s != expected:
logging.debug(" vttablet's binlog server in state %s != %s", s,
expected)
else:
break
logging.debug("sleeping a bit while we wait")
time.sleep(0.5)
timeout -= 0.5
if timeout <= 0:
raise utils.TestError("timeout waiting for binlog server state %s" % expected)
logging.debug("tablet %s binlog service is in state %s",
self.tablet_alias, expected)
def test_sigterm(self):
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as it is serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# start a 'vtctl Sleep' command in the background
sp = utils.run_bg(utils.vtroot+'/bin/vtctl -log_dir '+utils.tmp_root+' --alsologtostderr Sleep %s 60s' %
tablet_62344.tablet_alias,
stdout=PIPE, stderr=PIPE)
# wait for it to start, and let's kill it
time.sleep(2.0)
utils.run(['pkill', 'vtaction'])
out, err = sp.communicate()
# check the vtctl command got the right remote error back
if "vtaction interrupted by signal" not in err:
raise utils.TestError("cannot find expected output in error:", err)
logging.debug("vtaction was interrupted correctly:\n" + err)
tablet_62344.kill_vttablet()
def run_test_zkocc_qps():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start()
qpser = utils.run_bg(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2'
% utils.zkocc_port_base)
time.sleep(10)
utils.kill_sub_process(qpser)
# get the zkocc vars, make sure we have what we need
v = utils.get_vars(utils.zkocc_port_base)
if v['ZkReader']['test_nj']['State']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ',
v['ZkReader']['test_nj']['State']['Current'])
if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9:
raise utils.TestError(
'not enough time in Connected state',
v['ZkReader']['test_nj']['State']['DurationConnected'])
# some checks on performance / stats
# a typical workstation will do 15k QPS, check we have more than 3k
rpcCalls = v['ZkReader']['RpcCalls']
if rpcCalls < 30000:
raise utils.TestError('QPS is too low: %u < 30000', rpcCalls / 10)
cacheReads = v['ZkReader']['test_nj']['CacheReads']
if cacheReads < 30000:
raise utils.TestError('Cache QPS is too low: %u < 30000',
cacheReads / 10)
totalCacheReads = v['ZkReader']['total']['CacheReads']
if cacheReads != totalCacheReads:
raise utils.TestError('Rollup stats are wrong: %u != %u', cacheReads,
totalCacheReads)
if v['ZkReader']['UnknownCellErrors'] != 0:
raise utils.TestError('unexpected UnknownCellErrors',
v['ZkReader']['UnknownCellErrors'])
utils.zkocc_kill(zkocc_14850)
def run_test_restart_during_action():
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
tablet_62344.init_tablet('master', 'test_keyspace', '0')
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
tablet_62344.create_db('vt_test_keyspace')
tablet_62344.start_vttablet()
utils.run_vtctl('Ping ' + tablet_62344.tablet_alias)
# schedule long action
utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias,
stdout=devnull)
# ping blocks until the sleep finishes unless we have a schedule race
action_path, _ = utils.run_vtctl('-no-wait Ping ' +
tablet_62344.tablet_alias,
trap_output=True)
# kill agent leaving vtaction running
tablet_62344.kill_vttablet()
# restart agent
tablet_62344.start_vttablet()
# we expect this action with a short wait time to fail. this isn't the best
# and has some potential for flakiness.
utils.run_fail(
utils.vtroot +
'/bin/vtctl --alsologtostderr -wait-time 2s WaitForAction ' +
action_path)
# wait until the background sleep action is done, otherwise there will be
# a leftover vtaction whose result may overwrite running actions
# NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug:
# the zombie Sleep clobbers the Clone command in the following tests
utils.run(utils.vtroot +
'/bin/vtctl --alsologtostderr -wait-time 20s WaitForAction ' +
action_path)
# extra small test: we ran for a while, get the states we were in,
# make sure they're accounted for properly
# first the query engine States
v = utils.get_vars(tablet_62344.port)
utils.debug("vars: %s" % str(v))
if v['Voltron']['States']['DurationOPEN'] < 10e9:
raise utils.TestError('not enough time in Open state',
v['Voltron']['States']['DurationOPEN'])
# then the Zookeeper connections
if v['ZkMetaConn']['test_nj']['Current'] != 'Connected':
raise utils.TestError('invalid zk test_nj state: ',
v['ZkMetaConn']['test_nj']['Current'])
if v['ZkMetaConn']['global']['Current'] != 'Connected':
raise utils.TestError('invalid zk global state: ',
v['ZkMetaConn']['global']['Current'])
if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9:
raise utils.TestError('not enough time in Connected state',
v['ZkMetaConn']['test_nj']['DurationConnected'])
if v['tablet-type'] != 'master':
raise utils.TestError('tablet-type not exported correctly')
tablet_62344.kill_vttablet()
def run_test_reparent_from_outside():
utils.zk_wipe()
utils.run_vtctl('CreateKeyspace test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True)
# Reparent as a starting point
utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias)
# now manually reparent 1 out of 2 tablets
# 62044 will be the new master
# 31981 won't be re-parented, so it w2ill be busted
tablet_62044.mquery('', [
"RESET MASTER",
"STOP SLAVE",
"RESET SLAVE",
"CHANGE MASTER TO MASTER_HOST = ''",
])
new_pos = tablet_62044.mquery('', 'show master status')
utils.debug("New master position: %s" % str(new_pos))
# 62344 will now be a slave of 62044
tablet_62344.mquery('', [
"RESET MASTER",
"RESET SLAVE",
"change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# 41983 will be a slave of 62044
tablet_41983.mquery('', [
'stop slave',
"change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]),
'start slave'
])
# update zk with the new graph
utils.run_vtctl('ShardExternallyReparented -scrap-stragglers test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True)
# make sure the replication graph is fine
shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0')
utils.debug('shard_files: %s' % " ".join(shard_files))
if shard_files != ['action', 'actionlog', 'test_nj-0000062044']:
raise utils.TestError('unexpected zk content: %s' % " ".join(shard_files))
slave_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044')
utils.debug('slave_files: %s' % " ".join(slave_files))
if slave_files != ['test_nj-0000041983', 'test_nj-0000062344']:
raise utils.TestError('unexpected zk content: %s' % " ".join(slave_files))
tablet_31981.kill_vttablet()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
tablet_41983.kill_vttablet()
def _check_db_addr(db_addr, expected_addr):
# Run in the background to capture output.
proc = utils.run_bg(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=WARNING -zk.local-cell=test_nj Resolve ' + db_addr, stdout=PIPE)
stdout = proc.communicate()[0].strip()
if stdout != expected_addr:
raise utils.TestError('wrong zk address', db_addr, stdout, expected_addr)
def run_test_reparent_down_master():
utils.zk_wipe()
utils.run_vtctl('CreateKeyspace -force test_keyspace')
# create the database so vttablets start, as they are serving
tablet_62344.create_db('vt_test_keyspace')
tablet_62044.create_db('vt_test_keyspace')
tablet_41983.create_db('vt_test_keyspace')
tablet_31981.create_db('vt_test_keyspace')
# Start up a master mysql and vttablet
tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True)
# Create a few slaves for testing reparenting.
tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True)
tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True)
# Recompute the shard layout node - until you do that, it might not be valid.
utils.run_vtctl('RebuildShardGraph test_keyspace/0')
utils.validate_topology()
# Force the slaves to reparent assuming that all the datasets are identical.
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + tablet_62344.tablet_alias, auto_log=True)
utils.validate_topology()
# Make the master agent and database unavailable.
tablet_62344.kill_vttablet()
tablet_62344.shutdown_mysql().wait()
expected_addr = utils.hostname + ':' + str(tablet_62344.port)
_check_db_addr('test_keyspace.0.master:_vtocc', expected_addr)
# Perform a reparent operation - the Validate part will try to ping
# the master and fail somewhat quickly
stdout, stderr = utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=INFO -wait-time 5s ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias)
utils.debug("Failed ReparentShard output:\n" + stderr)
if 'ValidateShard verification failed: timed out during validate' not in stderr:
raise utils.TestError("didn't find the right error strings in failed ReparentShard: " + stderr)
# Should timeout and fail
stdout, stderr = utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=INFO -wait-time 5s ScrapTablet ' + tablet_62344.tablet_alias)
utils.debug("Failed ScrapTablet output:\n" + stderr)
if 'deadline exceeded' not in stderr:
raise utils.TestError("didn't find the right error strings in failed ScrapTablet: " + stderr)
# Should interrupt and fail
sp = utils.run_bg(utils.vtroot+'/bin/vtctl -log.level=INFO -wait-time 10s ScrapTablet ' + tablet_62344.tablet_alias, stdout=PIPE, stderr=PIPE)
# Need time for the process to start before killing it.
time.sleep(0.1)
os.kill(sp.pid, signal.SIGINT)
stdout, stderr = sp.communicate()
utils.debug("Failed ScrapTablet output:\n" + stderr)
if 'interrupted' not in stderr:
raise utils.TestError("didn't find the right error strings in failed ScrapTablet: " + stderr)
# Force the scrap action in zk even though tablet is not accessible.
tablet_62344.scrap(force=True)
utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=WARNING ChangeSlaveType -force %s idle' %
tablet_62344.tablet_alias)
# Remove pending locks (make this the force option to ReparentShard?)
utils.run_vtctl('PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action')
# Re-run reparent operation, this shoud now proceed unimpeded.
utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, auto_log=True)
utils.validate_topology()
expected_addr = utils.hostname + ':' + str(tablet_62044.port)
_check_db_addr('test_keyspace.0.master:_vtocc', expected_addr)
utils.run_vtctl('ChangeSlaveType -force %s idle' % tablet_62344.tablet_alias)
idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj', trap_output=True)
if '0000062344 <null> <null> idle' not in idle_tablets:
raise utils.TestError('idle tablet not found', idle_tablets)
tablet_62044.kill_vttablet()
tablet_41983.kill_vttablet()
tablet_31981.kill_vttablet()
# sothe other tests don't have any surprise
tablet_62344.start_mysql().wait()
def test_multisnapshot_and_restore():
tables = ['vt_insert_test', 'vt_insert_test1']
create_template = '''create table %s (
id bigint auto_increment,
msg varchar(64),
primary key (id),
index by_msg (msg)
) Engine=InnoDB'''
create_view = '''create view vt_insert_view(id, msg) as select id, msg from vt_insert_test'''
insert_template = "insert into %s (id, msg) values (%s, 'test %s')"
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force test_keyspace')
# Start three tablets for three different shards. At this point the
# sharding schema is not really important, as long as it is
# consistent.
new_spec = '-0000000000000028-'
old_tablets = [tablet_62044, tablet_41983, tablet_31981]
for i, tablet in enumerate(old_tablets):
tablet.init_tablet('master', 'test_keyspace', str(i))
utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i)
utils.validate_topology()
for i, tablet in enumerate(old_tablets):
tablet.populate(
"vt_test_keyspace",
[create_template % table for table in tables] + [create_view],
sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], []))
tablet.start_vttablet()
utils.run_vtctl('MultiSnapshot --force --spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True)
utils.pause("After snapshot")
# try to get the schema on the source, make sure the view is there
out, err = utils.run_vtctl('GetSchema --include-views ' +
tablet_62044.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
raise utils.TestError('Unexpected GetSchema --include-views output: %s' % err)
out, err = utils.run_vtctl('GetSchema ' +
tablet_62044.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' in err:
raise utils.TestError('Unexpected GetSchema output: %s' % err)
utils.run_vtctl('CreateKeyspace -force test_keyspace_new')
tablet_62344.init_tablet('master', 'test_keyspace_new', "0", dbname="not_vt_test_keyspace")
utils.run_vtctl('RebuildShardGraph test_keyspace_new/0')
utils.validate_topology()
tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace')
tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created
tablet_urls = ' '.join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets)
# 0x28 = 40
err = tablet_62344.mysqlctl("multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam,populateBlpRecovery(6514) not_vt_test_keyspace %s" % tablet_urls).wait()
if err != 0:
raise utils.TestError("mysqlctl failed: %u" % err)
for table in tables:
rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table)
if len(rows) == 0:
raise utils.TestError("There are no rows in the restored database.")
for row in rows:
if row[0] > 32:
raise utils.TestError("Bad row: %s" % row)
rows = tablet_62344.mquery('_vt', 'select * from blp_checkpoint')
if len(rows) != 3:
raise utils.TestError("Was expecting 3 rows in blp_checkpoint but got: %s" % str(rows))
# try to get the schema on multi-restored guy, make sure the view is there
out, err = utils.run_vtctl('GetSchema --include-views ' +
tablet_62344.tablet_alias,
log_level='INFO', trap_output=True)
if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err:
raise utils.TestError('Unexpected GetSchema --include-views output after multirestore: %s' % err)
for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344:
tablet.kill_vttablet()
def run_test_sharding():
utils.run_vtctl('CreateKeyspace test_keyspace')
shard_0_master.init_tablet('master', 'test_keyspace', '-80')
shard_0_replica.init_tablet('replica', 'test_keyspace', '-80')
shard_1_master.init_tablet('master', 'test_keyspace', '80-')
shard_1_replica.init_tablet('replica', 'test_keyspace', '80-')
utils.run_vtctl(
'RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*',
auto_log=True)
utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*',
auto_log=True)
# run checks now before we start the tablets
utils.validate_topology()
# create databases
shard_0_master.create_db('vt_test_keyspace')
shard_0_replica.create_db('vt_test_keyspace')
shard_1_master.create_db('vt_test_keyspace')
shard_1_replica.create_db('vt_test_keyspace')
# start the tablets
shard_0_master.start_vttablet()
shard_0_replica.start_vttablet()
shard_1_master.start_vttablet()
shard_1_replica.start_vttablet()
# apply the schema on the first shard through vtctl, so all tablets
# are the same (replication is not enabled yet, so allow_replication=false
# is just there to be tested)
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_master.tablet_alias
])
utils.run_vtctl([
'ApplySchema', '-stop-replication',
'-sql=' + create_vt_select_test.replace("\n", ""),
shard_0_replica.tablet_alias
])
# start zkocc, we'll use it later
zkocc_server = utils.zkocc_start()
utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' +
shard_0_master.tablet_alias,
auto_log=True)
utils.run_vtctl('ReparentShard -force test_keyspace/80- ' +
shard_1_master.tablet_alias,
auto_log=True)
# apply the schema on the second shard using a simple schema upgrade
utils.run_vtctl([
'ApplySchemaShard', '-simple',
'-sql=' + create_vt_select_test_reverse.replace("\n", ""),
'test_keyspace/80-'
])
# insert some values directly (db is RO after minority reparent)
# FIXME(alainjobart) these values don't match the shard map
utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias)
utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias)
shard_0_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (1, 'test 1')",
write=True)
shard_1_master.mquery(
'vt_test_keyspace',
"insert into vt_select_test (id, msg) values (10, 'test 10')",
write=True)
utils.validate_topology(ping_tablets=True)
utils.pause("Before the sql scatter query")
# note the order of the rows is not guaranteed, as the go routines
# doing the work can go out of order
check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"])
# write a value, re-read them all
utils.vtclient2(
3803,
"/test_nj/test_keyspace/master",
"insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')",
bindvars='{"keyspace_id": 2}',
driver="vtdb",
verbose=True)
check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"])
# make sure the '2' value was written on first shard
rows = shard_0_master.mquery(
'vt_test_keyspace', "select id, msg from vt_select_test order by id")
if (len(rows) != 2 or \
rows[0][0] != 1 or \
rows[1][0] != 2):
print "mysql_query returned:", rows
raise utils.TestError('wrong mysql_query output')
utils.pause("After db writes")
# now use zkocc or streaming or both for the same query
check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc")
check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-streaming")
check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"],
driver="vtdb-zkocc-streaming")
# make sure the schema checking works
check_rows_schema_diff("vtdb-zkocc")
check_rows_schema_diff("vtdb")
# throw in some schema validation step
# we created the schema differently, so it should show
utils.run_vtctl('ValidateSchemaShard test_keyspace/-80')
utils.run_vtctl('ValidateSchemaShard test_keyspace/80-')
out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace',
trap_output=True,
raise_on_error=False)
if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \
"test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err:
raise utils.TestError('wrong ValidateSchemaKeyspace output: ' + err)
# validate versions
utils.run_vtctl('ValidateVersionShard test_keyspace/-80', auto_log=True)
utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True)
# show and validate permissions
utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True)
utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80',
auto_log=True)
utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace', auto_log=True)
# and create zkns on this complex keyspace, make sure a few files are created
utils.run_vtctl('ExportZknsForKeyspace test_keyspace')
out, err = utils.run(utils.vtroot +
'/bin/zk ls -R /zk/test_nj/zk?s/vt/test_keysp*',
trap_output=True)
lines = out.splitlines()
for base in ['-80', '80-']:
for db_type in ['master', 'replica']:
for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']:
expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path
if expected not in lines:
raise utils.TestError('missing zkns part:\n%s\nin:%s' %
(expected, out))
# now try to connect using the python client and shard-aware connection
# to both shards
# first get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.-80.master:_vtocc")
if len(shard_0_master_addrs) != 1:
raise utils.TestError(
'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
utils.debug("shard 0 master addrs: %s" % " ".join(
["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# connect to shard -80
conn = tablet2.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "-80", 10.0)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select id, msg from vt_select_test order by id",
{})
if (len(results) != 2 or \
results[0][0] != 1 or \
results[1][0] != 2):
print "conn._execute returned:", results
raise utils.TestError('wrong conn._execute output')
# connect to shard 80-
shard_1_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.80-.master:_vtocc")
conn = tablet2.TabletConnection(
"%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]),
"test_keyspace", "80-", 10.0)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select id, msg from vt_select_test order by id",
{})
if (len(results) != 1 or \
results[0][0] != 10):
print "conn._execute returned:", results
raise utils.TestError('wrong conn._execute output')
# try to connect with bad shard
try:
conn = tablet2.TabletConnection("localhost:%u" % shard_0_master.port,
"test_keyspace", "-90", 10.0)
conn.dial()
raise utils.TestError('expected an exception')
except Exception as e:
if "fatal: Shard mismatch, expecting -80, received -90" not in str(e):
raise utils.TestError('unexpected exception: ' + str(e))
utils.kill_sub_process(zkocc_server)
shard_0_master.kill_vttablet()
shard_0_replica.kill_vttablet()
shard_1_master.kill_vttablet()
shard_1_replica.kill_vttablet()
def _run_test_vtctl_snapshot_restore(server_mode):
if server_mode:
snapshot_flags = '-server-mode -concurrency=8'
restore_flags = '-dont-wait-for-slave-start'
else:
snapshot_flags = '-concurrency=4'
restore_flags = ''
utils.zk_wipe()
# Start up a master mysql and vttablet
utils.run_vtctl('CreateKeyspace -force snapshot_test')
tablet_62344.init_tablet('master', 'snapshot_test', '0')
utils.run_vtctl('RebuildShardGraph snapshot_test/0')
utils.validate_topology()
tablet_62344.populate('vt_snapshot_test', create_vt_insert_test,
populate_vt_insert_test)
tablet_62044.create_db('vt_snapshot_test')
tablet_62344.start_vttablet()
# Need to force snapshot since this is a master db.
out, err = utils.run_vtctl('Snapshot -force %s %s ' % (snapshot_flags, tablet_62344.tablet_alias), log_level='INFO', trap_output=True)
results = {}
for name in ['Manifest', 'ParentAlias', 'SlaveStartRequired', 'ReadOnly', 'OriginalType']:
sepPos = err.find(name + ": ")
if sepPos != -1:
results[name] = err[sepPos+len(name)+2:].splitlines()[0]
if "Manifest" not in results:
raise utils.TestError("Snapshot didn't echo Manifest file", err)
if "ParentAlias" not in results:
raise utils.TestError("Snapshot didn't echo ParentAlias", err)
utils.pause("snapshot finished: " + results['Manifest'] + " " + results['ParentAlias'])
if server_mode:
if "SlaveStartRequired" not in results:
raise utils.TestError("Snapshot didn't echo SlaveStartRequired", err)
if "ReadOnly" not in results:
raise utils.TestError("Snapshot didn't echo ReadOnly", err)
if "OriginalType" not in results:
raise utils.TestError("Snapshot didn't echo OriginalType", err)
if (results['SlaveStartRequired'] != 'false' or
results['ReadOnly'] != 'true' or
results['OriginalType'] != 'master'):
raise utils.TestError("Bad values returned by Snapshot", err)
tablet_62044.init_tablet('idle', start=True)
# do not specify a MANIFEST, see if 'default' works
call(["touch", "/tmp/vtSimulateFetchFailures"])
utils.run_vtctl('Restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s default %s %s' %
(restore_flags, tablet_62344.tablet_alias,
tablet_62044.tablet_alias, results['ParentAlias']), auto_log=True)
utils.pause("restore finished")
tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)
utils.validate_topology()
# in server_mode, get the server out of it and check it
if server_mode:
utils.run_vtctl('SnapshotSourceEnd %s %s' % (tablet_62344.tablet_alias, results['OriginalType']), auto_log=True)
tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4)
utils.validate_topology()
tablet_62344.kill_vttablet()
tablet_62044.kill_vttablet()
def test_zkocc(self):
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(
extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u,localhost:%u" %
(bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj",
30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(logging.ERROR)
# FIXME(ryszard): This can be changed into a self.assertRaises.
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(logging.WARNING)
# get test
utils.prog_compile(['zkclient2'])
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True)
self.assertEqual(
err,
"/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n"
)
zk_data = zkocc_client.get("/zk/test_nj/zkocc1/data1")
self.assertDictContainsSubset(
{
'Data': "Test data 1",
'Cached': True,
'Stale': False,
}, zk_data)
self.assertDictContainsSubset({
'NumChildren': 0,
'Version': 0
}, zk_data['Stat'])
# getv test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3'
% utils.zkocc_port_base,
trap_output=True)
self.assertEqualNormalized(
err,
"""[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""")
zk_data = zkocc_client.getv([
"/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2",
"/zk/test_nj/zkocc1/data3"
])['Nodes']
self.assertEqual(len(zk_data), 3)
for i, d in enumerate(zk_data):
self.assertEqual(d['Data'], 'Test data %s' % (i + 1))
self.assertTrue(d['Cached'])
self.assertFalse(d['Stale'])
self.assertDictContainsSubset({
'NumChildren': 0,
'Version': 0
}, d['Stat'])
# children test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' %
utils.zkocc_port_base,
trap_output=True)
self.assertEqualNormalized(
err, """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""")
# zk command tests
self._check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
self._check_zk_output(
"ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zk_data[0]['Stat']['MTime']),
_format_time(zk_data[1]['Stat']['MTime']),
_format_time(zk_data[2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True,
raise_on_error=False)
self.assertIn("zkocc: cannot resolve local cell", err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg(
'/bin/bash -c "while true ; do ' + utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"'
% utils.zkocc_port_base,
stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u shutdown' %
(utils.zk_port_base, utils.zk_port_base + 1,
utils.zk_port_base + 2))
time.sleep(3)
utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base +
1, utils.zk_port_base + 2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
self.fail('unexpected stale state')
self.assertEqual(state, 2)
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
self.fail('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise
logging.getLogger().setLevel(logging.WARNING)
def assert_table_count(self, dbname, table, n, where=''):
result = self.mquery(dbname,
'select count(*) from ' + table + ' ' + where)
if result[0][0] != n:
raise utils.TestError('expected %d rows in %s' % (n, table),
result)
def openssl(cmd):
result = subprocess.call(["openssl"] + cmd)
if result != 0:
raise utils.TestError("OpenSSL command failed: %s" % " ".join(cmd))
def check_db_status(self, name, value):
row = self.get_db_status(name)
if row[1] != value:
raise utils.TestError('status not correct', name, row)
def check_db_var(self, name, value):
row = self.get_db_var(name)
if row != (name, value):
raise utils.TestError('variable not set correctly', name, row)
def run_test_zkocc():
_populate_zk()
# preload the test_nj cell
zkocc_14850 = utils.zkocc_start(
extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s'])
time.sleep(1)
# create a python client. The first address is bad, will test the retry logic
bad_port = utils.reserve_ports(3)
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u,localhost:%u" %
(bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30)
zkocc_client.dial()
# test failure for a python client that cannot connect
bad_zkocc_client = zkocc.ZkOccConnection(
"localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30)
try:
bad_zkocc_client.dial()
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.ERROR)
try:
bad_zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
# get test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True)
if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n":
raise utils.TestError('unexpected get output: ', err)
zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1")
if (zkNode['Data'] != "Test data 1" or \
zkNode['Stat']['NumChildren'] != 0 or \
zkNode['Stat']['Version'] != 0 or \
zkNode['Cached'] != True or \
zkNode['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.get output: ', zkNode)
# getv test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3'
% utils.zkocc_port_base,
trap_output=True)
if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)
[1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false)
[2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false)
""":
raise utils.TestError('unexpected getV output: ', err)
zkNodes = zkocc_client.getv([
"/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2",
"/zk/test_nj/zkocc1/data3"
])
if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \
zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][0]['Cached'] != True or \
zkNodes['Nodes'][0]['Stale'] != False or \
zkNodes['Nodes'][1]['Data'] != "Test data 2" or \
zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][1]['Cached'] != True or \
zkNodes['Nodes'][1]['Stale'] != False or \
zkNodes['Nodes'][2]['Data'] != "Test data 3" or \
zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \
zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \
zkNodes['Nodes'][2]['Cached'] != True or \
zkNodes['Nodes'][2]['Stale'] != False):
raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes)
# children test
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' %
utils.zkocc_port_base,
trap_output=True)
if err != """Path = /zk/test_nj
Child[0] = zkocc1
Child[1] = zkocc2
NumChildren = 2
CVersion = 4
Cached = false
Stale = false
""":
raise utils.TestError('unexpected children output: ', err)
# zk command tests
_check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1")
_check_zk_output(
"ls -l /zk/test_nj/zkocc1", """total: 3
-rw-rw-rw- zk zk 11 %s data1
-rw-rw-rw- zk zk 11 %s data2
-rw-rw-rw- zk zk 11 %s data3
""" % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][1]['Stat']['MTime']),
_format_time(zkNodes['Nodes'][2]['Stat']['MTime'])))
# test /zk/local is not resolved and rejected
out, err = utils.run(
utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' %
utils.zkocc_port_base,
trap_output=True,
raise_on_error=False)
if "zkocc: cannot resolve local cell" not in err:
raise utils.TestError('unexpected get output, not local cell error: ',
err)
# start a background process to query the same value over and over again
# while we kill the zk server and restart it
outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False)
filename = outfd.name
querier = utils.run_bg(
'/bin/bash -c "while true ; do ' + utils.vtroot +
'/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"'
% utils.zkocc_port_base,
stderr=outfd.file)
outfd.close()
time.sleep(1)
# kill zk server, sleep a bit, restart zk server, sleep a bit
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u shutdown' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.run(
utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname +
':%u:%u:%u start' %
(utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2))
time.sleep(3)
utils.kill_sub_process(querier)
utils.debug("Checking " + filename)
fd = open(filename, "r")
state = 0
for line in fd:
if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n":
stale = False
elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n":
stale = True
else:
raise utils.TestError('unexpected line: ', line)
if state == 0:
if stale:
state = 1
elif state == 1:
if not stale:
state = 2
else:
if stale:
raise utils.TestError('unexpected stale state')
if state != 2:
raise utils.TestError('unexpected ended stale state')
fd.close()
utils.zkocc_kill(zkocc_14850)
# check that after the server is gone, the python client fails correctly
logging.getLogger().setLevel(logging.ERROR)
try:
zkocc_client.get("/zk/test_nj/zkocc1/data1")
raise utils.TestError('exception expected')
except zkocc.ZkOccError as e:
if str(e) != "Cannot dial to any server":
raise utils.TestError('Unexpected exception: ', str(e))
logging.getLogger().setLevel(logging.WARNING)
def check_vttablet_count(cls):
if Tablet.tablets_running > 0:
raise utils.TestError('This test is not killing all its vttablets')
def run_test_secure():
zkocc_server = utils.zkocc_start()
# start the tablets
shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem")
shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem",
key=cert_dir + "/vt-server-key.pem",
repl_extra_flags={
'flags': 2048,
'ssl_ca': cert_dir + "/ca-cert.pem",
'ssl_cert': cert_dir + "/client-cert.pem",
'ssl_key': cert_dir + "/client-key.pem",
})
# Reparent using SSL
for t in [shard_0_master, shard_0_slave]:
t.reset_replication()
utils.run_vtctl('ReparentShard -force test_keyspace/0 ' +
shard_0_master.tablet_alias,
auto_log=True)
# then get the topology and check it
zkocc_client = zkocc.ZkOccConnection(
"localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0)
topology.read_keyspaces(zkocc_client)
shard_0_master_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vts")
if len(shard_0_master_addrs) != 1:
raise utils.TestError(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s'
% " ".join([
"%s:%u(%s)" % (h, p, str(e))
for (h, p, e) in shard_0_master_addrs
]))
if shard_0_master_addrs[0][2] != True:
raise utils.TestError(
'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted'
)
utils.debug("shard 0 master addrs: %s" % " ".join(
["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs]))
# make sure asking for optionally secure connections works too
auto_addrs = topology.get_host_port_by_name(
zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True)
if auto_addrs != shard_0_master_addrs:
raise utils.TestError(
'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s'
% (str(shard_0_master_addrs), str(auto_addrs)))
# try to connect with regular client
try:
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace", "0", 10.0)
conn.dial()
raise utils.TestError("No exception raised to secure port")
except tablet3.FatalError as e:
if not e.args[0][0].startswith('Unexpected EOF in handshake to'):
raise utils.TestError("Unexpected exception: %s" % str(e))
sconn = utils.get_vars(shard_0_master.port)["SecureConns"]
if sconn != 0:
raise utils.TestError("unexpected conns %s" % sconn)
# connect to encrypted port
conn = tablet3.TabletConnection(
"%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]),
"test_keyspace",
"0",
5.0,
encrypted=True)
conn.dial()
(results, rowcount, lastrowid,
fields) = conn._execute("select 1 from dual", {})
if (len(results) != 1 or \
results[0][0] != 1):
print "conn._execute returned:", results
raise utils.TestError('wrong conn._execute output')
sconn = utils.get_vars(shard_0_master.port)["SecureConns"]
if sconn != 1:
raise utils.TestError("unexpected conns %s" % sconn)
saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"]
if saccept == 0:
raise utils.TestError("unexpected accepts %s" % saccept)
# trigger a time out on a secure connection, see what exception we get
try:
conn._execute("select sleep(100) from dual", {})
raise utils.TestError("No timeout exception")
except tablet3.TimeoutError as e:
utils.debug("Got the right exception for SSL timeout: %s" % str(e))
# kill everything
utils.kill_sub_process(zkocc_server)
