def check_rows(to_look_for, driver="vtdb"): out, err = utils.vtclient2(0, "/test_nj/test_keyspace/master", "select id, msg from vt_select_test", driver=driver, verbose=True) for pattern in to_look_for: if pattern not in err: print "vtclient2 returned:" print out print err raise utils.TestError('wrong vtclient2 output, missing: ' + pattern) if utils.options.verbose: print out, err
def run_test_log_rotation(): start_position = _get_master_current_position() master_tablet.mquery('vt_test_keyspace', "flush logs") _exec_vt_txn(master_host, populate_vt_a(15)) _exec_vt_txn(master_host, ['delete from vt_a',]) master_conn = _get_master_stream_conn() master_conn.dial() binlog_pos, data, err = master_conn.stream_start(start_position) if err: raise utils.TestError("Update stream returned error '%s'", err) master_txn_count = 0 logs_correct = False while master_txn_count <=2: binlog_pos, data, err = master_conn.stream_next() if err: raise utils.TestError("Update stream returned error '%s'", err) if start_position['Position']['MasterFilename'] < binlog_pos['Position']['MasterFilename']: logs_correct = True utils.debug("Log rotation correctly interpreted") break if data['SqlType'] == 'COMMIT': master_txn_count +=1 if not logs_correct: raise utils.TestError("Flush logs didn't get properly interpreted")
def run_test_restart(): utils.zk_wipe() utils.run_vtctl('CreateKeyspace -force test_keyspace') # create the database so vttablets start, as it is serving tablet_62344.create_db('vt_test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') proc1 = tablet_62344.start_vttablet() proc2 = tablet_62344.start_vttablet() time.sleep(2.0) proc1.poll() if proc1.returncode is None: raise utils.TestError("proc1 still running") tablet_62344.kill_vttablet()
def check_rows_schema_diff(driver): out, err = utils.vtclient2(0, "/test_nj/test_keyspace/master", "select * from vt_select_test", driver=driver, verbose=False, raise_on_error=False) if "column[0] name mismatch: id != msg" not in err and \ "column[0] name mismatch: msg != id" not in err: print "vtclient2 returned:" print out print err raise utils.TestError( 'wrong vtclient2 output, missing "name mismatch" of some kind') if utils.options.verbose: print out, err
def check_srv_keyspace(cell, keyspace, expected): ks = utils.zk_cat_json('/zk/%s/vt/ns/%s' % (cell, keyspace)) result = "" for tablet_type in sorted(ks['Partitions'].keys()): result += "Partitions(%s):" % tablet_type partition = ks['Partitions'][tablet_type] for shard in partition['Shards']: result = result + " %s-%s" % (shard['KeyRange']['Start'], shard['KeyRange']['End']) result += "\n" result += "TabletTypes: " + ",".join(sorted(ks['TabletTypes'])) utils.debug("Cell %s keyspace %s has data:\n%s" % (cell, keyspace, result)) if result != expected: raise utils.TestError( "***** Expected srv keyspace:\n%s\nbut got:\n%s\n" % (expected, result))
def run_test_restart(): zkocc_server = utils.zkocc_start() shard_0_master.create_db('vt_test_keyspace') proc1 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") proc2 = shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") time.sleep(2.0) proc1.poll() if proc1.returncode is None: raise utils.TestError("proc1 still running") shard_0_master.kill_vttablet() utils.kill_sub_process(zkocc_server)
def test_vttablet_authenticated(self): utils.run_vtctl('CreateKeyspace test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.populate('vt_test_keyspace', self._create_vt_select_test, self._populate_vt_select_test) tablet_62344.start_vttablet(auth=True) utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias) err, out = tablet_62344.vquery('select * from vt_select_test', path='test_keyspace/0', user='******', password=r'ma kota') logging.debug("Got rows: " + out) if 'Row count: ' not in out: raise utils.TestError("query didn't go through: %s, %s" % (err, out)) tablet_62344.kill_vttablet()
def _run_test_vtctl_partial_clone(create, populate, start, end): utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force snapshot_test') tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl('RebuildShardGraph snapshot_test/0') utils.validate_topology() tablet_62344.populate('vt_snapshot_test', create, populate) tablet_62344.start_vttablet() tablet_62044.init_tablet('idle', start=True) # FIXME(alainjobart): not sure where the right place for this is, # but it doesn't seem it should right here. It should be either in # InitTablet (running an action on the vttablet), or in PartialClone # (instead of doing a 'USE dbname' it could do a 'CREATE DATABASE # dbname'). tablet_62044.mquery('', 'stop slave') tablet_62044.create_db('vt_snapshot_test') call(["touch", "/tmp/vtSimulateFetchFailures"]) utils.run_vtctl('PartialClone -force %s %s id %s %s' % (tablet_62344.tablet_alias, tablet_62044.tablet_alias, start, end)) utils.pause("after PartialClone") # grab the new tablet definition from zk, make sure the start and # end keys are set properly out = utils.zk_cat(tablet_62044.zk_tablet_path) if '"Start": "%s"' % start not in out or '"End": "%s"' % end not in out: print "Tablet output:" print "out" raise utils.TestError('wrong Start or End') tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 2) utils.validate_topology() tablet_62344.kill_vttablet() tablet_62044.kill_vttablet()
def run_test_sanity(): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.run_vtctl('RebuildKeyspaceGraph test_keyspace') utils.validate_topology() # if these statements don't run before the tablet it will wedge waiting for the # db to become accessible. this is more a bug than a feature. tablet_62344.populate('vt_test_keyspace', create_vt_select_test, populate_vt_select_test) tablet_62344.start_vttablet() # make sure the query service is started right away result, _ = utils.run_vtctl('Query test_nj test_keyspace "select * from vt_select_test"', trap_output=True) rows = result.splitlines() if len(rows) != 5: raise utils.TestError("expected 5 rows in vt_select_test", rows, result) # check Pings utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) utils.run_vtctl('RpcPing ' + tablet_62344.tablet_alias) # Quickly check basic actions. utils.run_vtctl('SetReadOnly ' + tablet_62344.tablet_alias) utils.wait_db_read_only(62344) utils.run_vtctl('SetReadWrite ' + tablet_62344.tablet_alias) utils.check_db_read_write(62344) utils.run_vtctl('DemoteMaster ' + tablet_62344.tablet_alias) utils.wait_db_read_only(62344) utils.validate_topology() utils.run_vtctl('ValidateKeyspace test_keyspace') # not pinging tablets, as it enables replication checks, and they # break because we only have a single master, no slaves utils.run_vtctl('ValidateShard -ping-tablets=false test_keyspace/0') tablet_62344.kill_vttablet() tablet_62344.init_tablet('idle') tablet_62344.scrap(force=True)
def run_test_rebuild(): utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') tablet_62044.init_tablet('replica', 'test_keyspace', '0') tablet_31981.init_tablet('experimental', 'test_keyspace', '0') # in ny by default utils.run_vtctl('RebuildKeyspaceGraph -cells=test_nj test_keyspace', auto_log=True) utils.run_fail(utils.vtroot+'/bin/zk cat /zk/test_ny/vt/ns/test_keyspace/0/master') utils.run_vtctl('RebuildKeyspaceGraph -cells=test_ny test_keyspace', auto_log=True) real_master = utils.zk_cat('/zk/test_nj/vt/ns/test_keyspace/0/master') master_alias = utils.zk_cat('/zk/test_ny/vt/ns/test_keyspace/0/master') if real_master != master_alias: raise utils.TestError('master serving graph in all cells failed:\n%s!=\n%s' % (real_master, master_alias))
def wait_for_vttablet_state(self, expected, timeout=5.0, port=None): while True: v = utils.get_vars(port or self.port) if v == None: logging.debug(" vttablet %s not answering at /debug/vars, waiting...", self.tablet_alias) else: if 'Voltron' not in v: logging.debug(" vttablet %s not exporting Voltron, waiting...", self.tablet_alias) else: s = v['Voltron']['States']['Current'] if s != expected: logging.debug(" vttablet %s in state %s != %s", self.tablet_alias, s, expected) else: break logging.debug("sleeping a bit while we wait") time.sleep(0.1) timeout -= 0.1 if timeout <= 0: raise utils.TestError("timeout waiting for state %s" % expected)
def wait_for_binlog_player_count(self, expected, timeout=30.0): while True: v = utils.get_vars(self.port) if v == None: logging.debug(" vttablet not answering at /debug/vars, waiting...") else: if 'BinlogPlayerMapSize' not in v: logging.debug(" vttablet not exporting BinlogPlayerMapSize, waiting...") else: s = v['BinlogPlayerMapSize'] if s != expected: logging.debug(" vttablet's binlog player map has count %u != %u", s, expected) else: break logging.debug("sleeping a bit while we wait") time.sleep(0.5) timeout -= 0.5 if timeout <= 0: raise utils.TestError("timeout waiting for binlog player count %d" % expected) logging.debug("tablet %s binlog player has %d players", self.tablet_alias, expected)
def wait_for_binlog_server_state(self, expected, timeout=30.0): while True: v = utils.get_vars(self.port) if v == None: logging.debug(" vttablet not answering at /debug/vars, waiting...") else: if 'UpdateStreamState' not in v: logging.debug(" vttablet not exporting BinlogServerState, waiting...") else: s = v['UpdateStreamState'] if s != expected: logging.debug(" vttablet's binlog server in state %s != %s", s, expected) else: break logging.debug("sleeping a bit while we wait") time.sleep(0.5) timeout -= 0.5 if timeout <= 0: raise utils.TestError("timeout waiting for binlog server state %s" % expected) logging.debug("tablet %s binlog service is in state %s", self.tablet_alias, expected)
def test_sigterm(self): utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as it is serving tablet_62344.create_db('vt_test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # start a 'vtctl Sleep' command in the background sp = utils.run_bg(utils.vtroot+'/bin/vtctl -log_dir '+utils.tmp_root+' --alsologtostderr Sleep %s 60s' % tablet_62344.tablet_alias, stdout=PIPE, stderr=PIPE) # wait for it to start, and let's kill it time.sleep(2.0) utils.run(['pkill', 'vtaction']) out, err = sp.communicate() # check the vtctl command got the right remote error back if "vtaction interrupted by signal" not in err: raise utils.TestError("cannot find expected output in error:", err) logging.debug("vtaction was interrupted correctly:\n" + err) tablet_62344.kill_vttablet()
def run_test_zkocc_qps(): _populate_zk() # preload the test_nj cell zkocc_14850 = utils.zkocc_start() qpser = utils.run_bg( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode qps /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2' % utils.zkocc_port_base) time.sleep(10) utils.kill_sub_process(qpser) # get the zkocc vars, make sure we have what we need v = utils.get_vars(utils.zkocc_port_base) if v['ZkReader']['test_nj']['State']['Current'] != 'Connected': raise utils.TestError('invalid zk global state: ', v['ZkReader']['test_nj']['State']['Current']) if v['ZkReader']['test_nj']['State']['DurationConnected'] < 9e9: raise utils.TestError( 'not enough time in Connected state', v['ZkReader']['test_nj']['State']['DurationConnected']) # some checks on performance / stats # a typical workstation will do 15k QPS, check we have more than 3k rpcCalls = v['ZkReader']['RpcCalls'] if rpcCalls < 30000: raise utils.TestError('QPS is too low: %u < 30000', rpcCalls / 10) cacheReads = v['ZkReader']['test_nj']['CacheReads'] if cacheReads < 30000: raise utils.TestError('Cache QPS is too low: %u < 30000', cacheReads / 10) totalCacheReads = v['ZkReader']['total']['CacheReads'] if cacheReads != totalCacheReads: raise utils.TestError('Rollup stats are wrong: %u != %u', cacheReads, totalCacheReads) if v['ZkReader']['UnknownCellErrors'] != 0: raise utils.TestError('unexpected UnknownCellErrors', v['ZkReader']['UnknownCellErrors']) utils.zkocc_kill(zkocc_14850)
def run_test_restart_during_action(): # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') tablet_62344.init_tablet('master', 'test_keyspace', '0') utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() tablet_62344.create_db('vt_test_keyspace') tablet_62344.start_vttablet() utils.run_vtctl('Ping ' + tablet_62344.tablet_alias) # schedule long action utils.run_vtctl('-no-wait Sleep %s 15s' % tablet_62344.tablet_alias, stdout=devnull) # ping blocks until the sleep finishes unless we have a schedule race action_path, _ = utils.run_vtctl('-no-wait Ping ' + tablet_62344.tablet_alias, trap_output=True) # kill agent leaving vtaction running tablet_62344.kill_vttablet() # restart agent tablet_62344.start_vttablet() # we expect this action with a short wait time to fail. this isn't the best # and has some potential for flakiness. utils.run_fail( utils.vtroot + '/bin/vtctl --alsologtostderr -wait-time 2s WaitForAction ' + action_path) # wait until the background sleep action is done, otherwise there will be # a leftover vtaction whose result may overwrite running actions # NOTE(alainjobart): Yes, I've seen it happen, it's a pain to debug: # the zombie Sleep clobbers the Clone command in the following tests utils.run(utils.vtroot + '/bin/vtctl --alsologtostderr -wait-time 20s WaitForAction ' + action_path) # extra small test: we ran for a while, get the states we were in, # make sure they're accounted for properly # first the query engine States v = utils.get_vars(tablet_62344.port) utils.debug("vars: %s" % str(v)) if v['Voltron']['States']['DurationOPEN'] < 10e9: raise utils.TestError('not enough time in Open state', v['Voltron']['States']['DurationOPEN']) # then the Zookeeper connections if v['ZkMetaConn']['test_nj']['Current'] != 'Connected': raise utils.TestError('invalid zk test_nj state: ', v['ZkMetaConn']['test_nj']['Current']) if v['ZkMetaConn']['global']['Current'] != 'Connected': raise utils.TestError('invalid zk global state: ', v['ZkMetaConn']['global']['Current']) if v['ZkMetaConn']['test_nj']['DurationConnected'] < 10e9: raise utils.TestError('not enough time in Connected state', v['ZkMetaConn']['test_nj']['DurationConnected']) if v['tablet-type'] != 'master': raise utils.TestError('tablet-type not exported correctly') tablet_62344.kill_vttablet()
def run_test_reparent_from_outside(): utils.zk_wipe() utils.run_vtctl('CreateKeyspace test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True) # Reparent as a starting point utils.run_vtctl('ReparentShard -force test_keyspace/0 %s' % tablet_62344.tablet_alias) # now manually reparent 1 out of 2 tablets # 62044 will be the new master # 31981 won't be re-parented, so it w2ill be busted tablet_62044.mquery('', [ "RESET MASTER", "STOP SLAVE", "RESET SLAVE", "CHANGE MASTER TO MASTER_HOST = ''", ]) new_pos = tablet_62044.mquery('', 'show master status') utils.debug("New master position: %s" % str(new_pos)) # 62344 will now be a slave of 62044 tablet_62344.mquery('', [ "RESET MASTER", "RESET SLAVE", "change master to master_host='%s', master_port=%u, master_log_file='%s', master_log_pos=%u" % (utils.hostname, tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # 41983 will be a slave of 62044 tablet_41983.mquery('', [ 'stop slave', "change master to master_port=%u, master_log_file='%s', master_log_pos=%u" % (tablet_62044.mysql_port, new_pos[0][0], new_pos[0][1]), 'start slave' ]) # update zk with the new graph utils.run_vtctl('ShardExternallyReparented -scrap-stragglers test_keyspace/0 %s' % tablet_62044.tablet_alias, auto_log=True) # make sure the replication graph is fine shard_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0') utils.debug('shard_files: %s' % " ".join(shard_files)) if shard_files != ['action', 'actionlog', 'test_nj-0000062044']: raise utils.TestError('unexpected zk content: %s' % " ".join(shard_files)) slave_files = utils.zk_ls('/zk/global/vt/keyspaces/test_keyspace/shards/0/test_nj-0000062044') utils.debug('slave_files: %s' % " ".join(slave_files)) if slave_files != ['test_nj-0000041983', 'test_nj-0000062344']: raise utils.TestError('unexpected zk content: %s' % " ".join(slave_files)) tablet_31981.kill_vttablet() tablet_62344.kill_vttablet() tablet_62044.kill_vttablet() tablet_41983.kill_vttablet()
def _check_db_addr(db_addr, expected_addr): # Run in the background to capture output. proc = utils.run_bg(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=WARNING -zk.local-cell=test_nj Resolve ' + db_addr, stdout=PIPE) stdout = proc.communicate()[0].strip() if stdout != expected_addr: raise utils.TestError('wrong zk address', db_addr, stdout, expected_addr)
def run_test_reparent_down_master(): utils.zk_wipe() utils.run_vtctl('CreateKeyspace -force test_keyspace') # create the database so vttablets start, as they are serving tablet_62344.create_db('vt_test_keyspace') tablet_62044.create_db('vt_test_keyspace') tablet_41983.create_db('vt_test_keyspace') tablet_31981.create_db('vt_test_keyspace') # Start up a master mysql and vttablet tablet_62344.init_tablet('master', 'test_keyspace', '0', start=True) # Create a few slaves for testing reparenting. tablet_62044.init_tablet('replica', 'test_keyspace', '0', start=True) tablet_41983.init_tablet('replica', 'test_keyspace', '0', start=True) tablet_31981.init_tablet('replica', 'test_keyspace', '0', start=True) # Recompute the shard layout node - until you do that, it might not be valid. utils.run_vtctl('RebuildShardGraph test_keyspace/0') utils.validate_topology() # Force the slaves to reparent assuming that all the datasets are identical. utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + tablet_62344.tablet_alias, auto_log=True) utils.validate_topology() # Make the master agent and database unavailable. tablet_62344.kill_vttablet() tablet_62344.shutdown_mysql().wait() expected_addr = utils.hostname + ':' + str(tablet_62344.port) _check_db_addr('test_keyspace.0.master:_vtocc', expected_addr) # Perform a reparent operation - the Validate part will try to ping # the master and fail somewhat quickly stdout, stderr = utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=INFO -wait-time 5s ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias) utils.debug("Failed ReparentShard output:\n" + stderr) if 'ValidateShard verification failed: timed out during validate' not in stderr: raise utils.TestError("didn't find the right error strings in failed ReparentShard: " + stderr) # Should timeout and fail stdout, stderr = utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=INFO -wait-time 5s ScrapTablet ' + tablet_62344.tablet_alias) utils.debug("Failed ScrapTablet output:\n" + stderr) if 'deadline exceeded' not in stderr: raise utils.TestError("didn't find the right error strings in failed ScrapTablet: " + stderr) # Should interrupt and fail sp = utils.run_bg(utils.vtroot+'/bin/vtctl -log.level=INFO -wait-time 10s ScrapTablet ' + tablet_62344.tablet_alias, stdout=PIPE, stderr=PIPE) # Need time for the process to start before killing it. time.sleep(0.1) os.kill(sp.pid, signal.SIGINT) stdout, stderr = sp.communicate() utils.debug("Failed ScrapTablet output:\n" + stderr) if 'interrupted' not in stderr: raise utils.TestError("didn't find the right error strings in failed ScrapTablet: " + stderr) # Force the scrap action in zk even though tablet is not accessible. tablet_62344.scrap(force=True) utils.run_fail(utils.vtroot+'/bin/vtctl -logfile=/dev/null -log.level=WARNING ChangeSlaveType -force %s idle' % tablet_62344.tablet_alias) # Remove pending locks (make this the force option to ReparentShard?) utils.run_vtctl('PurgeActions /zk/global/vt/keyspaces/test_keyspace/shards/0/action') # Re-run reparent operation, this shoud now proceed unimpeded. utils.run_vtctl('-wait-time 1m ReparentShard test_keyspace/0 ' + tablet_62044.tablet_alias, auto_log=True) utils.validate_topology() expected_addr = utils.hostname + ':' + str(tablet_62044.port) _check_db_addr('test_keyspace.0.master:_vtocc', expected_addr) utils.run_vtctl('ChangeSlaveType -force %s idle' % tablet_62344.tablet_alias) idle_tablets, _ = utils.run_vtctl('ListAllTablets test_nj', trap_output=True) if '0000062344 <null> <null> idle' not in idle_tablets: raise utils.TestError('idle tablet not found', idle_tablets) tablet_62044.kill_vttablet() tablet_41983.kill_vttablet() tablet_31981.kill_vttablet() # sothe other tests don't have any surprise tablet_62344.start_mysql().wait()
def test_multisnapshot_and_restore(): tables = ['vt_insert_test', 'vt_insert_test1'] create_template = '''create table %s ( id bigint auto_increment, msg varchar(64), primary key (id), index by_msg (msg) ) Engine=InnoDB''' create_view = '''create view vt_insert_view(id, msg) as select id, msg from vt_insert_test''' insert_template = "insert into %s (id, msg) values (%s, 'test %s')" utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force test_keyspace') # Start three tablets for three different shards. At this point the # sharding schema is not really important, as long as it is # consistent. new_spec = '-0000000000000028-' old_tablets = [tablet_62044, tablet_41983, tablet_31981] for i, tablet in enumerate(old_tablets): tablet.init_tablet('master', 'test_keyspace', str(i)) utils.run_vtctl('RebuildShardGraph test_keyspace/%s' % i) utils.validate_topology() for i, tablet in enumerate(old_tablets): tablet.populate( "vt_test_keyspace", [create_template % table for table in tables] + [create_view], sum([[insert_template % (table, 10*j + i, 10*j + i) for j in range(1, 8)] for table in tables], [])) tablet.start_vttablet() utils.run_vtctl('MultiSnapshot --force --spec=%s %s id' % (new_spec, tablet.tablet_alias), trap_output=True) utils.pause("After snapshot") # try to get the schema on the source, make sure the view is there out, err = utils.run_vtctl('GetSchema --include-views ' + tablet_62044.tablet_alias, log_level='INFO', trap_output=True) if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err: raise utils.TestError('Unexpected GetSchema --include-views output: %s' % err) out, err = utils.run_vtctl('GetSchema ' + tablet_62044.tablet_alias, log_level='INFO', trap_output=True) if 'vt_insert_view' in err: raise utils.TestError('Unexpected GetSchema output: %s' % err) utils.run_vtctl('CreateKeyspace -force test_keyspace_new') tablet_62344.init_tablet('master', 'test_keyspace_new', "0", dbname="not_vt_test_keyspace") utils.run_vtctl('RebuildShardGraph test_keyspace_new/0') utils.validate_topology() tablet_62344.mquery('', 'DROP DATABASE IF EXISTS not_vt_test_keyspace') tablet_62344.start_vttablet(wait_for_state='CONNECTING') # db not created tablet_urls = ' '.join("vttp://localhost:%s/vt_test_keyspace" % tablet.port for tablet in old_tablets) # 0x28 = 40 err = tablet_62344.mysqlctl("multirestore --end=0000000000000028 -strategy=skipAutoIncrement(vt_insert_test1),delayPrimaryKey,delaySecondaryIndexes,useMyIsam,populateBlpRecovery(6514) not_vt_test_keyspace %s" % tablet_urls).wait() if err != 0: raise utils.TestError("mysqlctl failed: %u" % err) for table in tables: rows = tablet_62344.mquery('not_vt_test_keyspace', 'select id from %s' % table) if len(rows) == 0: raise utils.TestError("There are no rows in the restored database.") for row in rows: if row[0] > 32: raise utils.TestError("Bad row: %s" % row) rows = tablet_62344.mquery('_vt', 'select * from blp_checkpoint') if len(rows) != 3: raise utils.TestError("Was expecting 3 rows in blp_checkpoint but got: %s" % str(rows)) # try to get the schema on multi-restored guy, make sure the view is there out, err = utils.run_vtctl('GetSchema --include-views ' + tablet_62344.tablet_alias, log_level='INFO', trap_output=True) if 'vt_insert_view' not in err or 'VIEW `{{.DatabaseName}}`.`vt_insert_view` AS select' not in err: raise utils.TestError('Unexpected GetSchema --include-views output after multirestore: %s' % err) for tablet in tablet_62044, tablet_41983, tablet_31981, tablet_62344: tablet.kill_vttablet()
def run_test_sharding(): utils.run_vtctl('CreateKeyspace test_keyspace') shard_0_master.init_tablet('master', 'test_keyspace', '-80') shard_0_replica.init_tablet('replica', 'test_keyspace', '-80') shard_1_master.init_tablet('master', 'test_keyspace', '80-') shard_1_replica.init_tablet('replica', 'test_keyspace', '80-') utils.run_vtctl( 'RebuildShardGraph /zk/global/vt/keyspaces/test_keyspace/shards/*', auto_log=True) utils.run_vtctl('RebuildKeyspaceGraph /zk/global/vt/keyspaces/*', auto_log=True) # run checks now before we start the tablets utils.validate_topology() # create databases shard_0_master.create_db('vt_test_keyspace') shard_0_replica.create_db('vt_test_keyspace') shard_1_master.create_db('vt_test_keyspace') shard_1_replica.create_db('vt_test_keyspace') # start the tablets shard_0_master.start_vttablet() shard_0_replica.start_vttablet() shard_1_master.start_vttablet() shard_1_replica.start_vttablet() # apply the schema on the first shard through vtctl, so all tablets # are the same (replication is not enabled yet, so allow_replication=false # is just there to be tested) utils.run_vtctl([ 'ApplySchema', '-stop-replication', '-sql=' + create_vt_select_test.replace("\n", ""), shard_0_master.tablet_alias ]) utils.run_vtctl([ 'ApplySchema', '-stop-replication', '-sql=' + create_vt_select_test.replace("\n", ""), shard_0_replica.tablet_alias ]) # start zkocc, we'll use it later zkocc_server = utils.zkocc_start() utils.run_vtctl('ReparentShard -force test_keyspace/-80 ' + shard_0_master.tablet_alias, auto_log=True) utils.run_vtctl('ReparentShard -force test_keyspace/80- ' + shard_1_master.tablet_alias, auto_log=True) # apply the schema on the second shard using a simple schema upgrade utils.run_vtctl([ 'ApplySchemaShard', '-simple', '-sql=' + create_vt_select_test_reverse.replace("\n", ""), 'test_keyspace/80-' ]) # insert some values directly (db is RO after minority reparent) # FIXME(alainjobart) these values don't match the shard map utils.run_vtctl('SetReadWrite ' + shard_0_master.tablet_alias) utils.run_vtctl('SetReadWrite ' + shard_1_master.tablet_alias) shard_0_master.mquery( 'vt_test_keyspace', "insert into vt_select_test (id, msg) values (1, 'test 1')", write=True) shard_1_master.mquery( 'vt_test_keyspace', "insert into vt_select_test (id, msg) values (10, 'test 10')", write=True) utils.validate_topology(ping_tablets=True) utils.pause("Before the sql scatter query") # note the order of the rows is not guaranteed, as the go routines # doing the work can go out of order check_rows(["Index\tid\tmsg", "1\ttest 1", "10\ttest 10"]) # write a value, re-read them all utils.vtclient2( 3803, "/test_nj/test_keyspace/master", "insert into vt_select_test (id, msg) values (:keyspace_id, 'test 2')", bindvars='{"keyspace_id": 2}', driver="vtdb", verbose=True) check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"]) # make sure the '2' value was written on first shard rows = shard_0_master.mquery( 'vt_test_keyspace', "select id, msg from vt_select_test order by id") if (len(rows) != 2 or \ rows[0][0] != 1 or \ rows[1][0] != 2): print "mysql_query returned:", rows raise utils.TestError('wrong mysql_query output') utils.pause("After db writes") # now use zkocc or streaming or both for the same query check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc") check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-streaming") check_rows(["Index\tid\tmsg", "1\ttest 1", "2\ttest 2", "10\ttest 10"], driver="vtdb-zkocc-streaming") # make sure the schema checking works check_rows_schema_diff("vtdb-zkocc") check_rows_schema_diff("vtdb") # throw in some schema validation step # we created the schema differently, so it should show utils.run_vtctl('ValidateSchemaShard test_keyspace/-80') utils.run_vtctl('ValidateSchemaShard test_keyspace/80-') out, err = utils.run_vtctl('ValidateSchemaKeyspace test_keyspace', trap_output=True, raise_on_error=False) if "test_nj-0000062344 and test_nj-0000062346 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err or \ "test_nj-0000062344 and test_nj-0000062347 disagree on schema for table vt_select_test:\nCREATE TABLE" not in err: raise utils.TestError('wrong ValidateSchemaKeyspace output: ' + err) # validate versions utils.run_vtctl('ValidateVersionShard test_keyspace/-80', auto_log=True) utils.run_vtctl('ValidateVersionKeyspace test_keyspace', auto_log=True) # show and validate permissions utils.run_vtctl('GetPermissions test_nj-0000062344', auto_log=True) utils.run_vtctl('ValidatePermissionsShard test_keyspace/-80', auto_log=True) utils.run_vtctl('ValidatePermissionsKeyspace test_keyspace', auto_log=True) # and create zkns on this complex keyspace, make sure a few files are created utils.run_vtctl('ExportZknsForKeyspace test_keyspace') out, err = utils.run(utils.vtroot + '/bin/zk ls -R /zk/test_nj/zk?s/vt/test_keysp*', trap_output=True) lines = out.splitlines() for base in ['-80', '80-']: for db_type in ['master', 'replica']: for sub_path in ['', '.vdns', '/0', '/_vtocc.vdns']: expected = '/zk/test_nj/zkns/vt/test_keyspace/' + base + '/' + db_type + sub_path if expected not in lines: raise utils.TestError('missing zkns part:\n%s\nin:%s' % (expected, out)) # now try to connect using the python client and shard-aware connection # to both shards # first get the topology and check it zkocc_client = zkocc.ZkOccConnection( "localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0) topology.read_keyspaces(zkocc_client) shard_0_master_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.-80.master:_vtocc") if len(shard_0_master_addrs) != 1: raise utils.TestError( 'topology.get_host_port_by_name failed for "test_keyspace.-80.master:_vtocc", got: %s' % " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) utils.debug("shard 0 master addrs: %s" % " ".join( ["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])) # connect to shard -80 conn = tablet2.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "test_keyspace", "-80", 10.0) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {}) if (len(results) != 2 or \ results[0][0] != 1 or \ results[1][0] != 2): print "conn._execute returned:", results raise utils.TestError('wrong conn._execute output') # connect to shard 80- shard_1_master_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.80-.master:_vtocc") conn = tablet2.TabletConnection( "%s:%u" % (shard_1_master_addrs[0][0], shard_1_master_addrs[0][1]), "test_keyspace", "80-", 10.0) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute("select id, msg from vt_select_test order by id", {}) if (len(results) != 1 or \ results[0][0] != 10): print "conn._execute returned:", results raise utils.TestError('wrong conn._execute output') # try to connect with bad shard try: conn = tablet2.TabletConnection("localhost:%u" % shard_0_master.port, "test_keyspace", "-90", 10.0) conn.dial() raise utils.TestError('expected an exception') except Exception as e: if "fatal: Shard mismatch, expecting -80, received -90" not in str(e): raise utils.TestError('unexpected exception: ' + str(e)) utils.kill_sub_process(zkocc_server) shard_0_master.kill_vttablet() shard_0_replica.kill_vttablet() shard_1_master.kill_vttablet() shard_1_replica.kill_vttablet()
def _run_test_vtctl_snapshot_restore(server_mode): if server_mode: snapshot_flags = '-server-mode -concurrency=8' restore_flags = '-dont-wait-for-slave-start' else: snapshot_flags = '-concurrency=4' restore_flags = '' utils.zk_wipe() # Start up a master mysql and vttablet utils.run_vtctl('CreateKeyspace -force snapshot_test') tablet_62344.init_tablet('master', 'snapshot_test', '0') utils.run_vtctl('RebuildShardGraph snapshot_test/0') utils.validate_topology() tablet_62344.populate('vt_snapshot_test', create_vt_insert_test, populate_vt_insert_test) tablet_62044.create_db('vt_snapshot_test') tablet_62344.start_vttablet() # Need to force snapshot since this is a master db. out, err = utils.run_vtctl('Snapshot -force %s %s ' % (snapshot_flags, tablet_62344.tablet_alias), log_level='INFO', trap_output=True) results = {} for name in ['Manifest', 'ParentAlias', 'SlaveStartRequired', 'ReadOnly', 'OriginalType']: sepPos = err.find(name + ": ") if sepPos != -1: results[name] = err[sepPos+len(name)+2:].splitlines()[0] if "Manifest" not in results: raise utils.TestError("Snapshot didn't echo Manifest file", err) if "ParentAlias" not in results: raise utils.TestError("Snapshot didn't echo ParentAlias", err) utils.pause("snapshot finished: " + results['Manifest'] + " " + results['ParentAlias']) if server_mode: if "SlaveStartRequired" not in results: raise utils.TestError("Snapshot didn't echo SlaveStartRequired", err) if "ReadOnly" not in results: raise utils.TestError("Snapshot didn't echo ReadOnly", err) if "OriginalType" not in results: raise utils.TestError("Snapshot didn't echo OriginalType", err) if (results['SlaveStartRequired'] != 'false' or results['ReadOnly'] != 'true' or results['OriginalType'] != 'master'): raise utils.TestError("Bad values returned by Snapshot", err) tablet_62044.init_tablet('idle', start=True) # do not specify a MANIFEST, see if 'default' works call(["touch", "/tmp/vtSimulateFetchFailures"]) utils.run_vtctl('Restore -fetch-concurrency=2 -fetch-retry-count=4 %s %s default %s %s' % (restore_flags, tablet_62344.tablet_alias, tablet_62044.tablet_alias, results['ParentAlias']), auto_log=True) utils.pause("restore finished") tablet_62044.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) utils.validate_topology() # in server_mode, get the server out of it and check it if server_mode: utils.run_vtctl('SnapshotSourceEnd %s %s' % (tablet_62344.tablet_alias, results['OriginalType']), auto_log=True) tablet_62344.assert_table_count('vt_snapshot_test', 'vt_insert_test', 4) utils.validate_topology() tablet_62344.kill_vttablet() tablet_62044.kill_vttablet()
def test_zkocc(self): # preload the test_nj cell zkocc_14850 = utils.zkocc_start( extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s']) time.sleep(1) # create a python client. The first address is bad, will test the retry logic bad_port = utils.reserve_ports(3) zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30) zkocc_client.dial() # test failure for a python client that cannot connect bad_zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30) try: bad_zkocc_client.dial() raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.ERROR) # FIXME(ryszard): This can be changed into a self.assertRaises. try: bad_zkocc_client.get("/zk/test_nj/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.WARNING) # get test utils.prog_compile(['zkclient2']) out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' % utils.zkocc_port_base, trap_output=True) self.assertEqual( err, "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n" ) zk_data = zkocc_client.get("/zk/test_nj/zkocc1/data1") self.assertDictContainsSubset( { 'Data': "Test data 1", 'Cached': True, 'Stale': False, }, zk_data) self.assertDictContainsSubset({ 'NumChildren': 0, 'Version': 0 }, zk_data['Stat']) # getv test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized( err, """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false) [1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false) [2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false) """) zk_data = zkocc_client.getv([ "/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2", "/zk/test_nj/zkocc1/data3" ])['Nodes'] self.assertEqual(len(zk_data), 3) for i, d in enumerate(zk_data): self.assertEqual(d['Data'], 'Test data %s' % (i + 1)) self.assertTrue(d['Cached']) self.assertFalse(d['Stale']) self.assertDictContainsSubset({ 'NumChildren': 0, 'Version': 0 }, d['Stat']) # children test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' % utils.zkocc_port_base, trap_output=True) self.assertEqualNormalized( err, """Path = /zk/test_nj Child[0] = zkocc1 Child[1] = zkocc2 NumChildren = 2 CVersion = 4 Cached = false Stale = false """) # zk command tests self._check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1") self._check_zk_output( "ls -l /zk/test_nj/zkocc1", """total: 3 -rw-rw-rw- zk zk 11 %s data1 -rw-rw-rw- zk zk 11 %s data2 -rw-rw-rw- zk zk 11 %s data3 """ % (_format_time(zk_data[0]['Stat']['MTime']), _format_time(zk_data[1]['Stat']['MTime']), _format_time(zk_data[2]['Stat']['MTime']))) # test /zk/local is not resolved and rejected out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False) self.assertIn("zkocc: cannot resolve local cell", err) # start a background process to query the same value over and over again # while we kill the zk server and restart it outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False) filename = outfd.name querier = utils.run_bg( '/bin/bash -c "while true ; do ' + utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file) outfd.close() time.sleep(1) # kill zk server, sleep a bit, restart zk server, sleep a bit utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.run(utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.kill_sub_process(querier) utils.debug("Checking " + filename) fd = open(filename, "r") state = 0 for line in fd: if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n": stale = False elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n": stale = True else: raise utils.TestError('unexpected line: ', line) if state == 0: if stale: state = 1 elif state == 1: if not stale: state = 2 else: if stale: self.fail('unexpected stale state') self.assertEqual(state, 2) fd.close() utils.zkocc_kill(zkocc_14850) # check that after the server is gone, the python client fails correctly logging.getLogger().setLevel(logging.ERROR) try: zkocc_client.get("/zk/test_nj/zkocc1/data1") self.fail('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise logging.getLogger().setLevel(logging.WARNING)
def assert_table_count(self, dbname, table, n, where=''): result = self.mquery(dbname, 'select count(*) from ' + table + ' ' + where) if result[0][0] != n: raise utils.TestError('expected %d rows in %s' % (n, table), result)
def openssl(cmd): result = subprocess.call(["openssl"] + cmd) if result != 0: raise utils.TestError("OpenSSL command failed: %s" % " ".join(cmd))
def check_db_status(self, name, value): row = self.get_db_status(name) if row[1] != value: raise utils.TestError('status not correct', name, row)
def check_db_var(self, name, value): row = self.get_db_var(name) if row != (name, value): raise utils.TestError('variable not set correctly', name, row)
def run_test_zkocc(): _populate_zk() # preload the test_nj cell zkocc_14850 = utils.zkocc_start( extra_params=['-connect-timeout=2s', '-cache-refresh-interval=1s']) time.sleep(1) # create a python client. The first address is bad, will test the retry logic bad_port = utils.reserve_ports(3) zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u,localhost:%u" % (bad_port, utils.zkocc_port_base, bad_port + 1), "test_nj", 30) zkocc_client.dial() # test failure for a python client that cannot connect bad_zkocc_client = zkocc.ZkOccConnection( "localhost:%u,localhost:%u" % (bad_port + 2, bad_port), "test_nj", 30) try: bad_zkocc_client.dial() raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise utils.TestError('Unexpected exception: ', str(e)) logging.getLogger().setLevel(logging.ERROR) try: bad_zkocc_client.get("/zk/test_nj/zkocc1/data1") raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise utils.TestError('Unexpected exception: ', str(e)) logging.getLogger().setLevel(logging.WARNING) # get test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1' % utils.zkocc_port_base, trap_output=True) if err != "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=false, Stale=false)\n": raise utils.TestError('unexpected get output: ', err) zkNode = zkocc_client.get("/zk/test_nj/zkocc1/data1") if (zkNode['Data'] != "Test data 1" or \ zkNode['Stat']['NumChildren'] != 0 or \ zkNode['Stat']['Version'] != 0 or \ zkNode['Cached'] != True or \ zkNode['Stale'] != False): raise utils.TestError('unexpected zkocc_client.get output: ', zkNode) # getv test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 /zk/test_nj/zkocc1/data2 /zk/test_nj/zkocc1/data3' % utils.zkocc_port_base, trap_output=True) if err != """[0] /zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false) [1] /zk/test_nj/zkocc1/data2 = Test data 2 (NumChildren=0, Version=0, Cached=false, Stale=false) [2] /zk/test_nj/zkocc1/data3 = Test data 3 (NumChildren=0, Version=0, Cached=false, Stale=false) """: raise utils.TestError('unexpected getV output: ', err) zkNodes = zkocc_client.getv([ "/zk/test_nj/zkocc1/data1", "/zk/test_nj/zkocc1/data2", "/zk/test_nj/zkocc1/data3" ]) if (zkNodes['Nodes'][0]['Data'] != "Test data 1" or \ zkNodes['Nodes'][0]['Stat']['NumChildren'] != 0 or \ zkNodes['Nodes'][0]['Stat']['Version'] != 0 or \ zkNodes['Nodes'][0]['Cached'] != True or \ zkNodes['Nodes'][0]['Stale'] != False or \ zkNodes['Nodes'][1]['Data'] != "Test data 2" or \ zkNodes['Nodes'][1]['Stat']['NumChildren'] != 0 or \ zkNodes['Nodes'][1]['Stat']['Version'] != 0 or \ zkNodes['Nodes'][1]['Cached'] != True or \ zkNodes['Nodes'][1]['Stale'] != False or \ zkNodes['Nodes'][2]['Data'] != "Test data 3" or \ zkNodes['Nodes'][2]['Stat']['NumChildren'] != 0 or \ zkNodes['Nodes'][2]['Stat']['Version'] != 0 or \ zkNodes['Nodes'][2]['Cached'] != True or \ zkNodes['Nodes'][2]['Stale'] != False): raise utils.TestError('unexpected zkocc_client.getv output: ', zkNodes) # children test out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u -mode children /zk/test_nj' % utils.zkocc_port_base, trap_output=True) if err != """Path = /zk/test_nj Child[0] = zkocc1 Child[1] = zkocc2 NumChildren = 2 CVersion = 4 Cached = false Stale = false """: raise utils.TestError('unexpected children output: ', err) # zk command tests _check_zk_output("cat /zk/test_nj/zkocc1/data1", "Test data 1") _check_zk_output( "ls -l /zk/test_nj/zkocc1", """total: 3 -rw-rw-rw- zk zk 11 %s data1 -rw-rw-rw- zk zk 11 %s data2 -rw-rw-rw- zk zk 11 %s data3 """ % (_format_time(zkNodes['Nodes'][0]['Stat']['MTime']), _format_time(zkNodes['Nodes'][1]['Stat']['MTime']), _format_time(zkNodes['Nodes'][2]['Stat']['MTime']))) # test /zk/local is not resolved and rejected out, err = utils.run( utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/local/zkocc1/data1' % utils.zkocc_port_base, trap_output=True, raise_on_error=False) if "zkocc: cannot resolve local cell" not in err: raise utils.TestError('unexpected get output, not local cell error: ', err) # start a background process to query the same value over and over again # while we kill the zk server and restart it outfd = tempfile.NamedTemporaryFile(dir=utils.tmp_root, delete=False) filename = outfd.name querier = utils.run_bg( '/bin/bash -c "while true ; do ' + utils.vtroot + '/bin/zkclient2 -server localhost:%u /zk/test_nj/zkocc1/data1 ; sleep 0.1 ; done"' % utils.zkocc_port_base, stderr=outfd.file) outfd.close() time.sleep(1) # kill zk server, sleep a bit, restart zk server, sleep a bit utils.run( utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u shutdown' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.run( utils.vtroot + '/bin/zkctl -zk.cfg 1@' + utils.hostname + ':%u:%u:%u start' % (utils.zk_port_base, utils.zk_port_base + 1, utils.zk_port_base + 2)) time.sleep(3) utils.kill_sub_process(querier) utils.debug("Checking " + filename) fd = open(filename, "r") state = 0 for line in fd: if line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=false)\n": stale = False elif line == "/zk/test_nj/zkocc1/data1 = Test data 1 (NumChildren=0, Version=0, Cached=true, Stale=true)\n": stale = True else: raise utils.TestError('unexpected line: ', line) if state == 0: if stale: state = 1 elif state == 1: if not stale: state = 2 else: if stale: raise utils.TestError('unexpected stale state') if state != 2: raise utils.TestError('unexpected ended stale state') fd.close() utils.zkocc_kill(zkocc_14850) # check that after the server is gone, the python client fails correctly logging.getLogger().setLevel(logging.ERROR) try: zkocc_client.get("/zk/test_nj/zkocc1/data1") raise utils.TestError('exception expected') except zkocc.ZkOccError as e: if str(e) != "Cannot dial to any server": raise utils.TestError('Unexpected exception: ', str(e)) logging.getLogger().setLevel(logging.WARNING)
def check_vttablet_count(cls): if Tablet.tablets_running > 0: raise utils.TestError('This test is not killing all its vttablets')
def run_test_secure(): zkocc_server = utils.zkocc_start() # start the tablets shard_0_master.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem") shard_0_slave.start_vttablet(cert=cert_dir + "/vt-server-cert.pem", key=cert_dir + "/vt-server-key.pem", repl_extra_flags={ 'flags': 2048, 'ssl_ca': cert_dir + "/ca-cert.pem", 'ssl_cert': cert_dir + "/client-cert.pem", 'ssl_key': cert_dir + "/client-key.pem", }) # Reparent using SSL for t in [shard_0_master, shard_0_slave]: t.reset_replication() utils.run_vtctl('ReparentShard -force test_keyspace/0 ' + shard_0_master.tablet_alias, auto_log=True) # then get the topology and check it zkocc_client = zkocc.ZkOccConnection( "localhost:%u" % utils.zkocc_port_base, "test_nj", 30.0) topology.read_keyspaces(zkocc_client) shard_0_master_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.0.master:_vts") if len(shard_0_master_addrs) != 1: raise utils.TestError( 'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts", got: %s' % " ".join([ "%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs ])) if shard_0_master_addrs[0][2] != True: raise utils.TestError( 'topology.get_host_port_by_name failed for "test_keyspace.0.master:_vts" is not encrypted' ) utils.debug("shard 0 master addrs: %s" % " ".join( ["%s:%u(%s)" % (h, p, str(e)) for (h, p, e) in shard_0_master_addrs])) # make sure asking for optionally secure connections works too auto_addrs = topology.get_host_port_by_name( zkocc_client, "test_keyspace.0.master:_vtocc", encrypted=True) if auto_addrs != shard_0_master_addrs: raise utils.TestError( 'topology.get_host_port_by_name doesn\'t resolve encrypted addresses properly: %s != %s' % (str(shard_0_master_addrs), str(auto_addrs))) # try to connect with regular client try: conn = tablet3.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "test_keyspace", "0", 10.0) conn.dial() raise utils.TestError("No exception raised to secure port") except tablet3.FatalError as e: if not e.args[0][0].startswith('Unexpected EOF in handshake to'): raise utils.TestError("Unexpected exception: %s" % str(e)) sconn = utils.get_vars(shard_0_master.port)["SecureConns"] if sconn != 0: raise utils.TestError("unexpected conns %s" % sconn) # connect to encrypted port conn = tablet3.TabletConnection( "%s:%u" % (shard_0_master_addrs[0][0], shard_0_master_addrs[0][1]), "test_keyspace", "0", 5.0, encrypted=True) conn.dial() (results, rowcount, lastrowid, fields) = conn._execute("select 1 from dual", {}) if (len(results) != 1 or \ results[0][0] != 1): print "conn._execute returned:", results raise utils.TestError('wrong conn._execute output') sconn = utils.get_vars(shard_0_master.port)["SecureConns"] if sconn != 1: raise utils.TestError("unexpected conns %s" % sconn) saccept = utils.get_vars(shard_0_master.port)["SecureAccepts"] if saccept == 0: raise utils.TestError("unexpected accepts %s" % saccept) # trigger a time out on a secure connection, see what exception we get try: conn._execute("select sleep(100) from dual", {}) raise utils.TestError("No timeout exception") except tablet3.TimeoutError as e: utils.debug("Got the right exception for SSL timeout: %s" % str(e)) # kill everything utils.kill_sub_process(zkocc_server)