def test_workload(self):
alpha = self.getPrimaryForShard(0)
beta = self.getReplicaForShard(0)
workload_ports = workload_runner.RDBPorts(host=alpha.host,
http_port=alpha.http_port,
rdb_port=alpha.driver_port,
db_name=self.dbName,
table_name=self.tableName)
with workload_runner.SplitOrContinuousWorkload(
opts, workload_ports) as workload:
utils.print_with_time('Workloads:\n%s' %
pprint.pformat(workload.opts))
utils.print_with_time("Running before workload")
workload.run_before()
utils.print_with_time("Before workload complete")
self.checkCluster()
workload.check()
utils.print_with_time("Demoting primary")
shardConfig = self.table.config()['shards'].run(self.conn)
shardConfig[0]['primary_replica'] = beta.name
self.table.config().update({'shards': shardConfig}).run(self.conn)
self.table.wait(wait_for='all_replicas_ready').run(self.conn)
self.checkCluster()
utils.print_with_time("Running after workload")
workload.run_after()
self.checkCluster()
utils.print_with_time("After workload complete")
def get_workload_ports(processes, tableName, dbName='test'):
for process in processes:
assert isinstance(process, (driver.Process, driver.ProxyProcess))
process = random.choice(processes)
return workload_runner.RDBPorts(
host = process.host,
http_port = process.http_port,
rdb_port = process.driver_port,
table_name = tableName,
db_name = dbName)
def get_workload_ports(table, processes):
for process in processes:
assert isinstance(process, (driver.Process, driver.ProxyProcess))
process = random.choice(processes)
return workload_runner.RDBPorts(
host = "localhost",
http_port = process.http_port,
rdb_port = process.driver_port,
table_name = table.name,
db_name = "test")
def get_workload_ports(parsed_opts, namespace, processes):
for process in processes:
assert isinstance(process, (driver.Process, driver.ProxyProcess))
process = random.choice(processes)
assert namespace.protocol == parsed_opts["protocol"]
if parsed_opts["protocol"] == "memcached":
return workload_runner.MemcachedPorts(host="localhost",
http_port=process.http_port,
memcached_port=namespace.port +
process.port_offset)
else:
return workload_runner.RDBPorts(host="localhost",
http_port=process.http_port,
rdb_port=28015 + process.port_offset,
table_name=namespace.name,
db_name="test")
def test_kill_secondary(self):
primary = self.getPrimaryForShard(0)
secondary = self.getReplicaForShard(0)
conn = self.r.connect(host=primary.host, port=primary.driver_port)
issues = list(self.r.db('rethinkdb').table('current_issues').run(conn))
self.assertEqual(issues, [], 'The issues list was not empty: %r' % issues)
workload_ports = workload_runner.RDBPorts(host=primary.host, http_port=primary.http_port, rdb_port=primary.driver_port, db_name=self.dbName, table_name=self.tableName)
with workload_runner.SplitOrContinuousWorkload(opts, workload_ports) as workload:
print_with_time("Starting workload")
workload.run_before()
self.cluster.check()
issues = list(self.r.db('rethinkdb').table('current_issues').run(conn))
self.assertEqual(issues, [], 'The issues list was not empty: %r' % issues)
print_with_time("Killing the secondary")
secondary.kill()
print_with_time("Checking that the table_availability issue shows up")
deadline = time.time() + 5
last_error = None
while time.time() < deadline:
try:
issues = list(self.r.db('rethinkdb').table('current_issues').filter({'type':'table_availability', 'info':{'db':self.dbName, 'table':self.tableName}}).run(conn))
self.assertEqual(len(issues), 1, 'The server did not record the single issue for the killed secondary server:\n%s' % pformat(issues))
issue = issues[0]
self.assertEqual(issue['critical'], False)
self.assertEqual(issue['info']['status']['ready_for_reads'], True)
self.assertEqual(issue['info']['status']['ready_for_writes'], True)
break
except Exception as e:
last_error = e
time.sleep(.2)
else:
raise last_error
print_with_time("Running after workload")
workload.run_after()
print_with_time("Done")
r.db(dbName).table_create(tableName).run(conn)
print("Pinning database to only the first server (%.2fs)" % (time.time() - startTime))
assert r.db(dbName).table(tableName).config() \
.update({'shards':
[{'primary_replica':database_server.name, 'replicas':[database_server.name]}]
}).run(conn)['errors'] == 0
r.db(dbName).wait().run(conn)
cluster.check()
assert [] == list(r.db('rethinkdb').table('current_issues').run(conn))
sys.stderr.write('before workoad: %s\n' % (repr(list(r.db(dbName).table(tableName).config().run(conn)))))
print("Starting workload (%.2fs)" % (time.time() - startTime))
workload_ports = workload_runner.RDBPorts(host=database_server.host, http_port=database_server.http_port, rdb_port=database_server.driver_port, db_name=dbName, table_name=tableName)
with workload_runner.ContinuousWorkload(opts["workload"], workload_ports) as workload:
workload.start()
print("Running workload for 10 seconds (%.2fs)" % (time.time() - startTime))
time.sleep(10)
cluster.check()
assert [] == list(r.db('rethinkdb').table('current_issues').run(conn))
print("Killing the access server (%.2fs)" % (time.time() - startTime))
access_server.kill()
issues = list(r.db('rethinkdb').table('current_issues').run(conn))
assert len(issues) == 0, 'Issues were raised when server stopped: %s' % repr(issues)
# Don't bother stopping the workload, just exit and it will get killed
assert r.db(dbName).table(tableName).config() \
.update({'shards':[
{'primary_replica':primary.name, 'replicas':[primary.name, replicaPool[0].name]}
]}).run(conn)['errors'] == 0
r.db(dbName).wait().run(conn)
cluster.check()
issues = list(r.db('rethinkdb').table('current_issues').run(conn))
assert len(
issues) == 0, 'There were issues on the server: %s' % str(issues)
print('Starting workload (%.2fs)' % (time.time() - startTime))
workload_ports = workload_runner.RDBPorts(host=primary.host,
http_port=primary.http_port,
rdb_port=primary.driver_port,
db_name=dbName,
table_name=tableName)
with workload_runner.SplitOrContinuousWorkload(opts,
workload_ports) as workload:
workload.run_before()
cluster.check()
assert list(r.db('rethinkdb').table('current_issues').run(conn)) == []
workload.check()
current = opts["sequence"].initial
for i, s in enumerate(opts["sequence"].steps):
if i != 0:
workload.run_between()
_, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
r = utils.import_python_driver()
dbName, tableName = utils.get_test_db_table()
utils.print_with_time("Starting cluster with one server")
with driver.Cluster(initial_servers=['first'],
output_folder='.',
command_prefix=command_prefix,
extra_options=serve_options,
wait_until_ready=True) as cluster:
server1 = cluster[0]
workload_ports1 = workload_runner.RDBPorts(host=server1.host,
http_port=server1.http_port,
rdb_port=server1.driver_port,
db_name=dbName,
table_name=tableName)
utils.print_with_time("Establishing ReQL connection")
conn1 = r.connect(server1.host, server1.driver_port)
utils.print_with_time("Creating db/table %s/%s" % (dbName, tableName))
if dbName not in r.db_list().run(conn1):
r.db_create(dbName).run(conn1)
if tableName in r.db(dbName).table_list().run(conn1):
r.db(dbName).table_drop(tableName).run(conn1)
r.db(dbName).table_create(tableName).run(conn1)
def test_failover(self):
'''Run a workload while killing a server to cause a failover to a secondary'''
# - setup
primary = self.getPrimaryForShard(0)
stable = self.getReplicaForShard(0)
stableConn = self.r.connect(host=stable.host, port=stable.driver_port)
workload_ports = workload_runner.RDBPorts(host=stable.host,
http_port=stable.http_port,
rdb_port=stable.driver_port,
db_name=dbName,
table_name=tableName)
# - run test
with workload_runner.SplitOrContinuousWorkload(
opts, workload_ports) as workload:
print_with_time("Starting workload before")
workload.run_before()
self.cluster.check()
issues = list(
self.r.db('rethinkdb').table('current_issues').run(stableConn))
self.assertEqual(
issues, [],
'The server recorded the following issues after the run_before:\n%s'
% pformat(issues))
print_with_time("Shutting down the primary")
primary.close()
print_with_time(
"Checking that the table_availability issue shows up")
deadline = time.time() + 5
last_error = None
while time.time() < deadline:
try:
issues = list(
self.r.db('rethinkdb').table('current_issues').filter({
'type':
'table_availability',
'info': {
'db': dbName,
'table': tableName
}
}).run(stableConn))
self.assertEqual(
len(issues), 1,
'The server did not record the single issue for the killed server:\n%s'
% pformat(issues))
break
except Exception as e:
last_error = e
time.sleep(.2)
else:
raise last_error
print_with_time("Waiting for the table to become available again")
timeout = 30
try:
self.table.wait(wait_for='ready_for_writes',
timeout=timeout).run(stableConn)
except self.r.ReqlRuntimeError as e:
raise AssertionError(
'Table did not become available after %d seconds.' %
timeout)
print_with_time("Running workload after")
workload.run_after()
print_with_time("Cleaning up")