def setUp(cls):
if cls.server is not None:
try:
cls.server.check()
except Exception:
cls.server = None
if cls.server is None:
port = str(DEFAULT_DRIVER_PORT) if cls.useDefaultPort else '0'
cls.serverConsoleOutput = tempfile.NamedTemporaryFile('w+')
cls.server = driver.Process(executable_path=rethinkdb_exe,
console_output=cls.serverConsoleOutput,
wait_until_ready=True,
extra_options=['--driver-port', port])
cls.host = cls.server.host
cls.port = cls.server.driver_port
if cls.authKey is not None:
conn = r.connect(host=cls.server.host,
port=cls.server.driver_port)
result = r.db('rethinkdb').table('users').get('admin').update({
'password':
cls.authKey
}).run(conn)
if result != {
'skipped': 0,
'deleted': 0,
'unchanged': 0,
'errors': 0,
'replaced': 1,
'inserted': 0
}:
cls.server = None
raise Exception('Unable to set authkey, got: %s' %
str(result))
def test_server_name_collision(self):
'''run two servers with the same name, then resolve the issue'''
alpha = self.cluster[0]
serverName = alpha.name
# -- stop the first server
alpha.stop()
# -- startup another server with the same name
beta = driver.Process(cluster=self.cluster, name=serverName, command_prefix=command_prefix, extra_options=server_options, console_output=True)
# -- restart the first server
alpha.start()
# -- check for the server name collision
issues = list(self.r.db("rethinkdb").table("current_issues").filter(self.r.row["type"] != "memory_error").run(self.conn))
assert len(issues) == 1, pprint.pformat(issues)
server_issue = issues[0]
assert server_issue["type"] == "server_name_collision", server_issue
assert server_issue["critical"], server_issue
assert server_issue["info"]["name"] == serverName, server_issue
assert set(server_issue["info"]["ids"]) == set([alpha.uuid, beta.uuid]), server_issue
# -- resolve the server name collision
res = self.r.db("rethinkdb").table("server_config").get(beta.uuid).update({"name":serverName + '2'}).run(self.conn)
assert res["replaced"] == 1 and res["errors"] == 0, res
# -- confirm the cluster is ok
self.cluster.check()
def up_down_server(server, files, new_state):
if new_state == 'up' and not server.running:
return driver.Process(cluster, files, console_output=True,
command_prefix=command_prefix, extra_options=serve_options)
elif new_state == 'down' and server.running:
server.close()
return server
def setUp(self):
global _httpbin_server
# -- startup local httpbin server
if _httpbin_server is None:
_httpbin_server = http_support.HttpTargetServer()
else:
try:
_httpbin_server.checkOnServer()
except Exception:
_httpbin_server = http_support.HttpTargetServer()
self.targetServer = _httpbin_server
# -- startup the RethinkDB server
if self.conn == None:
try:
self._server.driver_port = int(
os.getenv('RDB_DRIVER_PORT') or sys.argv[1])
except Exception:
self.__class__._server_log = tempfile.NamedTemporaryFile('w+')
self.__class__._server = driver.Process(
console_output=self._server_log.name)
self.__class__.conn = self._server.driver_port
self.__class__.conn = r.connect('localhost',
self._server.driver_port)
# -- patch the Python2.6 version of unittest to have assertRaisesRegex
if not hasattr(self, 'assertRaisesRegex'):
def assertRaisesRegex_replacement(self, exception, regexp,
function, *args, **kwds):
result = None
try:
result = function(*args, **kwds)
except Exception as e:
if not isinstance(e, exception):
raise AssertionError(
'Got the wrong type of exception: %s vs. expected: %s'
% (e.__class__.__name__, exception.__name__))
if not re.match(regexp, str(e)):
raise AssertionError(
'Error message: "%s" does not match "%s"' %
(str(regexp), str(e)))
return
else:
raise AssertionError(
'%s not raised for: %s, rather got: %s' %
(exception.__name__, repr(function), repr(result)))
self.__class__.assertRaisesRegex = assertRaisesRegex_replacement
def run_tests(build=None, data_dir='./'):
global connection, servers_settings
if not os.path.exists(data_dir):
os.makedirs(data_dir)
if not os.path.isdir(data_dir):
raise ValueError('data_dir is not a directory: %s' % str(data_dir))
executable_path = utils.find_rethinkdb_executable(
) if build is None else os.path.realpath(os.path.join(build, 'rethinkdb'))
if not os.path.basename(
os.path.dirname(executable_path)).startswith('release'):
sys.stderr.write('Warning: Testing a non-release build: %s\n' %
executable_path)
else:
print('Testing: %s' % executable_path)
i = 0
for settings in servers_settings:
print("Starting server with cache_size " +
str(settings["cache_size"]) + " MB...",
end=' ')
sys.stdout.flush()
serverFiles = driver.Files(server_name=settings["name"],
db_path=os.path.join(
data_dir, settings["name"]))
with driver.Process(
files=serverFiles,
executable_path=executable_path,
extra_options=['--cache-size',
str(settings["cache_size"])]) as server:
print(" Done.\nConnecting...", end=' ')
sys.stdout.flush()
connection = r.connect(host="localhost", port=server.driver_port)
print(" Done.")
sys.stdout.flush()
init_tables(connection)
# Tests
execute_read_write_queries(settings["name"])
if i == 0:
execute_constant_queries()
i = i + 1
save_compare_results()
def setUp(self):
global sharedServer, sharedServerOutput, sharedServerHost, sharedServerDriverPort
if sharedServer is not None:
try:
sharedServer.check()
except Exception:
# ToDo: figure out how to blame the last test
closeSharedServer()
if sharedServerDriverPort is None:
sharedServerOutput = tempfile.NamedTemporaryFile('w+')
sharedServer = driver.Process(executable_path=rethinkdb_exe, console_output=sharedServerOutput, wait_until_ready=True)
sharedServerHost = sharedServer.host
sharedServerDriverPort = sharedServer.driver_port
# - insure we are ready
checkSharedServer()
#!/usr/bin/env python
# Copyright 2010-2016 RethinkDB, all rights reserved.
import os, sys
sys.path.append(
os.path.abspath(
os.path.join(os.path.dirname(__file__), os.path.pardir, "common")))
import driver, utils
r = utils.import_python_driver()
with driver.Process() as process:
conn = r.connect(process.host, process.driver_port)
users = r.db("rethinkdb").table("users")
res = list(users.run(conn))
assert res == [{"id": "admin", "password": False}], res
res = users.get(None).run(conn)
assert res == None, res
res = users.get("admin").run(conn)
assert res == {"id": "admin", "password": False}, res
res = users.get("admin").delete().run(conn)
assert res["errors"] == 1, res
assert res["first_error"] == "The user `admin` can't be deleted.", res
res = users.get("admin").update({"password": "******"}).run(conn)
assert res["replaced"] == 1, res
res = users.get("admin").run(conn)
assert res == {"id": "admin", "password": True}, res
res = users.get("admin").update({"password": False}).run(conn)
import driver, scenario_common, utils, vcoptparse
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(op.parse(sys.argv))
r = utils.import_python_driver()
dbName, tableName = utils.get_test_db_table()
num_servers = 5
print("Starting cluster of %d servers (%.2fs)" % (num_servers, time.time() - startTime))
with driver.Cluster(output_folder='.') as cluster:
for i in xrange(1, num_servers+1):
driver.Process(cluster=cluster, files="s%d" % i, server_tags=["tag_%d" % i], command_prefix=command_prefix, extra_options=serve_options)
cluster.wait_until_ready()
cluster.check()
tag_table = {"default": ["s%d" % (i+1) for i in xrange(num_servers)]}
for i in xrange(num_servers):
tag_table["tag_%d" % (i+1)] = ["s%d" % (i+1)]
server_names = [server.name for server in cluster]
print("Establishing ReQl connections (%.2fs)" % (time.time() - startTime))
conn = r.connect(host=cluster[0].host, port=cluster[0].driver_port)
print("Creating a table (%.2fs)" % (time.time() - startTime))
os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
r = utils.import_python_driver()
utils.print_with_time("Spinning up two servers")
with driver.Cluster(output_folder='.') as cluster:
process1 = driver.Process(cluster,
name='a',
server_tags=["foo"],
command_prefix=command_prefix,
extra_options=serve_options)
process2 = driver.Process(cluster,
name='b',
server_tags=["foo", "bar"],
command_prefix=command_prefix,
extra_options=serve_options +
["--cache-size", "123"])
cluster.wait_until_ready()
utils.print_with_time("Establishing ReQL connection")
conn = r.connect(process1.host, process1.driver_port)
op = OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
op["workload1"] = PositionalArg()
op["workload2"] = PositionalArg()
op["timeout"] = IntFlag("--timeout", 600)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print "Starting cluster..."
files1 = driver.Files(metacluster, db_path = "db-first", log_path = "create-output-first",
executable_path = executable_path, command_prefix = command_prefix)
process1 = driver.Process(cluster, files1, log_path = "serve-output-first",
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
process1.wait_until_started_up()
print "Creating namespace..."
http1 = http_admin.ClusterAccess([("localhost", process1.http_port)])
dc = http1.add_datacenter()
http1.move_server_to_datacenter(files1.machine_name, dc)
ns = scenario_common.prepare_table_for_workload(opts, http1, primary = dc)
http1.wait_until_blueprint_satisfied(ns)
workload_ports_1 = scenario_common.get_workload_ports(opts, ns, [process1])
workload_runner.run(opts["protocol"], opts["workload1"], workload_ports_1, opts["timeout"])
print "Bringing up new server..."
files2 = driver.Files(metacluster, db_path = "db-second", log_path = "create-output-second",
executable_path = executable_path, command_prefix = command_prefix)
assert issubclass(ReqlTimeoutError, ReqlError)
assert issubclass(ReqlAvailabilityError, ReqlError)
assert issubclass(ReqlOpFailedError, ReqlAvailabilityError)
assert issubclass(ReqlOpIndeterminateError, ReqlAvailabilityError)
assert issubclass(ReqlDriverError, ReqlError)
assert issubclass(ReqlAuthError, ReqlDriverError)
assert issubclass(RqlError, Exception)
assert issubclass(RqlClientError, RqlError)
assert issubclass(RqlCompileError, RqlError)
assert issubclass(RqlRuntimeError, RqlError)
assert issubclass(RqlDriverError, Exception)
# -- simple tests
with driver.Process(wait_until_ready=True) as server:
# - connect
r.connect(host=server.host, port=server.driver_port)
conn = rethinkdb.connect(host=server.host, port=server.driver_port)
# - create database
if dbName not in r.db_list().run(conn):
r.db_create(dbName).run(conn)
# - create table
if tableName in r.db(dbName).table_list().run(conn):
r.db(dbName).table_drop(tableName).run(conn)
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
print "Starting cluster..."
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(
opts)
primary_files = driver.Files(metacluster,
db_path="db-1",
log_path="create-output-1",
executable_path=executable_path,
command_prefix=command_prefix)
primary = driver.Process(cluster,
primary_files,
log_path="serve-output-1",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
secondary_files = driver.Files(metacluster,
db_path="db-2",
log_path="create-output-2",
executable_path=executable_path,
command_prefix=command_prefix)
secondary = driver.Process(cluster,
secondary_files,
log_path="serve-output-2",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
primary.wait_until_started_up()
secondary.wait_until_started_up()
os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse
r = utils.import_python_driver()
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, server_options = scenario_common.parse_mode_flags(
op.parse(sys.argv))
with driver.Cluster(output_folder='./servers',
command_prefix=command_prefix,
extra_options=server_options) as cluster:
proc1 = driver.Process(cluster=cluster,
name='a',
server_tags='a_tag',
console_output=True,
command_prefix=command_prefix,
extra_options=server_options)
proc2 = driver.Process(cluster=cluster,
name='b',
server_tags='b_tag',
console_output=True,
command_prefix=command_prefix,
extra_options=server_options)
conn = r.connect(proc1.host, proc1.driver_port)
# wait for all of the startup log messages to be past
logMonitor = r.db('rethinkdb').table('logs').changes().run(conn)
while True:
try:
logMonitor.next(wait=2)
import driver, scenario_common, utils, vcoptparse
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(
op.parse(sys.argv))
r = utils.import_python_driver()
with driver.Metacluster() as metacluster:
cluster1 = driver.Cluster(metacluster)
print("Spinning up two processes (%.2fs)" % (time.time() - startTime))
serverA = driver.Process(cluster1,
console_output=True,
command_prefix=command_prefix,
extra_options=serve_options)
serverB = driver.Process(cluster1,
console_output=True,
command_prefix=command_prefix,
extra_options=serve_options)
cluster1.wait_until_ready()
cluster1.check()
print("Establishing ReQL connections (%.2fs)" % (time.time() - startTime))
connA = r.connect(host=serverA.host, port=serverA.driver_port)
connB = r.connect(host=serverB.host, port=serverB.driver_port)
print("Checking that both servers see each other (%.2fs)" %
(time.time() - startTime))
import http_admin, driver, workload_runner, scenario_common, rdb_workload_common
from vcoptparse import *
op = OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
workload_runner.prepare_option_parser_for_split_or_continuous_workload(op)
op["num-nodes"] = IntFlag("--num-nodes", 2)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print "Starting cluster..."
processes = [driver.Process(cluster,
driver.Files(metacluster, db_path = "db-%d" % i, log_path = "create-output-%d" % i,
executable_path = executable_path, command_prefix = command_prefix), log_path = "serve-output-%d" % i,
executable_path = executable_path,
command_prefix = command_prefix, extra_options = serve_options)
for i in xrange(opts["num-nodes"])]
for process in processes:
process.wait_until_started_up()
print "Creating table..."
http = http_admin.ClusterAccess([("localhost", p.http_port) for p in processes])
primary_dc = http.add_datacenter()
secondary_dc = http.add_datacenter()
machines = http.machines.keys()
http.move_server_to_datacenter(machines[0], primary_dc)
http.move_server_to_datacenter(machines[1], secondary_dc)
ns = scenario_common.prepare_table_for_workload(http, primary = primary_dc)
http.wait_until_blueprint_satisfied(ns)
import sys, os, time
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, http_admin, scenario_common
from vcoptparse import *
op = OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster1 = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print "Spinning up two processes..."
files1 = driver.Files(metacluster, log_path = "create-output-1",
executable_path = executable_path, command_prefix = command_prefix)
proc1 = driver.Process(cluster1, files1,
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
files2 = driver.Files(metacluster, log_path = "create-output-2",
executable_path = executable_path, command_prefix = command_prefix)
proc2 = driver.Process(cluster1, files2,
executable_path = executable_path, command_prefix = command_prefix, extra_options = serve_options)
proc1.wait_until_started_up()
proc2.wait_until_started_up()
cluster1.check()
access1 = http_admin.ClusterAccess([("localhost", proc1.http_port)])
access2 = http_admin.ClusterAccess([("localhost", proc2.http_port)])
assert len(access1.get_directory()) == len(access2.get_directory()) == 2
print "Splitting cluster, then waiting 20s..."
cluster2 = driver.Cluster(metacluster)
metacluster.move_processes(cluster1, cluster2, [proc2])
time.sleep(20)
print "Checking that they detected the netsplit..."
dbName, tableName = utils.get_test_db_table()
numReplicas = opts["sequence"].peak()
print('Starting cluster (%.2fs)' % (time.time() - startTime))
with driver.Cluster(output_folder='.') as cluster:
print('Starting primary server (%.2fs)' % (time.time() - startTime))
primary_files = driver.Files(cluster.metacluster,
db_path="db-primary",
console_output=True,
command_prefix=command_prefix)
primary_process = driver.Process(cluster,
primary_files,
console_output=True,
command_prefix=command_prefix,
extra_options=serve_options)
print('Starting %d replicas (%.2fs)' %
(numReplicas, time.time() - startTime))
replica_processes = [
driver.Process(cluster=cluster,
console_output=True,
command_prefix=command_prefix,
extra_options=serve_options)
for i in xrange(numReplicas)
]
cluster.wait_until_ready()
primary = cluster[0]
if dbName not in r.db_list().run(conn1):
r.db_create(dbName).run(conn1)
if tableName in r.db(dbName).table_list().run(conn1):
r.db(dbName).table_drop(tableName).run(conn1)
r.db(dbName).table_create(tableName).run(conn1)
utils.print_with_time("Starting first workload")
workload_runner.run(opts["workload1"], workload_ports1, opts["timeout"])
utils.print_with_time("Bringing up new server")
server2 = driver.Process(cluster=cluster,
name='second',
command_prefix=command_prefix,
extra_options=serve_options,
wait_until_ready=True)
issues = list(
r.db('rethinkdb').table('current_issues').filter(
r.row["type"] != "memory_error").run(conn1))
assert [] == issues, 'The issues list was not empty: %s' % repr(issues)
utils.print_with_time("Explicitly adding server to the table")
assert r.db(dbName).table(tableName).config() \
.update({'shards':[
{'primary_replica':server2.name, 'replicas':[server2.name, server1.name]}
]})['errors'].run(conn1) == 0
utils.print_with_time("Waiting for backfill")
def setUp(self):
# -- start the servers
# - check on an existing cluster
if self.cluster is not None:
try:
self.checkCluster()
except:
try:
self.cluster.check_and_stop()
except Exception:
pass
self.__class__.cluster = None
self.__class__._conn = None
self.__class__.table = None
# - ensure we have a cluster
if self.cluster is None:
self.__class__.cluster = driver.Cluster()
# - make sure we have any named servers
if hasattr(self.servers, '__iter__'):
for name in self.servers:
firstServer = len(self.cluster) == 0
if not name in self.cluster:
driver.Process(cluster=self.cluster,
name=name,
console_output=True,
command_prefix=self.server_command_prefix,
extra_options=self.server_extra_options,
wait_until_ready=firstServer)
# - ensure we have the proper number of servers
# note: we start up enough servers to make sure they each have only one role
serverCount = max(
self.shards * self.replicas,
len(self.servers)
if hasattr(self.servers, '__iter__') else self.servers)
for _ in range(serverCount - len(self.cluster)):
firstServer = len(self.cluster) == 0
driver.Process(cluster=self.cluster,
wait_until_ready=firstServer,
command_prefix=self.server_command_prefix,
extra_options=self.server_extra_options)
self.cluster.wait_until_ready()
# -- ensure db is available
if self.dbName is not None and self.dbName not in self.r.db_list().run(
self.conn):
self.r.db_create(self.dbName).run(self.conn)
# -- setup test table
if self.tableName is not None:
# - ensure we have a clean table
if self.tableName in self.r.db(self.dbName).table_list().run(
self.conn):
self.r.db(self.dbName).table_drop(self.tableName).run(
self.conn)
self.r.db(self.dbName).table_create(self.tableName).run(self.conn)
self.__class__.table = self.r.db(self.dbName).table(self.tableName)
# - add initial records
if self.recordsToGenerate:
utils.populateTable(conn=self.conn,
table=self.table,
records=self.recordsToGenerate,
fieldName=self.fieldName)
# - shard and replicate the table
primaries = iter(self.cluster[:self.shards])
replicas = iter(self.cluster[self.shards:])
shardPlan = []
for primary in primaries:
chosenReplicas = [
replicas.next().name for _ in range(0, self.replicas - 1)
]
shardPlan.append({
'primary_replica': primary.name,
'replicas': [primary.name] + chosenReplicas
})
assert (self.r.db(self.dbName).table(
self.tableName).config().update({
'shards': shardPlan
}).run(self.conn))['errors'] == 0
self.r.db(self.dbName).table(self.tableName).wait().run(self.conn)
try:
result = funct(*args, **kwds)
except exception:
pass
else:
if result is None:
raise Exception('Should have raised %s, but did not' %
exception.__class__.__name__)
else:
raise Exception('Should have raised %s, but rather returned: %s' %
(exception.__class__.__name__, repr(result)))
print("Starting a server (%.2fs)" % (time.time() - startTime))
with driver.Process(output_folder='.',
command_prefix=command_prefix,
extra_options=serve_options,
wait_until_ready=True) as server:
print("Establishing ReQL Connection (%.2fs)" % (time.time() - startTime))
conn = r.connect(host=server.host, port=server.driver_port)
print("Creating db/table %s/%s (%.2fs)" %
(dbName, tableName, time.time() - startTime))
if dbName not in r.db_list().run(conn):
r.db_create(dbName).run(conn)
if tableName in r.db(dbName).table_list().run(conn):
r.db(dbName).table_drop(tableName).run(conn)
r.db(dbName).table_create(tableName).run(conn)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(opts)
print('Spinning up %d processes...' % len(server_names))
servers = []
for i in xrange(len(server_names)):
info = {'name': server_names[i]}
info['files'] = driver.Files(metacluster,
db_path='db-%d' % i,
console_output='create-output-%d' % i,
server_name=info['name'],
command_prefix=command_prefix)
info['process'] = driver.Process(cluster,
info['files'],
console_output='serve-output-%d' % i,
command_prefix=command_prefix,
extra_options=serve_options)
servers.append(info)
for server in servers:
server['process'].wait_until_started_up()
r.connect(servers[0]['process'].host,
servers[0]['process'].driver_port).repl()
r.db_create(db_name).run()
r.db(db_name).table_create(table_name).run()
tbl = r.db(db_name).table(table_name)
table_id = tbl.config()['id'].run()
op["timeout"] = IntFlag("--timeout", 600)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(
opts)
print "Starting cluster..."
serve_files = driver.Files(metacluster,
db_path="db",
log_path="create-output",
executable_path=executable_path,
command_prefix=command_prefix)
serve_process = driver.Process(cluster,
serve_files,
log_path="serve-output",
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
proxy_process = driver.ProxyProcess(cluster,
'proxy-logfile',
log_path='proxy-output',
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
processes = [serve_process, proxy_process]
for process in processes:
process.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", proxy_process.http_port)])
dc = http.add_datacenter()
op["workload2"] = PositionalArg()
op["timeout"] = IntFlag("--timeout", 600)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(
opts)
print "Starting cluster..."
files = driver.Files(metacluster,
log_path="create-output",
executable_path=executable_path,
command_prefix=command_prefix)
process = driver.Process(cluster,
files,
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options)
process.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", process.http_port)])
dc = http.add_datacenter()
http.move_server_to_datacenter(http.machines.keys()[0], dc)
ns = scenario_common.prepare_table_for_workload(opts, http, primary=dc)
http.wait_until_blueprint_satisfied(ns)
workload_ports = scenario_common.get_workload_ports(opts, ns, [process])
workload_runner.run(opts["protocol"], opts["workload1"], workload_ports,
opts["timeout"])
print "Restarting server..."
process.check_and_stop()
process2 = driver.Process(cluster,
def setUp(self):
# -- start the servers
# - check on an existing cluster
if self.cluster is not None:
try:
self.checkCluster()
except:
try:
self.cluster.check_and_stop()
except Exception:
pass
self.__class__.__cluster = None
self.__class__.__conn = None
self.__class__.__table = None
# - ensure we have a cluster
if self.cluster is None:
self.__class__.__cluster = driver.Cluster(tls=self.use_tls)
# - make sure we have any named servers
if hasattr(self.servers, '__iter__'):
for name in self.servers:
firstServer = len(self.cluster) == 0
if not name in self.cluster:
driver.Process(cluster=self.cluster,
name=name,
console_output=True,
command_prefix=self.server_command_prefix,
extra_options=self.server_extra_options,
wait_until_ready=firstServer)
# - ensure we have the proper number of servers
# note: we start up enough servers to make sure they each have only one role
serverCount = max(
self.shards * self.replicas,
len(self.servers)
if hasattr(self.servers, '__iter__') else self.servers or 0)
for _ in range(serverCount - len(self.cluster)):
firstServer = len(self.cluster) == 0
driver.Process(cluster=self.cluster,
console_output=True,
command_prefix=self.server_command_prefix,
extra_options=self.server_extra_options,
wait_until_ready=firstServer)
self.cluster.wait_until_ready()
# -- ensure db is available
self.r.expr([self.dbName]).set_difference(self.r.db_list()).for_each(
self.r.db_create(self.r.row)).run(self.conn)
# -- setup test tables
# - drop all tables unless cleanTables is set to False
if self.cleanTables:
self.r.db('rethinkdb').table('table_config').filter({
'db':
self.dbName
}).delete().run(self.conn)
for tableName in (x for x in (self.tableNames or []) if x not in
self.r.db(self.dbName).table_list().run(self.conn)):
# - create the table
self.r.db(self.dbName).table_create(tableName).run(self.conn)
table = self.db.table(tableName)
# - add initial records
self.populateTable(conn=self.conn,
table=table,
records=self.recordsToGenerate,
fieldName=self.fieldName)
# - shard and replicate the table
primaries = iter(self.cluster[:self.shards])
replicas = iter(self.cluster[self.shards:])
shardPlan = []
for primary in primaries:
chosenReplicas = [
replicas.next().name for _ in range(0, self.replicas - 1)
]
shardPlan.append({
'primary_replica': primary.name,
'replicas': [primary.name] + chosenReplicas
})
assert (table.config().update({
'shards': shardPlan
}).run(self.conn))['errors'] == 0
table.wait().run(self.conn)
# -- run setUpClass if not run otherwise
if not hasattr(unittest.TestCase, 'setUpClass') and hasattr(
self.__class__, 'setUpClass') and not hasattr(
self.__class__, self.__class__.__name__ + '_setup'):
self.setUpClass()
setattr(self.__class__, self.__class__.__name__ + '_setup', True)
import os, pprint, resource, sys, time
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, utils, scenario_common, vcoptparse
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(op.parse(sys.argv))
r = utils.import_python_driver()
utils.print_with_time("Setting resource limit")
size_limit = 10 * 1024 * 1024
resource.setrlimit(resource.RLIMIT_FSIZE, (size_limit, resource.RLIM_INFINITY))
with driver.Process(name='.', command_prefix=command_prefix, extra_options=serve_options) as process:
conn = r.connect(process.host, process.driver_port)
utils.print_with_time("Un-setting resource limit")
resource.setrlimit(resource.RLIMIT_FSIZE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
utils.print_with_time("log size: %d" % os.path.getsize(process.logfile_path))
utils.print_with_time("Filling log file to %d bytes" % size_limit)
with open(process.logfile_path, "a") as f:
while f.tell() < size_limit:
f.write("X" * min(100, (size_limit - f.tell()) - 1) + "\n")
utils.print_with_time("log size: %d" % os.path.getsize(process.logfile_path))
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(
op.parse(sys.argv))
r = utils.import_python_driver()
dbName, tableName = utils.get_test_db_table()
cycles = 10
# == start first instance of server
utils.print_with_time("Starting server")
with driver.Process(console_output=True,
name='.',
command_prefix=command_prefix,
extra_options=serve_options,
wait_until_ready=True) as server:
utils.print_with_time("Establishing ReQL connection")
conn = r.connect(host=server.host, port=server.driver_port)
utils.print_with_time("Creating db/table %s/%s" % (dbName, tableName))
if dbName not in r.db_list().run(conn):
r.db_create(dbName).run(conn)
if tableName in r.db(dbName).table_list().run(conn):
r.db(dbName).table_drop(tableName).run(conn)
r.db(dbName).table_create(tableName).run(conn)
utils.print_with_time("Collecting metadata for first run")
initalMetadata = r.db('rethinkdb').table('server_config').get(
import http_admin, driver
from vcoptparse import *
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
print "Starting cluster..."
num_nodes = 2
files = [
driver.Files(metacluster,
db_path="db-%d" % i,
log_path="create-output-%d" % i)
for i in xrange(num_nodes)
]
processes = [
driver.Process(cluster,
files[i],
log_path="serve-output-%d" % i,
executable_path=driver.find_rethinkdb_executable())
for i in xrange(num_nodes)
]
time.sleep(3)
print "Creating table..."
http = http_admin.ClusterAccess([("localhost", p.http_port)
for p in processes])
db = http.add_database("test")
dc = http.add_datacenter()
for machine_id in http.machines:
http.move_server_to_datacenter(machine_id, dc)
ns = http.add_namespace(protocol="rdb",
primary=dc,
name="stress",
database=db)
from vcoptparse import *
op = OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
opts = op.parse(sys.argv)
with driver.Metacluster() as metacluster:
cluster = driver.Cluster(metacluster)
executable_path, command_prefix, serve_options = scenario_common.parse_mode_flags(
opts)
print "Starting cluster..."
processes = [
driver.Process(cluster,
driver.Files(metacluster,
log_path=("create-output-%d" % (i + 1)),
executable_path=executable_path,
command_prefix=command_prefix),
executable_path=executable_path,
command_prefix=command_prefix,
extra_options=serve_options) for i in xrange(2)
]
for process in processes:
process.wait_until_started_up()
print "Creating namespace..."
http = http_admin.ClusterAccess([("localhost", p.http_port)
for p in processes])
dc = http.add_datacenter()
for machine_id in http.machines:
http.move_server_to_datacenter(machine_id, dc)
ns = http.add_namespace(protocol="memcached", primary=dc)
http.wait_until_blueprint_satisfied(ns)
"""The `interface.db_config` test checks that the special `rethinkdb.db_config` table behaves as expected."""
import os, sys
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse
r = utils.import_python_driver()
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(op.parse(sys.argv))
utils.print_with_time("Starting server")
with driver.Process(name='./a', command_prefix=command_prefix, extra_options=serve_options, wait_until_ready=True) as server:
utils.print_with_time("Establishing ReQL connection")
conn = r.connect(host=server.host, port=server.driver_port)
utils.print_with_time("Starting tests")
res = list(r.db("rethinkdb").table("db_config").run(conn))
assert res == [], res
res = r.db_create("foo").run(conn)
assert res["dbs_created"] == 1
assert len(res["config_changes"]) == 1
assert res["config_changes"][0]["old_val"] is None
assert res["config_changes"][0]["new_val"] == r.db("foo").config().run(conn)
rows = list(r.db("rethinkdb").table("db_config").run(conn))
assert len(rows) == 1 and rows[0]["name"] == "foo"
os.path.join(os.path.dirname(__file__), os.path.pardir, 'common')))
import driver, scenario_common, utils, vcoptparse
# --
op = vcoptparse.OptParser()
scenario_common.prepare_option_parser_mode_flags(op)
_, command_prefix, serve_options = scenario_common.parse_mode_flags(
op.parse(sys.argv))
r = utils.import_python_driver()
# --
utils.print_with_time("Starting a server")
with driver.Process(name="./the_server") as server:
server.check()
conn = r.connect(host=server.host, port=server.driver_port)
utils.print_with_time("Making a log entry")
res = r.db("rethinkdb").table("server_config").update({
"name": "xyz"
}).run(conn)
assert res["errors"] == 0 and res["replaced"] == 1, res
utils.print_with_time("Verifying the log entry")
log = list(r.db("rethinkdb").table("logs").order_by("timestamp").run(conn))
assert len(log) > 5
