def run_tests(build=None, data_dir='./'):
global connection, servers_settings
if not os.path.exists(data_dir):
os.makedirs(data_dir)
if not os.path.isdir(data_dir):
raise ValueError('data_dir is not a directory: %s' % str(data_dir))
executable_path = utils.find_rethinkdb_executable(
) if build is None else os.path.realpath(os.path.join(build, 'rethinkdb'))
if not os.path.basename(
os.path.dirname(executable_path)).startswith('release'):
sys.stderr.write('Warning: Testing a non-release build: %s\n' %
executable_path)
else:
print('Testing: %s' % executable_path)
i = 0
for settings in servers_settings:
print("Starting server with cache_size " +
str(settings["cache_size"]) + " MB...",
end=' ')
sys.stdout.flush()
serverFiles = driver.Files(server_name=settings["name"],
db_path=os.path.join(
data_dir, settings["name"]))
with driver.Process(
files=serverFiles,
executable_path=executable_path,
extra_options=['--cache-size',
str(settings["cache_size"])]) as server:
print(" Done.\nConnecting...", end=' ')
sys.stdout.flush()
connection = r.connect(host="localhost", port=server.driver_port)
print(" Done.")
sys.stdout.flush()
init_tables(connection)
# Tests
execute_read_write_queries(settings["name"])
if i == 0:
execute_constant_queries()
i = i + 1
save_compare_results()
def parse_mode_flags(parsed_opts):
mode = parsed_opts["mode"]
if parsed_opts["valgrind"]:
assert parsed_opts["wrapper"] is None
command_prefix = ["valgrind"]
for valgrind_option in shlex.split(parsed_opts["valgrind-options"]):
command_prefix.append(valgrind_option)
# Make sure we use the valgrind build
# this assumes that the 'valgrind' substring goes at the end of the specific build string
if "valgrind" not in mode and mode != "":
mode = mode + "-valgrind"
elif parsed_opts["wrapper"] is not None:
command_prefix = shlex.split(parsed_opts["wrapper"])
else:
command_prefix = []
return utils.find_rethinkdb_executable(mode=mode), command_prefix, shlex.split(parsed_opts["serve-flags"])
def run_tests(build=None, data_dir='./'):
global connection, servers_settings
if not os.path.exists(data_dir):
os.makedirs(data_dir)
if not os.path.isdir(data_dir):
raise ValueError('data_dir is not a directory: %s' % str(data_dir))
executable_path = utils.find_rethinkdb_executable() if build is None else os.path.realpath(os.path.join(build, 'rethinkdb'))
if not os.path.basename(os.path.dirname(executable_path)).startswith('release'):
sys.stderr.write('Warning: Testing a non-release build: %s\n' % executable_path)
else:
print('Testing: %s' % executable_path)
i = 0
for settings in servers_settings:
print("Starting server with cache_size " + str(settings["cache_size"]) + " MB...", end=' ')
sys.stdout.flush()
with driver.Process(name=os.path.join(data_dir, settings["name"]), executable_path=executable_path, extra_options=['--cache-size', str(settings["cache_size"])]) as server:
print(" Done.\nConnecting...", end=' ')
sys.stdout.flush()
connection = r.connect(host="localhost", port=server.driver_port)
print(" Done.")
sys.stdout.flush()
init_tables(connection)
# Tests
execute_read_write_queries(settings["name"])
if i == 0:
execute_constant_queries()
i = i + 1
save_compare_results()
xrange = range
# -- import the rethinkdb driver
r = utils.import_python_driver()
# -- use asyncio subdriver
r.set_loop_type("twisted")
# -- get settings
DEFAULT_DRIVER_PORT = 28015
rethinkdb_exe = (sys.argv[1]
if len(sys.argv) > 1 else utils.find_rethinkdb_executable())
use_default_port = bool(int(sys.argv[2])) if len(sys.argv) > 2 else 0
# -- shared server
sharedServer = None
sharedServerOutput = None
sharedServerHost = None
sharedServerDriverPort = None
if 'RDB_DRIVER_PORT' in os.environ:
sharedServerDriverPort = int(os.environ['RDB_DRIVER_PORT'])
if 'RDB_SERVER_HOST' in os.environ:
sharedServerHost = os.environ['RDB_SERVER_HOST']
else:
sharedServerHost = 'localhost'
def __init__(self,
cluster=None,
name=None,
console_output=None,
executable_path=None,
server_tags=None,
command_prefix=None,
extra_options=None,
wait_until_ready=True,
tls=None):
global runningServers
# -- validate/default input
# - cluster
assert tls in (None, False,
True), 'tls must be True, False, or None (False)'
if cluster is None:
cluster = Cluster(tls=tls is True)
else:
assert tls is None or tls is (
cluster.tlsCertPath is None
), 'A server added to a cluster must have the same tls setting (cluster: %s vs %s)' % (
cluster.tlsCertPath is None, tls)
assert isinstance(
cluster,
Cluster), 'cluster must be a Cluster or None, got: %r' % cluster
self.cluster = cluster
self.cluster.processes.append(self)
# - name/data_path
assert isinstance(
name,
(str, unicode,
None.__class__)), 'name was not an expected value: %r' % name
if name is None:
# default name and data_path
name = '%s_%s' % (self.server_type,
self.cluster.metacluster.get_new_unique_id(
server_type=self.server_type))
data_path = self.genPath(name, self.cluster.output_folder)
elif name == '.':
# random name in the current directory
name = '%s_%s' % (self.server_type,
self.cluster.metacluster.get_new_unique_id(
server_type=self.server_type))
data_path = self.genPath(name, os.path.realpath('.'))
elif os.sep in name: # if it looks like a path, it is one, possibly relative
data_path = os.path.abspath(
os.path.join(self.cluster.output_folder,
name)) # note: an absolute path survives this
name = os.path.basename(name)
if os.path.exists(data_path):
assert os.path.isdir(
data_path
), 'The data_path was something other than a directory: %s' % data_path
if os.path.isfile(os.path.join(data_path, 'log_file')):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, 'log_file')
elif os.path.isfile(os.path.join(data_path, 'log_file.txt')):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, 'log_file.txt')
else:
# folder for holding multiple server data folders
data_path = self.genPath(name, data_path)
else:
# specified path, use it exactly
assert os.path.isdir(
os.path.dirname(data_path)
), 'The enclosing directory did not exist or was not a directory: %s' % data_path
else:
# just a name
data_path = self.genPath(name, self.cluster.output_folder)
self.data_path = data_path
self._desired_name = name.replace('-', '_')
# - console_output/console_file - can be: path, file-object, True/False
if console_output is None:
# default to stdout
self._console_file_path = None
self._console_file = sys.stdout
elif console_output is False:
# throw away file
self._console_file_path = None
self._console_file = tempfile.NamedTemporaryFile(mode='w+')
elif console_output is True:
# keep it with the data, but create it in the encosing folder until we are running
self._console_file_path = os.path.join(self.data_path,
self._console_file_name)
self._console_file = tempfile.NamedTemporaryFile(
mode='w+', dir=os.path.dirname(self.data_path), delete=False)
elif hasattr(console_output, 'write'):
# file-like object:
self._console_file_path = None
self._console_file = console_output
else:
# a path
assert isinstance(
console_output,
(str, unicode)), 'Unknown console_output: %r' % console_output
console_output = os.path.realpath(console_output)
assert os.path.isdir(
os.path.dirname(console_output)
), 'console_output parent directory was not a folder: %r' % console_output
assert os.path.isfile(console_output) or not os.path.exists(
console_output
), 'console_output location was not useable: %r' % console_output
self._console_file_path = console_output
self._console_file = open(console_output, 'a')
# - server_tags
if server_tags is None:
self.server_tags = []
elif hasattr(server_tags,
'__iter__') and not hasattr(server_tags, 'capitalize'):
self.server_tags = [str(x) for x in server_tags]
else:
self.server_tags = [str(server_tags)]
# - executable_path
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path,
os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
# - command_prefix
if command_prefix is None:
self.command_prefix = []
elif not hasattr(command_prefix, '__iter__'):
raise ValueError(
'command_prefix must be an array of command line options, got: %r'
% command_prefix)
else:
self.command_prefix = command_prefix
# - extra_options
self.options = []
if extra_options is None:
extra_options = []
elif not hasattr(extra_options, '__iter__'):
raise ValueError(
'extra_options must be an array of command line options, got: %r'
% extra_options)
else:
extra_options = [str(x) for x in extra_options]
# -- defaults
if not '--bind' in extra_options and not any(
[x.startswith('--bind=') for x in extra_options]):
self.options += ['--bind', 'all']
if not '--cluster-port' in extra_options and not any(
[x.startswith('--cluster-port=') for x in extra_options]):
self.options += ['--cluster-port', '0']
if not '--driver-port' in extra_options and not any(
[x.startswith('--driver-port=') for x in extra_options]):
self.options += ['--driver-port', '0']
if not '--http-port' in extra_options and not any(
[x.startswith('--http-port=') for x in extra_options]):
self.options += ['--http-port', '0']
for i, option in enumerate(extra_options or []):
if option == '--log-file':
assert len(
self.options
) > i + 1, '--log-file specified in options without a path'
self.logfile_path = utils.translatePath(
os.path.realpath(extra_options[i + 1]))
del extra_options[i + 1]
del extra_options[i]
break
elif option.startswith('--log-file='):
self.logfile_path = utils.translatePath(
os.path.realpath(option[len('--log-file='):].strip('\'"')))
del extra_options[i]
break
else:
self.logfile_path = os.path.join(self.data_path, "log_file.txt")
if not '--no-update-check' in extra_options:
self.options += ['--no-update-check'
] # supress update checks/reporting in
self.options += extra_options
# - subclass modifications
self.subclass_init()
# - save the args
self.args = self.command_prefix + [self.executable_path] + self.options
# -- start the server process
self.start(wait_until_ready=wait_until_ready)
# -- import the rethinkdb driver
r = utils.import_python_driver()
# -- use asyncio subdriver
r.set_loop_type("asyncio")
# -- get settings
DEFAULT_DRIVER_PORT = 28015
rethinkdb_exe = (sys.argv[1]
if len(sys.argv) > 1
else utils.find_rethinkdb_executable())
use_default_port = bool(int(sys.argv[2])) if len(sys.argv) > 2 else 0
# -- shared server
sharedServer = None
sharedServerOutput = None
sharedServerHost = None
sharedServerDriverPort = None
if 'RDB_DRIVER_PORT' in os.environ:
sharedServerDriverPort = int(os.environ['RDB_DRIVER_PORT'])
if 'RDB_SERVER_HOST' in os.environ:
sharedServerHost = os.environ['RDB_SERVER_HOST']
else:
sharedServerHost = 'localhost'
def __init__(self, cluster, options, console_output=None, executable_path=None, command_prefix=None):
global runningServers
# - validate input
assert isinstance(cluster, Cluster)
assert cluster.metacluster is not None
if command_prefix is None:
command_prefix = []
if options is None:
options = []
elif not hasattr(options, 'index'):
raise ValueError('options must be an array of command line options, got: %s' % str(options))
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path, os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
for other_cluster in cluster.metacluster.clusters:
if other_cluster is not cluster:
other_cluster._block_process(self)
if self.console_file is None:
if console_output is None:
self.console_file = sys.stdout
elif console_output in (False, True):
self._close_console_output = True
if console_output is False or self.files is None:
self.console_file = tempfile.NamedTemporaryFile(mode='w+')
else:
self.console_file = open(os.path.join(self.files.db_path, 'console.txt'), 'w+')
elif hasattr(console_output, 'write'):
self.console_file = console_output
else:
self._close_console_output = True
self.console_file = open(console_output, "a")
# - add to the cluster
self.cluster = cluster
self.cluster.processes.append(self)
# - set defaults
if not '--bind' in options:
options += ['--bind', 'all']
if not '--cluster-port' in options:
options += ['--cluster-port', '0']
if not '--driver-port' in options:
options += ['--driver-port', '0']
if not '--http-port' in options:
options += ['--http-port', '0']
if not '--client-port' in options: # allows resunder to know what port to block
self.local_cluster_port = utils.get_avalible_port()
options += ['--client-port', str(self.local_cluster_port)]
if not '--log-file' in options:
options += ['--log-file', str(self.logfile_path)]
if not '--no-update-check' in options:
options += ['--no-update-check'] # supress update checks/reporting in
# - set to join the cluster
for peer in cluster.processes:
if peer != self:
options += ["--join", peer.host + ":" + str(peer.cluster_port)]
break
# -
try:
self.args = command_prefix + [self.executable_path] + options
if os.path.exists(self.logfile_path):
os.unlink(self.logfile_path)
self.console_file.write("Launching:\n%s\n" % str(self.args))
self.process = subprocess.Popen(self.args, stdout=self.console_file, stderr=subprocess.STDOUT, preexec_fn=os.setpgrp)
runningServers.append(self)
self.process_group_id = self.process.pid
self.running = True
# - start thread to tail output for needed info
thread.start_new_thread(self.read_ports_from_log, ())
except Exception:
# `close()` won't be called because we haven't put ourself into
# `cluster.processes` yet, so we have to clean up manually
for other_cluster in cluster.metacluster.clusters:
if other_cluster is not cluster:
other_cluster._unblock_process(self)
if self.cluster and self in self.cluster.processes:
self.cluster.processes.remove(self)
assert self not in self.cluster.processes
raise
def __init__(self, metacluster=None, server_name=None, server_tags=None, db_path=None, db_containter=None, console_output=None, executable_path=None, command_prefix=None):
# -- input validation/defaulting
if metacluster is None:
metacluster = Metacluster() # ToDo: make this be the "default" Metacluster
assert isinstance(metacluster, Metacluster)
assert not metacluster.closed
self.metacluster = metacluster
self.id_number = self.metacluster.get_new_unique_id()
if server_name is None:
self.server_name = "node_%d" % self.id_number
else:
self.server_name = str(server_name)
if db_path is None:
folderName = server_name + '_data' if server_name else 'db-%d' % self.id_number
if db_containter is None:
self.db_path = os.path.join(self.metacluster.dbs_path, folderName)
else:
assert os.path.isdir(db_containter)
self.db_path = os.path.join(db_containter, folderName)
else:
db_containter = db_containter or '.'
self.db_path = os.path.join(db_containter, str(db_path))
assert not os.path.exists(self.db_path), 'The path given for the new files already existed: %s' % self.db_path
moveConsole = False
if console_output is None:
print("Redirecting console_output to /dev/null.")
console_output = "/dev/null"
elif console_output is True:
console_output = tempfile.NamedTemporaryFile('w+', dir=self.metacluster.dbs_path)
moveConsole = True
elif console_output is False:
console_output = tempfile.NamedTemporaryFile('w+')
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path, os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
if command_prefix is None:
command_prefix = []
if server_tags is None:
server_tags = []
# -- create the directory
create_args = command_prefix + [
self.executable_path, "create",
"--directory", self.db_path,
"--server-name", self.server_name]
for tag in server_tags:
create_args += ["--server-tag", tag]
if hasattr(console_output, 'write'):
subprocess.check_call(create_args, stdout=console_output, stderr=console_output)
else:
with open(console_output, "a") as console_file:
subprocess.check_call(create_args, stdout=console_file, stderr=console_file)
def __init__(
self,
cluster=None,
name=None,
console_output=None,
executable_path=None,
server_tags=None,
command_prefix=None,
extra_options=None,
wait_until_ready=True,
tls=None,
):
global runningServers
# -- validate/default input
# - cluster
assert tls in (None, False, True), "tls must be True, False, or None (False)"
if cluster is None:
cluster = Cluster(tls=tls is True)
else:
assert tls is None or tls is (cluster.tlsCertPath is None), (
"A server added to a cluster must have the same tls setting (cluster: %s vs %s)"
% (cluster.tlsCertPath is None, tls)
)
assert isinstance(cluster, Cluster), "cluster must be a Cluster or None, got: %r" % cluster
self.cluster = cluster
self.cluster.processes.append(self)
# - name/data_path
assert isinstance(name, (str, unicode, None.__class__)), "name was not an expected value: %r" % name
if name is None:
# default name and data_path
name = "%s_%s" % (
self.server_type,
self.cluster.metacluster.get_new_unique_id(server_type=self.server_type),
)
data_path = self.genPath(name, self.cluster.output_folder)
elif name == ".":
# random name in the current directory
name = "%s_%s" % (
self.server_type,
self.cluster.metacluster.get_new_unique_id(server_type=self.server_type),
)
data_path = self.genPath(name, os.path.realpath("."))
elif os.sep in name: # if it looks like a path, it is one, possibly relative
data_path = os.path.abspath(
os.path.join(self.cluster.output_folder, name)
) # note: an absolute path survives this
name = os.path.basename(name)
if os.path.exists(data_path):
assert os.path.isdir(data_path), "The data_path was something other than a directory: %s" % data_path
if os.path.isfile(os.path.join(data_path, "log_file")):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, "log_file")
elif os.path.isfile(os.path.join(data_path, "log_file.txt")):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, "log_file.txt")
else:
# folder for holding multiple server data folders
data_path = self.genPath(name, data_path)
else:
# specified path, use it exactly
assert os.path.isdir(os.path.dirname(data_path)), (
"The enclosing directory did not exist or was not a directory: %s" % data_path
)
else:
# just a name
data_path = self.genPath(name, self.cluster.output_folder)
self.data_path = data_path
self._desired_name = name.replace("-", "_")
# - console_output/console_file - can be: path, file-object, True/False
if console_output is None:
# default to stdout
self._console_file_path = None
self._console_file = sys.stdout
elif console_output is False:
# throw away file
self._console_file_path = None
self._console_file = tempfile.NamedTemporaryFile(mode="w+")
elif console_output is True:
# keep it with the data, but create it in the encosing folder until we are running
self._console_file_path = os.path.join(self.data_path, self._console_file_name)
self._console_file = tempfile.NamedTemporaryFile(
mode="w+", dir=os.path.dirname(self.data_path), delete=False
)
elif hasattr(console_output, "write"):
# file-like object:
self._console_file_path = None
self._console_file = console_output
else:
# a path
assert isinstance(console_output, (str, unicode)), "Unknown console_output: %r" % console_output
console_output = os.path.realpath(console_output)
assert os.path.isdir(os.path.dirname(console_output)), (
"console_output parent directory was not a folder: %r" % console_output
)
assert os.path.isfile(console_output) or not os.path.exists(console_output), (
"console_output location was not useable: %r" % console_output
)
self._console_file_path = console_output
self._console_file = open(console_output, "a")
# - server_tags
if server_tags is None:
self.server_tags = []
elif hasattr(server_tags, "__iter__") and not hasattr(server_tags, "capitalize"):
self.server_tags = [str(x) for x in server_tags]
else:
self.server_tags = [str(server_tags)]
# - executable_path
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path, os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
# - command_prefix
if command_prefix is None:
self.command_prefix = []
elif not hasattr(command_prefix, "__iter__"):
raise ValueError("command_prefix must be an array of command line options, got: %r" % command_prefix)
else:
self.command_prefix = command_prefix
# - extra_options
self.options = []
if extra_options is None:
extra_options = []
elif not hasattr(extra_options, "__iter__"):
raise ValueError("extra_options must be an array of command line options, got: %r" % extra_options)
else:
extra_options = [str(x) for x in extra_options]
# -- defaults
if not "--bind" in extra_options and not any([x.startswith("--bind=") for x in extra_options]):
self.options += ["--bind", "all"]
if not "--cluster-port" in extra_options and not any([x.startswith("--cluster-port=") for x in extra_options]):
self.options += ["--cluster-port", "0"]
if not "--driver-port" in extra_options and not any([x.startswith("--driver-port=") for x in extra_options]):
self.options += ["--driver-port", "0"]
if not "--http-port" in extra_options and not any([x.startswith("--http-port=") for x in extra_options]):
self.options += ["--http-port", "0"]
for i, option in enumerate(extra_options or []):
if option == "--log-file":
assert len(self.options) > i + 1, "--log-file specified in options without a path"
self.logfile_path = utils.translatePath(os.path.realpath(extra_options[i + 1]))
del extra_options[i + 1]
del extra_options[i]
break
elif option.startswith("--log-file="):
self.logfile_path = utils.translatePath(os.path.realpath(option[len("--log-file=") :].strip("'\"")))
del extra_options[i]
break
else:
self.logfile_path = os.path.join(self.data_path, "log_file.txt")
if not "--no-update-check" in extra_options:
self.options += ["--no-update-check"] # supress update checks/reporting in
self.options += extra_options
# - subclass modifications
self.subclass_init()
# - save the args
self.args = self.command_prefix + [self.executable_path] + self.options
# -- start the server process
self.start(wait_until_ready=wait_until_ready)
