def subclass_init(self):
"""Complete init for this subclass."""
# -- modify options
# add --cache-size if it is not specified
if not "--cache-size" in self.options and not any([x.startswith("--cache-size=") for x in self.options]):
self.options += ["--cache-size", "512"]
# add --directory
for i, option in enumerate(self.options):
if option == "--directory" or option.startswith("--directory="):
raise ValueError("The --directory should not be provided in extra_options")
# add 'serve' before the rest of the options
self.options.insert(0, "serve")
# -- create the data folder if it does not already exists
# get --server-tag entries
server_tags = []
for tag in self.server_tags:
server_tags += ["--server-tag", tag]
if not os.path.exists(os.path.join(self.data_path, "metadata")):
self._console_file.write(
"Creating data directory at %s (%s)\n" % (self.data_path, datetime.datetime.now().isoformat())
)
self._console_file.flush()
command = (
self.command_prefix
+ [self.executable_path]
+ [
"create",
"--server-name",
self._desired_name,
"--directory",
utils.translatePath(self.data_path),
"--log-file",
utils.translatePath(self.logfile_path),
]
+ server_tags
)
subprocess.check_call(command, stdout=self._console_file, stderr=subprocess.STDOUT)
def subclass_init(self):
'''Complete init for this subclass.'''
# -- modify options
# add --cache-size if it is not specified
if not '--cache-size' in self.options and not any(
[x.startswith('--cache-size=') for x in self.options]):
self.options += ['--cache-size', '512']
# add --directory
for i, option in enumerate(self.options):
if option == '--directory' or option.startswith('--directory='):
raise ValueError(
'The --directory should not be provided in extra_options')
# add 'serve' before the rest of the options
self.options.insert(0, 'serve')
# -- create the data folder if it does not already exists
# get --server-tag entries
server_tags = []
for tag in self.server_tags:
server_tags += ['--server-tag', tag]
if not os.path.exists(os.path.join(self.data_path, 'metadata')):
self._console_file.write(
"Creating data directory at %s (%s)\n" %
(self.data_path, datetime.datetime.now().isoformat()))
self._console_file.flush()
command = self.command_prefix + [self.executable_path] + [
'create', '--server-name', self._desired_name, '--directory',
utils.translatePath(self.data_path), '--log-file',
utils.translatePath(self.logfile_path)
] + server_tags
subprocess.check_call(command,
stdout=self._console_file,
stderr=subprocess.STDOUT)
def start(self, wait_until_ready=True):
'''Start up the server'''
global runningServers
assert not self.running, 'Trying to start a server where running = %r' % self.running
self.returncode = None
self.killed = False
# -- setup options
# - copy the options
options = copy.copy(self.args)
# - set the local_cluster_port
if not '--client-port' in options and not any(
[x.startswith('--client-port=') for x in options]):
# allows resunder to know what port to block
options += ['--client-port', str(self.local_cluster_port)]
# - tls options
if self.cluster.tlsKeyPath and self.cluster.tlsCertPath:
options += [
'--http-tls-key',
utils.translatePath(self.cluster.tlsKeyPath),
'--http-tls-cert',
utils.translatePath(self.cluster.tlsCertPath),
'--driver-tls-key',
utils.translatePath(self.cluster.tlsKeyPath),
'--driver-tls-cert',
utils.translatePath(self.cluster.tlsCertPath),
'--cluster-tls-key',
utils.translatePath(self.cluster.tlsKeyPath),
'--cluster-tls-cert',
utils.translatePath(self.cluster.tlsCertPath),
'--cluster-tls-ca',
utils.translatePath(self.cluster.tlsCertPath)
]
if platform.system() == 'Darwin':
# because the driver will probably be using the system version of OpenSSL (old), enable TLSv1
options += [
'--tls-min-protocol', 'TLSv1', '--tls-ciphers',
'EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:AES256-SHA'
]
# - log file
options += ['--log-file', utils.translatePath(str(self.logfile_path))]
# -- get the length of an exiting log file
if os.path.isfile(self.logfile_path):
self._existing_log_len = os.path.getsize(self.logfile_path)
# -- set to join the cluster
try:
self.cluster._startLock.acquire()
self.cluster._hasStartLock = self
for peer in self.cluster.processes:
if peer != self and peer.ready:
options += [
"--join", peer.host + ":" + str(peer.cluster_port)
]
break
finally:
# release the lock if we are joining a running cluster
if "--join" in options and self.cluster._hasStartLock is self:
self.cluster._hasStartLock = None
self.cluster._startLock.release()
# -- allow subclasses to modify the options array
self.subclass_options(options) # in-place edit
# -- start the process
try:
self._console_file.write(
"Launching at %s:\n\t%s\n" %
(datetime.datetime.now().isoformat(), " ".join(options)))
self._console_file.flush()
self.process = subprocess.Popen(options,
stdout=self._console_file,
stderr=subprocess.STDOUT,
preexec_fn=os.setpgrp)
if not self in runningServers:
runningServers.append(self)
# - start thread to tail output for needed info
thread.start_new_thread(self.read_ports_from_log, ())
except Exception:
self.stop()
raise
finally:
# - move console file if necessary
if self._console_file_path and not os.path.exists(
self._console_file_path):
os.rename(self._console_file.name, self._console_file_path)
# -- wait until ready (if requested)
if wait_until_ready:
self.wait_until_ready()
def __init__(self,
cluster=None,
name=None,
console_output=None,
executable_path=None,
server_tags=None,
command_prefix=None,
extra_options=None,
wait_until_ready=True,
tls=None):
global runningServers
# -- validate/default input
# - cluster
assert tls in (None, False,
True), 'tls must be True, False, or None (False)'
if cluster is None:
cluster = Cluster(tls=tls is True)
else:
assert tls is None or tls is (
cluster.tlsCertPath is None
), 'A server added to a cluster must have the same tls setting (cluster: %s vs %s)' % (
cluster.tlsCertPath is None, tls)
assert isinstance(
cluster,
Cluster), 'cluster must be a Cluster or None, got: %r' % cluster
self.cluster = cluster
self.cluster.processes.append(self)
# - name/data_path
assert isinstance(
name,
(str, unicode,
None.__class__)), 'name was not an expected value: %r' % name
if name is None:
# default name and data_path
name = '%s_%s' % (self.server_type,
self.cluster.metacluster.get_new_unique_id(
server_type=self.server_type))
data_path = self.genPath(name, self.cluster.output_folder)
elif name == '.':
# random name in the current directory
name = '%s_%s' % (self.server_type,
self.cluster.metacluster.get_new_unique_id(
server_type=self.server_type))
data_path = self.genPath(name, os.path.realpath('.'))
elif os.sep in name: # if it looks like a path, it is one, possibly relative
data_path = os.path.abspath(
os.path.join(self.cluster.output_folder,
name)) # note: an absolute path survives this
name = os.path.basename(name)
if os.path.exists(data_path):
assert os.path.isdir(
data_path
), 'The data_path was something other than a directory: %s' % data_path
if os.path.isfile(os.path.join(data_path, 'log_file')):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, 'log_file')
elif os.path.isfile(os.path.join(data_path, 'log_file.txt')):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, 'log_file.txt')
else:
# folder for holding multiple server data folders
data_path = self.genPath(name, data_path)
else:
# specified path, use it exactly
assert os.path.isdir(
os.path.dirname(data_path)
), 'The enclosing directory did not exist or was not a directory: %s' % data_path
else:
# just a name
data_path = self.genPath(name, self.cluster.output_folder)
self.data_path = data_path
self._desired_name = name.replace('-', '_')
# - console_output/console_file - can be: path, file-object, True/False
if console_output is None:
# default to stdout
self._console_file_path = None
self._console_file = sys.stdout
elif console_output is False:
# throw away file
self._console_file_path = None
self._console_file = tempfile.NamedTemporaryFile(mode='w+')
elif console_output is True:
# keep it with the data, but create it in the encosing folder until we are running
self._console_file_path = os.path.join(self.data_path,
self._console_file_name)
self._console_file = tempfile.NamedTemporaryFile(
mode='w+', dir=os.path.dirname(self.data_path), delete=False)
elif hasattr(console_output, 'write'):
# file-like object:
self._console_file_path = None
self._console_file = console_output
else:
# a path
assert isinstance(
console_output,
(str, unicode)), 'Unknown console_output: %r' % console_output
console_output = os.path.realpath(console_output)
assert os.path.isdir(
os.path.dirname(console_output)
), 'console_output parent directory was not a folder: %r' % console_output
assert os.path.isfile(console_output) or not os.path.exists(
console_output
), 'console_output location was not useable: %r' % console_output
self._console_file_path = console_output
self._console_file = open(console_output, 'a')
# - server_tags
if server_tags is None:
self.server_tags = []
elif hasattr(server_tags,
'__iter__') and not hasattr(server_tags, 'capitalize'):
self.server_tags = [str(x) for x in server_tags]
else:
self.server_tags = [str(server_tags)]
# - executable_path
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path,
os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
# - command_prefix
if command_prefix is None:
self.command_prefix = []
elif not hasattr(command_prefix, '__iter__'):
raise ValueError(
'command_prefix must be an array of command line options, got: %r'
% command_prefix)
else:
self.command_prefix = command_prefix
# - extra_options
self.options = []
if extra_options is None:
extra_options = []
elif not hasattr(extra_options, '__iter__'):
raise ValueError(
'extra_options must be an array of command line options, got: %r'
% extra_options)
else:
extra_options = [str(x) for x in extra_options]
# -- defaults
if not '--bind' in extra_options and not any(
[x.startswith('--bind=') for x in extra_options]):
self.options += ['--bind', 'all']
if not '--cluster-port' in extra_options and not any(
[x.startswith('--cluster-port=') for x in extra_options]):
self.options += ['--cluster-port', '0']
if not '--driver-port' in extra_options and not any(
[x.startswith('--driver-port=') for x in extra_options]):
self.options += ['--driver-port', '0']
if not '--http-port' in extra_options and not any(
[x.startswith('--http-port=') for x in extra_options]):
self.options += ['--http-port', '0']
for i, option in enumerate(extra_options or []):
if option == '--log-file':
assert len(
self.options
) > i + 1, '--log-file specified in options without a path'
self.logfile_path = utils.translatePath(
os.path.realpath(extra_options[i + 1]))
del extra_options[i + 1]
del extra_options[i]
break
elif option.startswith('--log-file='):
self.logfile_path = utils.translatePath(
os.path.realpath(option[len('--log-file='):].strip('\'"')))
del extra_options[i]
break
else:
self.logfile_path = os.path.join(self.data_path, "log_file.txt")
if not '--no-update-check' in extra_options:
self.options += ['--no-update-check'
] # supress update checks/reporting in
self.options += extra_options
# - subclass modifications
self.subclass_init()
# - save the args
self.args = self.command_prefix + [self.executable_path] + self.options
# -- start the server process
self.start(wait_until_ready=wait_until_ready)
def subclass_options(self, options):
options += ['--directory', utils.translatePath(self.data_path)]
def start(self, wait_until_ready=True):
"""Start up the server"""
global runningServers
assert not self.running, "Trying to start a server where running = %r" % self.running
self.returncode = None
self.killed = False
# -- setup options
# - copy the options
options = copy.copy(self.args)
# - set the local_cluster_port
if not "--client-port" in options and not any([x.startswith("--client-port=") for x in options]):
# allows resunder to know what port to block
options += ["--client-port", str(self.local_cluster_port)]
# - tls options
if self.cluster.tlsKeyPath and self.cluster.tlsCertPath:
options += [
"--http-tls-key",
utils.translatePath(self.cluster.tlsKeyPath),
"--http-tls-cert",
utils.translatePath(self.cluster.tlsCertPath),
"--driver-tls-key",
utils.translatePath(self.cluster.tlsKeyPath),
"--driver-tls-cert",
utils.translatePath(self.cluster.tlsCertPath),
"--cluster-tls-key",
utils.translatePath(self.cluster.tlsKeyPath),
"--cluster-tls-cert",
utils.translatePath(self.cluster.tlsCertPath),
"--cluster-tls-ca",
utils.translatePath(self.cluster.tlsCertPath),
]
if platform.system() == "Darwin":
# because the driver will probably be using the system version of OpenSSL (old), enable TLSv1
options += [
"--tls-min-protocol",
"TLSv1",
"--tls-ciphers",
"EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH:AES256-SHA",
]
# - log file
options += ["--log-file", utils.translatePath(str(self.logfile_path))]
# -- get the length of an exiting log file
if os.path.isfile(self.logfile_path):
self._existing_log_len = os.path.getsize(self.logfile_path)
# -- set to join the cluster
try:
self.cluster._startLock.acquire()
self.cluster._hasStartLock = self
for peer in self.cluster.processes:
if peer != self and peer.ready:
options += ["--join", peer.host + ":" + str(peer.cluster_port)]
break
finally:
# release the lock if we are joining a running cluster
if "--join" in options and self.cluster._hasStartLock is self:
self.cluster._hasStartLock = None
self.cluster._startLock.release()
# -- allow subclasses to modify the options array
self.subclass_options(options) # in-place edit
# -- start the process
try:
self._console_file.write(
"Launching at %s:\n\t%s\n" % (datetime.datetime.now().isoformat(), " ".join(options))
)
self._console_file.flush()
self.process = subprocess.Popen(
options, stdout=self._console_file, stderr=subprocess.STDOUT, preexec_fn=os.setpgrp
)
if not self in runningServers:
runningServers.append(self)
# - start thread to tail output for needed info
thread.start_new_thread(self.read_ports_from_log, ())
except Exception:
self.stop()
raise
finally:
# - move console file if necessary
if self._console_file_path and not os.path.exists(self._console_file_path):
os.rename(self._console_file.name, self._console_file_path)
# -- wait until ready (if requested)
if wait_until_ready:
self.wait_until_ready()
def __init__(
self,
cluster=None,
name=None,
console_output=None,
executable_path=None,
server_tags=None,
command_prefix=None,
extra_options=None,
wait_until_ready=True,
tls=None,
):
global runningServers
# -- validate/default input
# - cluster
assert tls in (None, False, True), "tls must be True, False, or None (False)"
if cluster is None:
cluster = Cluster(tls=tls is True)
else:
assert tls is None or tls is (cluster.tlsCertPath is None), (
"A server added to a cluster must have the same tls setting (cluster: %s vs %s)"
% (cluster.tlsCertPath is None, tls)
)
assert isinstance(cluster, Cluster), "cluster must be a Cluster or None, got: %r" % cluster
self.cluster = cluster
self.cluster.processes.append(self)
# - name/data_path
assert isinstance(name, (str, unicode, None.__class__)), "name was not an expected value: %r" % name
if name is None:
# default name and data_path
name = "%s_%s" % (
self.server_type,
self.cluster.metacluster.get_new_unique_id(server_type=self.server_type),
)
data_path = self.genPath(name, self.cluster.output_folder)
elif name == ".":
# random name in the current directory
name = "%s_%s" % (
self.server_type,
self.cluster.metacluster.get_new_unique_id(server_type=self.server_type),
)
data_path = self.genPath(name, os.path.realpath("."))
elif os.sep in name: # if it looks like a path, it is one, possibly relative
data_path = os.path.abspath(
os.path.join(self.cluster.output_folder, name)
) # note: an absolute path survives this
name = os.path.basename(name)
if os.path.exists(data_path):
assert os.path.isdir(data_path), "The data_path was something other than a directory: %s" % data_path
if os.path.isfile(os.path.join(data_path, "log_file")):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, "log_file")
elif os.path.isfile(os.path.join(data_path, "log_file.txt")):
# existing data folder, record server log file path
self.logfile_path = os.path.join(data_path, "log_file.txt")
else:
# folder for holding multiple server data folders
data_path = self.genPath(name, data_path)
else:
# specified path, use it exactly
assert os.path.isdir(os.path.dirname(data_path)), (
"The enclosing directory did not exist or was not a directory: %s" % data_path
)
else:
# just a name
data_path = self.genPath(name, self.cluster.output_folder)
self.data_path = data_path
self._desired_name = name.replace("-", "_")
# - console_output/console_file - can be: path, file-object, True/False
if console_output is None:
# default to stdout
self._console_file_path = None
self._console_file = sys.stdout
elif console_output is False:
# throw away file
self._console_file_path = None
self._console_file = tempfile.NamedTemporaryFile(mode="w+")
elif console_output is True:
# keep it with the data, but create it in the encosing folder until we are running
self._console_file_path = os.path.join(self.data_path, self._console_file_name)
self._console_file = tempfile.NamedTemporaryFile(
mode="w+", dir=os.path.dirname(self.data_path), delete=False
)
elif hasattr(console_output, "write"):
# file-like object:
self._console_file_path = None
self._console_file = console_output
else:
# a path
assert isinstance(console_output, (str, unicode)), "Unknown console_output: %r" % console_output
console_output = os.path.realpath(console_output)
assert os.path.isdir(os.path.dirname(console_output)), (
"console_output parent directory was not a folder: %r" % console_output
)
assert os.path.isfile(console_output) or not os.path.exists(console_output), (
"console_output location was not useable: %r" % console_output
)
self._console_file_path = console_output
self._console_file = open(console_output, "a")
# - server_tags
if server_tags is None:
self.server_tags = []
elif hasattr(server_tags, "__iter__") and not hasattr(server_tags, "capitalize"):
self.server_tags = [str(x) for x in server_tags]
else:
self.server_tags = [str(server_tags)]
# - executable_path
if executable_path is None:
executable_path = utils.find_rethinkdb_executable()
assert os.access(executable_path, os.X_OK), "no such executable: %r" % executable_path
self.executable_path = executable_path
# - command_prefix
if command_prefix is None:
self.command_prefix = []
elif not hasattr(command_prefix, "__iter__"):
raise ValueError("command_prefix must be an array of command line options, got: %r" % command_prefix)
else:
self.command_prefix = command_prefix
# - extra_options
self.options = []
if extra_options is None:
extra_options = []
elif not hasattr(extra_options, "__iter__"):
raise ValueError("extra_options must be an array of command line options, got: %r" % extra_options)
else:
extra_options = [str(x) for x in extra_options]
# -- defaults
if not "--bind" in extra_options and not any([x.startswith("--bind=") for x in extra_options]):
self.options += ["--bind", "all"]
if not "--cluster-port" in extra_options and not any([x.startswith("--cluster-port=") for x in extra_options]):
self.options += ["--cluster-port", "0"]
if not "--driver-port" in extra_options and not any([x.startswith("--driver-port=") for x in extra_options]):
self.options += ["--driver-port", "0"]
if not "--http-port" in extra_options and not any([x.startswith("--http-port=") for x in extra_options]):
self.options += ["--http-port", "0"]
for i, option in enumerate(extra_options or []):
if option == "--log-file":
assert len(self.options) > i + 1, "--log-file specified in options without a path"
self.logfile_path = utils.translatePath(os.path.realpath(extra_options[i + 1]))
del extra_options[i + 1]
del extra_options[i]
break
elif option.startswith("--log-file="):
self.logfile_path = utils.translatePath(os.path.realpath(option[len("--log-file=") :].strip("'\"")))
del extra_options[i]
break
else:
self.logfile_path = os.path.join(self.data_path, "log_file.txt")
if not "--no-update-check" in extra_options:
self.options += ["--no-update-check"] # supress update checks/reporting in
self.options += extra_options
# - subclass modifications
self.subclass_init()
# - save the args
self.args = self.command_prefix + [self.executable_path] + self.options
# -- start the server process
self.start(wait_until_ready=wait_until_ready)
def subclass_options(self, options):
options += ["--directory", utils.translatePath(self.data_path)]