def _test_connection(self, max_retries=5, messages=None):
"""
Test that the H2O cluster can be reached, and retrieve basic cluster status info.
:param max_retries: Number of times to try to connect to the cluster (with 0.2s intervals).
:returns: Cluster information (an H2OCluster object)
:raises H2OConnectionError, H2OServerError:
"""
if messages is None:
messages = ("Connecting to H2O server at {url} ..", "successful.",
"failed.")
self._print(messages[0].format(url=self._base_url), end="")
cld = None
errors = []
for _ in range(max_retries):
self._print(".", end="", flush=True)
if self._local_server and not self._local_server.is_running():
raise H2OServerError("Local server was unable to start")
try:
define_classes_from_schema(_classes_defined_from_schema_, self)
cld = self.request("GET /3/Cloud")
if self.name and cld.cloud_name != self.name:
raise H2OConnectionError(
"Connected to cloud %s but requested %s." %
(cld.cloud_name, self.name))
if cld.consensus and cld.cloud_healthy:
self._print(" " + messages[1])
return cld
else:
if cld.consensus and not cld.cloud_healthy:
msg = "in consensus but not healthy"
elif not cld.consensus and cld.cloud_healthy:
msg = "not in consensus but healthy"
else:
msg = "not in consensus and not healthy"
errors.append(
"Cloud is in a bad shape: %s (size = %d, bad nodes = %d)"
% (msg, cld.cloud_size, cld.bad_nodes))
except (H2OConnectionError, H2OServerError) as e:
message = str(e)
if "\n" in message: message = message[:message.index("\n")]
errors.append("[%s.%02d] %s: %s" %
(time.strftime("%M:%S"), int(time.time() * 100) %
100, e.__class__.__name__, message))
# Cloud too small, or voting in progress, or server is not up yet; sleep then try again
time.sleep(0.2)
self._print(" " + messages[2])
if cld and not cld.cloud_healthy:
raise H2OServerError("Cluster reports unhealthy status")
if cld and not cld.consensus:
raise H2OServerError("Cluster cannot reach consensus")
else:
raise H2OConnectionError(
"Could not establish link to the H2O cloud %s after %d retries\n%s"
% (self._base_url, max_retries, "\n".join(errors)))
def _process_response(response, save_to):
"""
Given a response object, prepare it to be handed over to the external caller.
Preparation steps include:
* detect if the response has error status, and convert it to an appropriate exception;
* detect Content-Type, and based on that either parse the response as JSON or return as plain text.
"""
status_code = response.status_code
if status_code == 200 and save_to:
if save_to.startswith("~"): save_to = os.path.expanduser(save_to)
if os.path.isdir(save_to) or save_to.endswith(os.path.sep):
dirname = os.path.abspath(save_to)
filename = H2OConnection._find_file_name(response)
else:
dirname, filename = os.path.split(os.path.abspath(save_to))
fullname = os.path.join(dirname, filename)
try:
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(fullname, "wb") as f:
for chunk in response.iter_content(chunk_size=65536):
if chunk: # Empty chunks may occasionally happen
f.write(chunk)
except OSError as e:
raise H2OValueError("Cannot write to file %s: %s" %
(fullname, e))
return fullname
content_type = response.headers.get("Content-Type", "")
if ";" in content_type: # Remove a ";charset=..." part
content_type = content_type[:content_type.index(";")]
# Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is.
if content_type == "application/json":
try:
data = response.json(object_pairs_hook=H2OResponse)
except (JSONDecodeError,
requests.exceptions.ContentDecodingError) as e:
raise H2OServerError("Malformed JSON from server (%s):\n%s" %
(str(e), response.text))
else:
data = response.text
# Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content")
if status_code in {200, 201, 202, 204}:
return data
# Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed")
if status_code in {400, 404, 412} and isinstance(
data, (H2OErrorV3, H2OModelBuilderErrorV3)):
raise H2OResponseError(data)
# Server errors (notably 500 = "Server Error")
# Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server
# did not provide the correct status code.
raise H2OServerError("HTTP %d %s:\n%r" %
(status_code, response.reason, data))
def _process_response(response):
"""
Given a response object, prepare it to be handed over to the external caller.
Preparation steps include:
* detect if the response has error status, and convert it to an appropriate exception;
* detect Content-Type, and based on that either parse the response as JSON or return as plain text.
"""
content_type = response.headers[
"Content-Type"] if "Content-Type" in response.headers else ""
if ";" in content_type: # Remove a ";charset=..." part
content_type = content_type[:content_type.index(";")]
status_code = response.status_code
# Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is.
if content_type == "application/json":
try:
data = response.json(object_pairs_hook=H2OResponse)
except (JSONDecodeError,
requests.exceptions.ContentDecodingError) as e:
raise H2OServerError("Malformed JSON from server (%s):\n%s" %
(str(e), response.text))
else:
data = response.text
# Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content")
if status_code in {200, 201, 202, 204}:
return data
# Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed")
if status_code in {400, 404, 412} and isinstance(
data, (H2OErrorV3, H2OModelBuilderErrorV3)):
raise H2OResponseError(data)
# Server errors (notably 500 = "Server Error")
# Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server
# did not provide the correct status code.
raise H2OServerError("HTTP %d %s:\n%r" %
(status_code, response.reason, data))
def _launch_server(self,
port,
baseport,
mmax,
mmin,
ea,
nthreads,
jvm_custom_args,
bind_to_localhost,
log_dir=None,
log_level=None,
max_log_file_size=None):
"""Actually start the h2o.jar executable (helper method for `.start()`)."""
self._ip = "127.0.0.1"
# Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object)
java = self._find_java()
self._check_java(java, self._verbose)
if self._verbose:
print(" Starting server from " + self._jar_path)
print(" Ice root: " + self._ice_root)
# Construct java command to launch the process
cmd = [java]
# ...add JVM options
cmd += ["-ea"] if ea else []
for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]:
if num is None: continue
numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \
"%dM" % (num >> 20) if num == (num >> 20) << 20 else \
str(num)
cmd += [mq + numstr]
if jvm_custom_args is not None:
for arg in jvm_custom_args:
assert type(arg) is str
cmd += [arg]
# This should be the last JVM option
if self._extra_classpath is None:
# Use jar file directly
cmd += ["-jar", self._jar_path]
else:
# Combine jar path with the optional extra classpath
classpath = [self._jar_path] + self._extra_classpath
cmd += ["-cp", os.pathsep.join(classpath), "water.H2OApp"]
# ...add H2O options
cmd += ["-ip", self._ip]
if bind_to_localhost:
cmd += ["-web_ip", self._ip]
cmd += ["-port", str(port)] if port else []
cmd += ["-baseport", str(baseport)] if baseport else []
cmd += ["-ice_root", self._ice_root]
cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else []
if log_dir:
cmd += ["-log_dir", log_dir]
if log_level:
cmd += ["-log_level", log_level]
if max_log_file_size:
cmd += ["-max_log_file_size", max_log_file_size]
if not self._name:
self._name = "H2O_from_python_%s" % self._tmp_file("salt")
cmd += ["-name", self._name]
# Warning: do not change to any higher log-level, otherwise we won't be able to know which port the
# server is listening to.
cmd += ["-log_level", "INFO"]
cmd += ["-allow_unsupported_java"]
# Create stdout and stderr files
self._stdout = self._tmp_file("stdout")
self._stderr = self._tmp_file("stderr")
cwd = os.path.abspath(os.getcwd())
out = open(self._stdout, "w", encoding='utf-8')
err = open(self._stderr, "w", encoding='utf-8')
if self._verbose:
print(" JVM stdout: " + out.name)
print(" JVM stderr: " + err.name)
# Launch the process
win32 = sys.platform == "win32"
flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP",
0) if win32 else 0
prex = os.setsid if not win32 else None
try:
proc = subprocess.Popen(args=cmd,
stdout=out,
stderr=err,
cwd=cwd,
creationflags=flags,
preexec_fn=prex)
except OSError as e:
traceback = getattr(e, "child_traceback", None)
raise H2OServerError("Unable to start server: %s" % e, traceback)
# Wait until the server is up-and-running
giveup_time = time.time() + self._TIME_TO_START
while True:
if proc.poll() is not None:
if proc.returncode == self._BAD_JAVA_VERSION_RETURN_CODE_:
error_message = "Server process terminated because of unsupported Java version"
else:
error_message = "Server process terminated with error code %d" % proc.returncode
if os.stat(self._stderr).st_size > 0:
error_message += ": %s" % open(self._stderr,
encoding='utf-8').read()
else:
error_message += "."
raise H2OServerError(error_message)
ret = self._get_server_info_from_logs()
if ret:
self._scheme = ret[0]
self._ip = ret[1]
self._port = ret[2]
self._process = proc
break
if time.time() > giveup_time:
elapsed_time = time.time() - (giveup_time -
self._TIME_TO_START)
raise H2OServerError(
"Server wasn't able to start in %f seconds." %
elapsed_time)
time.sleep(0.2)
def _launch_server(self, port, baseport, mmax, mmin, ea, nthreads):
"""Actually start the h2o.jar executable (helper method for `.start()`)."""
self._ip = "127.0.0.1"
# Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object)
java = self._find_java()
jver_bytes = subprocess.check_output([java, "-version"], stderr=subprocess.STDOUT)
jver = jver_bytes.decode(encoding="utf-8", errors="ignore")
if self._verbose:
print(" Java Version: " + jver.strip().replace("\n", "; "))
if "GNU libgcj" in jver:
raise H2OStartupError("Sorry, GNU Java is not supported for H2O.\n"
"Please download the latest 64-bit Java SE JDK from Oracle.")
if "Client VM" in jver:
warn(" You have a 32-bit version of Java. H2O works best with 64-bit Java.\n"
" Please download the latest 64-bit Java SE JDK from Oracle.\n")
if self._verbose:
print(" Starting server from " + self._jar_path)
print(" Ice root: " + self._ice_root)
# Construct java command to launch the process
cmd = [java]
# ...add JVM options
cmd += ["-ea"] if ea else []
for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]:
if num is None: continue
numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \
"%dM" % (num >> 20) if num == (num >> 20) << 20 else \
str(num)
cmd += [mq + numstr]
cmd += ["-verbose:gc", "-XX:+PrintGCDetails", "-XX:+PrintGCTimeStamps"]
cmd += ["-jar", self._jar_path] # This should be the last JVM option
# ...add H2O options
cmd += ["-ip", self._ip]
cmd += ["-port", str(port)] if port else []
cmd += ["-baseport", str(baseport)] if baseport else []
cmd += ["-ice_root", self._ice_root]
cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else []
cmd += ["-name", "H2O_from_python_%s" % self._tmp_file("salt")]
# Warning: do not change to any higher log-level, otherwise we won't be able to know which port the
# server is listening to.
cmd += ["-log_level", "INFO"]
# Create stdout and stderr files
self._stdout = self._tmp_file("stdout")
self._stderr = self._tmp_file("stderr")
cwd = os.path.abspath(os.getcwd())
out = open(self._stdout, "w")
err = open(self._stderr, "w")
if self._verbose:
print(" JVM stdout: " + out.name)
print(" JVM stderr: " + err.name)
# Launch the process
win32 = sys.platform == "win32"
flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if win32 else 0
prex = os.setsid if not win32 else None
try:
proc = subprocess.Popen(args=cmd, stdout=out, stderr=err, cwd=cwd, creationflags=flags, preexec_fn=prex)
except OSError as e:
traceback = getattr(e, "child_traceback", None)
raise H2OServerError("Unable to start server: %s" % e, traceback)
# Wait until the server is up-and-running
giveup_time = time.time() + self._TIME_TO_START
while True:
if proc.poll() is not None:
raise H2OServerError("Server process terminated with error code %d" % proc.returncode)
ret = self._get_server_info_from_logs()
if ret:
self._scheme = ret[0]
self._ip = ret[1]
self._port = ret[2]
self._process = proc
break
if time.time() > giveup_time:
elapsed_time = time.time() - (giveup_time - self._TIME_TO_START)
raise H2OServerError("Server wasn't able to start in %f seconds." % elapsed_time)
time.sleep(0.2)