def _test_connection(self, max_retries=5, messages=None): """ Test that the H2O cluster can be reached, and retrieve basic cluster status info. :param max_retries: Number of times to try to connect to the cluster (with 0.2s intervals). :returns: Cluster information (an H2OCluster object) :raises H2OConnectionError, H2OServerError: """ if messages is None: messages = ("Connecting to H2O server at {url} ..", "successful.", "failed.") self._print(messages[0].format(url=self._base_url), end="") cld = None errors = [] for _ in range(max_retries): self._print(".", end="", flush=True) if self._local_server and not self._local_server.is_running(): raise H2OServerError("Local server was unable to start") try: define_classes_from_schema(_classes_defined_from_schema_, self) cld = self.request("GET /3/Cloud") if self.name and cld.cloud_name != self.name: raise H2OConnectionError( "Connected to cloud %s but requested %s." % (cld.cloud_name, self.name)) if cld.consensus and cld.cloud_healthy: self._print(" " + messages[1]) return cld else: if cld.consensus and not cld.cloud_healthy: msg = "in consensus but not healthy" elif not cld.consensus and cld.cloud_healthy: msg = "not in consensus but healthy" else: msg = "not in consensus and not healthy" errors.append( "Cloud is in a bad shape: %s (size = %d, bad nodes = %d)" % (msg, cld.cloud_size, cld.bad_nodes)) except (H2OConnectionError, H2OServerError) as e: message = str(e) if "\n" in message: message = message[:message.index("\n")] errors.append("[%s.%02d] %s: %s" % (time.strftime("%M:%S"), int(time.time() * 100) % 100, e.__class__.__name__, message)) # Cloud too small, or voting in progress, or server is not up yet; sleep then try again time.sleep(0.2) self._print(" " + messages[2]) if cld and not cld.cloud_healthy: raise H2OServerError("Cluster reports unhealthy status") if cld and not cld.consensus: raise H2OServerError("Cluster cannot reach consensus") else: raise H2OConnectionError( "Could not establish link to the H2O cloud %s after %d retries\n%s" % (self._base_url, max_retries, "\n".join(errors)))
def _process_response(response, save_to): """ Given a response object, prepare it to be handed over to the external caller. Preparation steps include: * detect if the response has error status, and convert it to an appropriate exception; * detect Content-Type, and based on that either parse the response as JSON or return as plain text. """ status_code = response.status_code if status_code == 200 and save_to: if save_to.startswith("~"): save_to = os.path.expanduser(save_to) if os.path.isdir(save_to) or save_to.endswith(os.path.sep): dirname = os.path.abspath(save_to) filename = H2OConnection._find_file_name(response) else: dirname, filename = os.path.split(os.path.abspath(save_to)) fullname = os.path.join(dirname, filename) try: if not os.path.exists(dirname): os.makedirs(dirname) with open(fullname, "wb") as f: for chunk in response.iter_content(chunk_size=65536): if chunk: # Empty chunks may occasionally happen f.write(chunk) except OSError as e: raise H2OValueError("Cannot write to file %s: %s" % (fullname, e)) return fullname content_type = response.headers.get("Content-Type", "") if ";" in content_type: # Remove a ";charset=..." part content_type = content_type[:content_type.index(";")] # Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is. if content_type == "application/json": try: data = response.json(object_pairs_hook=H2OResponse) except (JSONDecodeError, requests.exceptions.ContentDecodingError) as e: raise H2OServerError("Malformed JSON from server (%s):\n%s" % (str(e), response.text)) else: data = response.text # Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content") if status_code in {200, 201, 202, 204}: return data # Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed") if status_code in {400, 404, 412} and isinstance( data, (H2OErrorV3, H2OModelBuilderErrorV3)): raise H2OResponseError(data) # Server errors (notably 500 = "Server Error") # Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server # did not provide the correct status code. raise H2OServerError("HTTP %d %s:\n%r" % (status_code, response.reason, data))
def _process_response(response): """ Given a response object, prepare it to be handed over to the external caller. Preparation steps include: * detect if the response has error status, and convert it to an appropriate exception; * detect Content-Type, and based on that either parse the response as JSON or return as plain text. """ content_type = response.headers[ "Content-Type"] if "Content-Type" in response.headers else "" if ";" in content_type: # Remove a ";charset=..." part content_type = content_type[:content_type.index(";")] status_code = response.status_code # Auto-detect response type by its content-type. Decode JSON, all other responses pass as-is. if content_type == "application/json": try: data = response.json(object_pairs_hook=H2OResponse) except (JSONDecodeError, requests.exceptions.ContentDecodingError) as e: raise H2OServerError("Malformed JSON from server (%s):\n%s" % (str(e), response.text)) else: data = response.text # Success (200 = "Ok", 201 = "Created", 202 = "Accepted", 204 = "No Content") if status_code in {200, 201, 202, 204}: return data # Client errors (400 = "Bad Request", 404 = "Not Found", 412 = "Precondition Failed") if status_code in {400, 404, 412} and isinstance( data, (H2OErrorV3, H2OModelBuilderErrorV3)): raise H2OResponseError(data) # Server errors (notably 500 = "Server Error") # Note that it is possible to receive valid H2OErrorV3 object in this case, however it merely means the server # did not provide the correct status code. raise H2OServerError("HTTP %d %s:\n%r" % (status_code, response.reason, data))
def _launch_server(self, port, baseport, mmax, mmin, ea, nthreads, jvm_custom_args, bind_to_localhost, log_dir=None, log_level=None, max_log_file_size=None): """Actually start the h2o.jar executable (helper method for `.start()`).""" self._ip = "127.0.0.1" # Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object) java = self._find_java() self._check_java(java, self._verbose) if self._verbose: print(" Starting server from " + self._jar_path) print(" Ice root: " + self._ice_root) # Construct java command to launch the process cmd = [java] # ...add JVM options cmd += ["-ea"] if ea else [] for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]: if num is None: continue numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \ "%dM" % (num >> 20) if num == (num >> 20) << 20 else \ str(num) cmd += [mq + numstr] if jvm_custom_args is not None: for arg in jvm_custom_args: assert type(arg) is str cmd += [arg] # This should be the last JVM option if self._extra_classpath is None: # Use jar file directly cmd += ["-jar", self._jar_path] else: # Combine jar path with the optional extra classpath classpath = [self._jar_path] + self._extra_classpath cmd += ["-cp", os.pathsep.join(classpath), "water.H2OApp"] # ...add H2O options cmd += ["-ip", self._ip] if bind_to_localhost: cmd += ["-web_ip", self._ip] cmd += ["-port", str(port)] if port else [] cmd += ["-baseport", str(baseport)] if baseport else [] cmd += ["-ice_root", self._ice_root] cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else [] if log_dir: cmd += ["-log_dir", log_dir] if log_level: cmd += ["-log_level", log_level] if max_log_file_size: cmd += ["-max_log_file_size", max_log_file_size] if not self._name: self._name = "H2O_from_python_%s" % self._tmp_file("salt") cmd += ["-name", self._name] # Warning: do not change to any higher log-level, otherwise we won't be able to know which port the # server is listening to. cmd += ["-log_level", "INFO"] cmd += ["-allow_unsupported_java"] # Create stdout and stderr files self._stdout = self._tmp_file("stdout") self._stderr = self._tmp_file("stderr") cwd = os.path.abspath(os.getcwd()) out = open(self._stdout, "w", encoding='utf-8') err = open(self._stderr, "w", encoding='utf-8') if self._verbose: print(" JVM stdout: " + out.name) print(" JVM stderr: " + err.name) # Launch the process win32 = sys.platform == "win32" flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if win32 else 0 prex = os.setsid if not win32 else None try: proc = subprocess.Popen(args=cmd, stdout=out, stderr=err, cwd=cwd, creationflags=flags, preexec_fn=prex) except OSError as e: traceback = getattr(e, "child_traceback", None) raise H2OServerError("Unable to start server: %s" % e, traceback) # Wait until the server is up-and-running giveup_time = time.time() + self._TIME_TO_START while True: if proc.poll() is not None: if proc.returncode == self._BAD_JAVA_VERSION_RETURN_CODE_: error_message = "Server process terminated because of unsupported Java version" else: error_message = "Server process terminated with error code %d" % proc.returncode if os.stat(self._stderr).st_size > 0: error_message += ": %s" % open(self._stderr, encoding='utf-8').read() else: error_message += "." raise H2OServerError(error_message) ret = self._get_server_info_from_logs() if ret: self._scheme = ret[0] self._ip = ret[1] self._port = ret[2] self._process = proc break if time.time() > giveup_time: elapsed_time = time.time() - (giveup_time - self._TIME_TO_START) raise H2OServerError( "Server wasn't able to start in %f seconds." % elapsed_time) time.sleep(0.2)
def _launch_server(self, port, baseport, mmax, mmin, ea, nthreads): """Actually start the h2o.jar executable (helper method for `.start()`).""" self._ip = "127.0.0.1" # Find Java and check version. (Note that subprocess.check_output returns the output as a bytes object) java = self._find_java() jver_bytes = subprocess.check_output([java, "-version"], stderr=subprocess.STDOUT) jver = jver_bytes.decode(encoding="utf-8", errors="ignore") if self._verbose: print(" Java Version: " + jver.strip().replace("\n", "; ")) if "GNU libgcj" in jver: raise H2OStartupError("Sorry, GNU Java is not supported for H2O.\n" "Please download the latest 64-bit Java SE JDK from Oracle.") if "Client VM" in jver: warn(" You have a 32-bit version of Java. H2O works best with 64-bit Java.\n" " Please download the latest 64-bit Java SE JDK from Oracle.\n") if self._verbose: print(" Starting server from " + self._jar_path) print(" Ice root: " + self._ice_root) # Construct java command to launch the process cmd = [java] # ...add JVM options cmd += ["-ea"] if ea else [] for (mq, num) in [("-Xms", mmin), ("-Xmx", mmax)]: if num is None: continue numstr = "%dG" % (num >> 30) if num == (num >> 30) << 30 else \ "%dM" % (num >> 20) if num == (num >> 20) << 20 else \ str(num) cmd += [mq + numstr] cmd += ["-verbose:gc", "-XX:+PrintGCDetails", "-XX:+PrintGCTimeStamps"] cmd += ["-jar", self._jar_path] # This should be the last JVM option # ...add H2O options cmd += ["-ip", self._ip] cmd += ["-port", str(port)] if port else [] cmd += ["-baseport", str(baseport)] if baseport else [] cmd += ["-ice_root", self._ice_root] cmd += ["-nthreads", str(nthreads)] if nthreads > 0 else [] cmd += ["-name", "H2O_from_python_%s" % self._tmp_file("salt")] # Warning: do not change to any higher log-level, otherwise we won't be able to know which port the # server is listening to. cmd += ["-log_level", "INFO"] # Create stdout and stderr files self._stdout = self._tmp_file("stdout") self._stderr = self._tmp_file("stderr") cwd = os.path.abspath(os.getcwd()) out = open(self._stdout, "w") err = open(self._stderr, "w") if self._verbose: print(" JVM stdout: " + out.name) print(" JVM stderr: " + err.name) # Launch the process win32 = sys.platform == "win32" flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) if win32 else 0 prex = os.setsid if not win32 else None try: proc = subprocess.Popen(args=cmd, stdout=out, stderr=err, cwd=cwd, creationflags=flags, preexec_fn=prex) except OSError as e: traceback = getattr(e, "child_traceback", None) raise H2OServerError("Unable to start server: %s" % e, traceback) # Wait until the server is up-and-running giveup_time = time.time() + self._TIME_TO_START while True: if proc.poll() is not None: raise H2OServerError("Server process terminated with error code %d" % proc.returncode) ret = self._get_server_info_from_logs() if ret: self._scheme = ret[0] self._ip = ret[1] self._port = ret[2] self._process = proc break if time.time() > giveup_time: elapsed_time = time.time() - (giveup_time - self._TIME_TO_START) raise H2OServerError("Server wasn't able to start in %f seconds." % elapsed_time) time.sleep(0.2)