def close(self): logger.info("Closing down client") quiet_close(self.stream) quiet_shutdown(self.socket) quiet_close(self.socket) self.socket = None self.stream = None
def testRedirectToFile(self): (_, outpath) = tempfile.mkstemp(text=True) (_, errpath) = tempfile.mkstemp(text=True) stdout = open(outpath, "w") stderr = open(errpath, "w") try: self.gateway = JavaGateway.launch_gateway( redirect_stdout=stdout, redirect_stderr=stderr) for i in range(10): self.gateway.jvm.System.out.println("Test") self.gateway.jvm.System.err.println("Test2") # Should not be necessary quiet_close(stdout) quiet_close(stderr) # Test that the redirect files were written to correctly with open(outpath, "r") as stdout: lines = stdout.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test\n", lines[0]) with open(errpath, "r") as stderr: lines = stderr.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test2\n", lines[0]) finally: os.unlink(outpath) os.unlink(errpath)
def testRedirectToFile(self): (_, outpath) = tempfile.mkstemp(text=True) (_, errpath) = tempfile.mkstemp(text=True) stdout = open(outpath, "w") stderr = open(errpath, "w") try: self.gateway = JavaGateway.launch_gateway(redirect_stdout=stdout, redirect_stderr=stderr) for i in range(10): self.gateway.jvm.System.out.println("Test") self.gateway.jvm.System.err.println("Test2") # Should not be necessary quiet_close(stdout) quiet_close(stderr) # Test that the redirect files were written to correctly with open(outpath, "r") as stdout: lines = stdout.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test\n", lines[0]) with open(errpath, "r") as stderr: lines = stderr.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test2\n", lines[0]) finally: os.unlink(outpath) os.unlink(errpath)
def close(self): logger.info("Closing down client") quiet_close(self.stream) quiet_shutdown(self.socket) quiet_close(self.socket) self.socket = None self.stream = None
def close(self, reset=False): logger.info("Closing down clientserver connection") if not self.socket: return if reset: set_linger(self.socket) quiet_close(self.stream) if not reset: quiet_shutdown(self.socket) quiet_close(self.socket) already_closed = self.socket is None self.socket = None self.stream = None if not self.initiated_from_client and self.python_server and not already_closed: server_connection_stopped.send(self.python_server, connection=self)
def close(self, reset=False): logger.info("Closing down clientserver connection") if not self.socket: return if reset: set_linger(self.socket) quiet_close(self.stream) if not reset: quiet_shutdown(self.socket) quiet_close(self.socket) already_closed = self.socket is None self.socket = None self.stream = None if not self.initiated_from_client and self.python_server and\ not already_closed: server_connection_stopped.send(self.python_server, connection=self)
def connect_to_java_server(self): try: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.ssl_context: self.socket = self.ssl_context.wrap_socket( self.socket, server_hostname=self.java_address) self.socket.connect((self.java_address, self.java_port)) self.stream = self.socket.makefile("rb", 0) self.is_connected = True self.initiated_from_client = True except Exception: quiet_close(self.socket) quiet_close(self.stream) self.socket = None self.stream = None self.is_connected = False raise
def connect_to_java_server(self): try: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if self.ssl_context: self.socket = self.ssl_context.wrap_socket( self.socket, server_hostname=self.java_address) self.socket.connect((self.java_address, self.java_port)) self.stream = self.socket.makefile("rb", 0) self.is_connected = True self.initiated_from_client = True except Exception: quiet_close(self.socket) quiet_close(self.stream) self.socket = None self.stream = None self.is_connected = False raise
def shutdown_gateway(self): """Sends a shutdown command to the Java side. This will close the ClientServer on the Java side: all active connections will be closed. This may be useful if the lifecycle of the Java program must be tied to the Python program. """ if not self.is_connected: raise Py4JError("Gateway must be connected to send shutdown cmd.") try: quiet_close(self.stream) self.socket.sendall(proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8")) self.close() except Exception: # Do nothing! Exceptions might occur anyway. logger.debug("Exception occurred while shutting down gateway", exc_info=True)
def shutdown_gateway(self): """Sends a shutdown command to the Java side. This will close the ClientServer on the Java side: all active connections will be closed. This may be useful if the lifecycle of the Java program must be tied to the Python program. """ if not self.is_connected: raise Py4JError("Gateway must be connected to send shutdown cmd.") try: quiet_close(self.stream) self.socket.sendall( proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8")) self.close() except Exception: # Do nothing! Exceptions might occur anyway. logger.debug("Exception occurred while shutting down gateway", exc_info=True)
def clean_closed_connections(): from py4j.java_gateway import quiet_close, quiet_shutdown callback_server = self._gateway._callback_server with callback_server.lock: try: closed_connections = [] for connection in callback_server.connections: if not connection.isAlive(): quiet_close(connection.input) quiet_shutdown(connection.socket) quiet_close(connection.socket) closed_connections.append(connection) for closed_connection in closed_connections: callback_server.connections.remove(closed_connection) except Exception: import traceback traceback.print_exc() self._start_timer(clean_closed_connections)
def clean_closed_connections(): from py4j.java_gateway import quiet_close, quiet_shutdown callback_server = self._gateway._callback_server if callback_server: with callback_server.lock: try: closed_connections = [] for connection in callback_server.connections: if not connection.isAlive(): quiet_close(connection.input) quiet_shutdown(connection.socket) quiet_close(connection.socket) closed_connections.append(connection) for closed_connection in closed_connections: callback_server.connections.remove(closed_connection) except Exception: import traceback traceback.print_exc() self._start_timer(clean_closed_connections)
def testRedirectToFile(self): end = os.linesep (out_handle, outpath) = tempfile.mkstemp(text=True) (err_handle, errpath) = tempfile.mkstemp(text=True) stdout = open(outpath, "w") stderr = open(errpath, "w") try: self.gateway = JavaGateway.launch_gateway( redirect_stdout=stdout, redirect_stderr=stderr) for i in range(10): self.gateway.jvm.System.out.println("Test") self.gateway.jvm.System.err.println("Test2") self.gateway.shutdown() sleep() # Should not be necessary quiet_close(stdout) quiet_close(stderr) # Test that the redirect files were written to correctly with open(outpath, "r") as stdout: lines = stdout.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test{0}".format(end), lines[0]) with open(errpath, "r") as stderr: lines = stderr.readlines() if not stderr_is_polluted(lines[0]): self.assertEqual(10, len(lines)) # XXX Apparently, it's \n by default even on windows... # Go figure self.assertEqual("Test2\n", lines[0]) finally: os.close(out_handle) os.close(err_handle) os.unlink(outpath) os.unlink(errpath)
def testRedirectToFile(self): end = os.linesep (out_handle, outpath) = tempfile.mkstemp(text=True) (err_handle, errpath) = tempfile.mkstemp(text=True) stdout = open(outpath, "w") stderr = open(errpath, "w") try: self.gateway = JavaGateway.launch_gateway(redirect_stdout=stdout, redirect_stderr=stderr) for i in range(10): self.gateway.jvm.System.out.println("Test") self.gateway.jvm.System.err.println("Test2") self.gateway.shutdown() sleep() # Should not be necessary quiet_close(stdout) quiet_close(stderr) # Test that the redirect files were written to correctly with open(outpath, "r") as stdout: lines = stdout.readlines() self.assertEqual(10, len(lines)) self.assertEqual("Test{0}".format(end), lines[0]) with open(errpath, "r") as stderr: lines = stderr.readlines() if not stderr_is_polluted(lines[0]): self.assertEqual(10, len(lines)) # XXX Apparently, it's \n by default even on windows... # Go figure self.assertEqual("Test2\n", lines[0]) finally: os.close(out_handle) os.close(err_handle) os.unlink(outpath) os.unlink(errpath)
def launch_gateway(port=0, jarpath="", classpath="", javaopts=[], die_on_exit=False, redirect_stdout=None, redirect_stderr=None, daemonize_redirect=True, java_path="java", create_new_process_group=False, enable_auth=False, return_proc=False): """Launch a `Gateway` in a new Java process. The redirect parameters accept file-like objects, Queue, or deque. When text lines are sent to the stdout or stderr of the child JVM, these lines are redirected to the file-like object (``write(line)``), the Queue (``put(line)``), or the deque (``appendleft(line)``). The text line will contain a newline character. Only text output is accepted on stdout and stderr. If you wish to communicate with the child JVM through bytes, you need to create your own helper function. :param port: the port to launch the Java Gateway on. If no port is specified then an ephemeral port is used. :param jarpath: the path to the Py4J jar. Only necessary if the jar was installed at a non-standard location or if Python is using a different `sys.prefix` than the one that Py4J was installed under. :param classpath: the classpath used to launch the Java Gateway. :param javaopts: an array of extra options to pass to Java (the classpath should be specified using the `classpath` parameter, not `javaopts`.) :param die_on_exit: if `True`, the Java gateway process will die when this Python process exits or is killed. :param redirect_stdout: where to redirect the JVM stdout. If None (default) stdout is redirected to os.devnull. Otherwise accepts a file descriptor, a queue, or a deque. Will send one line at a time to these objects. :param redirect_stderr: where to redirect the JVM stdout. If None (default) stderr is redirected to os.devnull. Otherwise accepts a file descriptor, a queue, or a deque. Will send one line at a time to these objects. :param daemonize_redirect: if True, the consumer threads will be daemonized and will not prevent the main Python process from exiting. This means the file descriptors (stderr, stdout, redirect_stderr, redirect_stdout) might not be properly closed. This is not usually a problem, but in case of errors related to file descriptors, set this flag to False. :param java_path: If None, Py4J will use $JAVA_HOME/bin/java if $JAVA_HOME is defined, otherwise it will use "java". :param create_new_process_group: If True, the JVM is started in a new process group. This ensures that signals sent to the parent Python process are not forwarded to the JVM. For example, sending Ctrl-C/SIGINT won't interrupt the JVM. If the python process dies, the Java process will stay alive, which may be a problem for some scenarios though. :param enable_auth: If True, the server will require clients to provide an authentication token when connecting. :param return_proc: If True, returns the Popen object returned when the JVM process was created. :rtype: the port number of the `Gateway` server or, when auth enabled, a 2-tuple with the port number and the auth token. """ popen_kwargs = {} if not jarpath: jarpath = find_jar_path() if not java_path: java_home = os.environ.get("JAVA_HOME") if java_home: java_path = os.path.join(java_home, "bin", "java") else: java_path = "java" # Fail if the jar does not exist. if not os.path.exists(jarpath): raise Py4JError("Could not find py4j jar at {0}".format(jarpath)) # Launch the server in a subprocess. classpath = os.pathsep.join((jarpath, classpath)) command = [java_path, "-classpath", classpath] + javaopts + \ ["py4j.GatewayServer"] if die_on_exit: command.append("--die-on-broken-pipe") if enable_auth: command.append("--enable-auth") command.append(str(port)) logger.debug("Launching gateway with command {0}".format(command)) # stderr redirection close_stderr = False if redirect_stderr is None: stderr = open(os.devnull, "w") close_stderr = True elif isinstance(redirect_stderr, Queue) or\ isinstance(redirect_stderr, deque): stderr = PIPE else: stderr = redirect_stderr # we don't need this anymore redirect_stderr = None # stdout redirection if redirect_stdout is None: redirect_stdout = open(os.devnull, "w") if create_new_process_group: popen_kwargs.update(get_create_new_process_group_kwargs()) proc = Popen(command, stdout=PIPE, stdin=PIPE, stderr=stderr, **popen_kwargs) # Determine which port the server started on (needed to support # ephemeral ports) _port = int(proc.stdout.readline()) # Read the auth token from the server if enabled. _auth_token = None if enable_auth: _auth_token = proc.stdout.readline()[:-1] # Start consumer threads so process does not deadlock/hangs OutputConsumer(redirect_stdout, proc.stdout, daemon=daemonize_redirect).start() if redirect_stderr is not None: OutputConsumer(redirect_stderr, proc.stderr, daemon=daemonize_redirect).start() ProcessConsumer(proc, [redirect_stdout], daemon=daemonize_redirect).start() if close_stderr: # XXX This will quiet ResourceWarning in Python 3.5+ # This only close the fd in this process, not in the JVM process, which # makes sense. quiet_close(stderr) if enable_auth: output = (_port, _auth_token) else: output = _port if return_proc: if isinstance(output, tuple): output = output + (proc, ) else: output = (_port, proc) return output
def start(): global __pythonManager, __gateway, __dependency_manager, __gmql_jar_path, __py4j_path logger = logging.getLogger() master = get_master() if master.lower().startswith('local'): logger.debug("Starting LOCAL backend (master: {})".format( master.lower())) java_home = os.environ.get("JAVA_HOME") if java_home is None: raise SystemError("The environment variable JAVA_HOME is not set") java_path = os.path.join(java_home, "bin", "java") _port = launch_gateway(classpath=__gmql_jar_path, die_on_exit=True, java_path=java_path, javaopts=get_local_java_options(), jarpath=__py4j_path) __gateway = JavaGateway(gateway_parameters=GatewayParameters( port=_port, auto_convert=True)) python_api_package = get_python_api_package(__gateway) __pythonManager = start_gmql_manager(python_api_package) conf = get_configuration() conf.set_master(master.lower()) _set_spark_configuration(conf) _set_system_configuration(conf) else: # use spark-submit logger.debug("Submitting backend to {}".format(master)) master = re.sub("^spark_", "", master.lower()) configs = get_spark_configs() spark_location = find() logger.debug("Found spark at location: {}".format(spark_location)) command = [ os.path.join(spark_location, 'bin', 'spark-submit'), '--master', master, '--deploy-mode', "client" ] for cname, c in configs.items(): command.extend(['--conf', '{}={}'.format(cname, c)]) command.append(__gmql_jar_path) stderr = open(os.devnull, "w") proc = Popen(command, stdout=PIPE, stdin=PIPE, stderr=stderr) while True: try: _port = int(proc.stdout.readline()) break except ValueError: pass logger.debug("Backend listening at port {}".format(_port)) redirect_stdout = open(os.devnull, "w") OutputConsumer(redirect_stdout, proc.stdout, daemon=True).start() ProcessConsumer(proc, [redirect_stdout], daemon=True).start() quiet_close(stderr) __gateway = JavaGateway(gateway_parameters=GatewayParameters( port=_port, auto_convert=True)) pm = __gateway.entry_point.getPythonManager() pm.startEngine() __pythonManager = pm