def close(self):
logger.info("Closing down client")
quiet_close(self.stream)
quiet_shutdown(self.socket)
quiet_close(self.socket)
self.socket = None
self.stream = None
def testRedirectToFile(self):
(_, outpath) = tempfile.mkstemp(text=True)
(_, errpath) = tempfile.mkstemp(text=True)
stdout = open(outpath, "w")
stderr = open(errpath, "w")
try:
self.gateway = JavaGateway.launch_gateway(
redirect_stdout=stdout, redirect_stderr=stderr)
for i in range(10):
self.gateway.jvm.System.out.println("Test")
self.gateway.jvm.System.err.println("Test2")
# Should not be necessary
quiet_close(stdout)
quiet_close(stderr)
# Test that the redirect files were written to correctly
with open(outpath, "r") as stdout:
lines = stdout.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test\n", lines[0])
with open(errpath, "r") as stderr:
lines = stderr.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test2\n", lines[0])
finally:
os.unlink(outpath)
os.unlink(errpath)
def testRedirectToFile(self):
(_, outpath) = tempfile.mkstemp(text=True)
(_, errpath) = tempfile.mkstemp(text=True)
stdout = open(outpath, "w")
stderr = open(errpath, "w")
try:
self.gateway = JavaGateway.launch_gateway(redirect_stdout=stdout,
redirect_stderr=stderr)
for i in range(10):
self.gateway.jvm.System.out.println("Test")
self.gateway.jvm.System.err.println("Test2")
# Should not be necessary
quiet_close(stdout)
quiet_close(stderr)
# Test that the redirect files were written to correctly
with open(outpath, "r") as stdout:
lines = stdout.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test\n", lines[0])
with open(errpath, "r") as stderr:
lines = stderr.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test2\n", lines[0])
finally:
os.unlink(outpath)
os.unlink(errpath)
def close(self):
logger.info("Closing down client")
quiet_close(self.stream)
quiet_shutdown(self.socket)
quiet_close(self.socket)
self.socket = None
self.stream = None
def close(self, reset=False):
logger.info("Closing down clientserver connection")
if not self.socket:
return
if reset:
set_linger(self.socket)
quiet_close(self.stream)
if not reset:
quiet_shutdown(self.socket)
quiet_close(self.socket)
already_closed = self.socket is None
self.socket = None
self.stream = None
if not self.initiated_from_client and self.python_server and not already_closed:
server_connection_stopped.send(self.python_server, connection=self)
def close(self, reset=False):
logger.info("Closing down clientserver connection")
if not self.socket:
return
if reset:
set_linger(self.socket)
quiet_close(self.stream)
if not reset:
quiet_shutdown(self.socket)
quiet_close(self.socket)
already_closed = self.socket is None
self.socket = None
self.stream = None
if not self.initiated_from_client and self.python_server and\
not already_closed:
server_connection_stopped.send(self.python_server, connection=self)
def connect_to_java_server(self):
try:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.ssl_context:
self.socket = self.ssl_context.wrap_socket(
self.socket, server_hostname=self.java_address)
self.socket.connect((self.java_address, self.java_port))
self.stream = self.socket.makefile("rb", 0)
self.is_connected = True
self.initiated_from_client = True
except Exception:
quiet_close(self.socket)
quiet_close(self.stream)
self.socket = None
self.stream = None
self.is_connected = False
raise
def connect_to_java_server(self):
try:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if self.ssl_context:
self.socket = self.ssl_context.wrap_socket(
self.socket, server_hostname=self.java_address)
self.socket.connect((self.java_address, self.java_port))
self.stream = self.socket.makefile("rb", 0)
self.is_connected = True
self.initiated_from_client = True
except Exception:
quiet_close(self.socket)
quiet_close(self.stream)
self.socket = None
self.stream = None
self.is_connected = False
raise
def shutdown_gateway(self):
"""Sends a shutdown command to the Java side.
This will close the ClientServer on the Java side: all active
connections will be closed. This may be useful if the lifecycle
of the Java program must be tied to the Python program.
"""
if not self.is_connected:
raise Py4JError("Gateway must be connected to send shutdown cmd.")
try:
quiet_close(self.stream)
self.socket.sendall(proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8"))
self.close()
except Exception:
# Do nothing! Exceptions might occur anyway.
logger.debug("Exception occurred while shutting down gateway", exc_info=True)
def shutdown_gateway(self):
"""Sends a shutdown command to the Java side.
This will close the ClientServer on the Java side: all active
connections will be closed. This may be useful if the lifecycle
of the Java program must be tied to the Python program.
"""
if not self.is_connected:
raise Py4JError("Gateway must be connected to send shutdown cmd.")
try:
quiet_close(self.stream)
self.socket.sendall(
proto.SHUTDOWN_GATEWAY_COMMAND_NAME.encode("utf-8"))
self.close()
except Exception:
# Do nothing! Exceptions might occur anyway.
logger.debug("Exception occurred while shutting down gateway",
exc_info=True)
def clean_closed_connections():
from py4j.java_gateway import quiet_close, quiet_shutdown
callback_server = self._gateway._callback_server
with callback_server.lock:
try:
closed_connections = []
for connection in callback_server.connections:
if not connection.isAlive():
quiet_close(connection.input)
quiet_shutdown(connection.socket)
quiet_close(connection.socket)
closed_connections.append(connection)
for closed_connection in closed_connections:
callback_server.connections.remove(closed_connection)
except Exception:
import traceback
traceback.print_exc()
self._start_timer(clean_closed_connections)
def clean_closed_connections():
from py4j.java_gateway import quiet_close, quiet_shutdown
callback_server = self._gateway._callback_server
if callback_server:
with callback_server.lock:
try:
closed_connections = []
for connection in callback_server.connections:
if not connection.isAlive():
quiet_close(connection.input)
quiet_shutdown(connection.socket)
quiet_close(connection.socket)
closed_connections.append(connection)
for closed_connection in closed_connections:
callback_server.connections.remove(closed_connection)
except Exception:
import traceback
traceback.print_exc()
self._start_timer(clean_closed_connections)
def testRedirectToFile(self):
end = os.linesep
(out_handle, outpath) = tempfile.mkstemp(text=True)
(err_handle, errpath) = tempfile.mkstemp(text=True)
stdout = open(outpath, "w")
stderr = open(errpath, "w")
try:
self.gateway = JavaGateway.launch_gateway(
redirect_stdout=stdout, redirect_stderr=stderr)
for i in range(10):
self.gateway.jvm.System.out.println("Test")
self.gateway.jvm.System.err.println("Test2")
self.gateway.shutdown()
sleep()
# Should not be necessary
quiet_close(stdout)
quiet_close(stderr)
# Test that the redirect files were written to correctly
with open(outpath, "r") as stdout:
lines = stdout.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test{0}".format(end), lines[0])
with open(errpath, "r") as stderr:
lines = stderr.readlines()
if not stderr_is_polluted(lines[0]):
self.assertEqual(10, len(lines))
# XXX Apparently, it's \n by default even on windows...
# Go figure
self.assertEqual("Test2\n", lines[0])
finally:
os.close(out_handle)
os.close(err_handle)
os.unlink(outpath)
os.unlink(errpath)
def testRedirectToFile(self):
end = os.linesep
(out_handle, outpath) = tempfile.mkstemp(text=True)
(err_handle, errpath) = tempfile.mkstemp(text=True)
stdout = open(outpath, "w")
stderr = open(errpath, "w")
try:
self.gateway = JavaGateway.launch_gateway(redirect_stdout=stdout,
redirect_stderr=stderr)
for i in range(10):
self.gateway.jvm.System.out.println("Test")
self.gateway.jvm.System.err.println("Test2")
self.gateway.shutdown()
sleep()
# Should not be necessary
quiet_close(stdout)
quiet_close(stderr)
# Test that the redirect files were written to correctly
with open(outpath, "r") as stdout:
lines = stdout.readlines()
self.assertEqual(10, len(lines))
self.assertEqual("Test{0}".format(end), lines[0])
with open(errpath, "r") as stderr:
lines = stderr.readlines()
if not stderr_is_polluted(lines[0]):
self.assertEqual(10, len(lines))
# XXX Apparently, it's \n by default even on windows...
# Go figure
self.assertEqual("Test2\n", lines[0])
finally:
os.close(out_handle)
os.close(err_handle)
os.unlink(outpath)
os.unlink(errpath)
def launch_gateway(port=0,
jarpath="",
classpath="",
javaopts=[],
die_on_exit=False,
redirect_stdout=None,
redirect_stderr=None,
daemonize_redirect=True,
java_path="java",
create_new_process_group=False,
enable_auth=False,
return_proc=False):
"""Launch a `Gateway` in a new Java process.
The redirect parameters accept file-like objects, Queue, or deque. When
text lines are sent to the stdout or stderr of the child JVM, these lines
are redirected to the file-like object (``write(line)``), the Queue
(``put(line)``), or the deque (``appendleft(line)``).
The text line will contain a newline character.
Only text output is accepted on stdout and stderr. If you wish to
communicate with the child JVM through bytes, you need to create your own
helper function.
:param port: the port to launch the Java Gateway on. If no port is
specified then an ephemeral port is used.
:param jarpath: the path to the Py4J jar. Only necessary if the jar
was installed at a non-standard location or if Python is using
a different `sys.prefix` than the one that Py4J was installed
under.
:param classpath: the classpath used to launch the Java Gateway.
:param javaopts: an array of extra options to pass to Java (the classpath
should be specified using the `classpath` parameter, not `javaopts`.)
:param die_on_exit: if `True`, the Java gateway process will die when
this Python process exits or is killed.
:param redirect_stdout: where to redirect the JVM stdout. If None (default)
stdout is redirected to os.devnull. Otherwise accepts a
file descriptor, a queue, or a deque. Will send one line at a time
to these objects.
:param redirect_stderr: where to redirect the JVM stdout. If None (default)
stderr is redirected to os.devnull. Otherwise accepts a
file descriptor, a queue, or a deque. Will send one line at a time to
these objects.
:param daemonize_redirect: if True, the consumer threads will be daemonized
and will not prevent the main Python process from exiting. This means
the file descriptors (stderr, stdout, redirect_stderr, redirect_stdout)
might not be properly closed. This is not usually a problem, but in
case of errors related to file descriptors, set this flag to False.
:param java_path: If None, Py4J will use $JAVA_HOME/bin/java if $JAVA_HOME
is defined, otherwise it will use "java".
:param create_new_process_group: If True, the JVM is started in a new
process group. This ensures that signals sent to the parent Python
process are not forwarded to the JVM. For example, sending
Ctrl-C/SIGINT won't interrupt the JVM. If the python process dies, the
Java process will stay alive, which may be a problem for some scenarios
though.
:param enable_auth: If True, the server will require clients to provide an
authentication token when connecting.
:param return_proc: If True, returns the Popen object returned when the JVM
process was created.
:rtype: the port number of the `Gateway` server or, when auth enabled,
a 2-tuple with the port number and the auth token.
"""
popen_kwargs = {}
if not jarpath:
jarpath = find_jar_path()
if not java_path:
java_home = os.environ.get("JAVA_HOME")
if java_home:
java_path = os.path.join(java_home, "bin", "java")
else:
java_path = "java"
# Fail if the jar does not exist.
if not os.path.exists(jarpath):
raise Py4JError("Could not find py4j jar at {0}".format(jarpath))
# Launch the server in a subprocess.
classpath = os.pathsep.join((jarpath, classpath))
command = [java_path, "-classpath", classpath] + javaopts + \
["py4j.GatewayServer"]
if die_on_exit:
command.append("--die-on-broken-pipe")
if enable_auth:
command.append("--enable-auth")
command.append(str(port))
logger.debug("Launching gateway with command {0}".format(command))
# stderr redirection
close_stderr = False
if redirect_stderr is None:
stderr = open(os.devnull, "w")
close_stderr = True
elif isinstance(redirect_stderr, Queue) or\
isinstance(redirect_stderr, deque):
stderr = PIPE
else:
stderr = redirect_stderr
# we don't need this anymore
redirect_stderr = None
# stdout redirection
if redirect_stdout is None:
redirect_stdout = open(os.devnull, "w")
if create_new_process_group:
popen_kwargs.update(get_create_new_process_group_kwargs())
proc = Popen(command,
stdout=PIPE,
stdin=PIPE,
stderr=stderr,
**popen_kwargs)
# Determine which port the server started on (needed to support
# ephemeral ports)
_port = int(proc.stdout.readline())
# Read the auth token from the server if enabled.
_auth_token = None
if enable_auth:
_auth_token = proc.stdout.readline()[:-1]
# Start consumer threads so process does not deadlock/hangs
OutputConsumer(redirect_stdout, proc.stdout,
daemon=daemonize_redirect).start()
if redirect_stderr is not None:
OutputConsumer(redirect_stderr, proc.stderr,
daemon=daemonize_redirect).start()
ProcessConsumer(proc, [redirect_stdout], daemon=daemonize_redirect).start()
if close_stderr:
# XXX This will quiet ResourceWarning in Python 3.5+
# This only close the fd in this process, not in the JVM process, which
# makes sense.
quiet_close(stderr)
if enable_auth:
output = (_port, _auth_token)
else:
output = _port
if return_proc:
if isinstance(output, tuple):
output = output + (proc, )
else:
output = (_port, proc)
return output
def start():
global __pythonManager, __gateway, __dependency_manager, __gmql_jar_path, __py4j_path
logger = logging.getLogger()
master = get_master()
if master.lower().startswith('local'):
logger.debug("Starting LOCAL backend (master: {})".format(
master.lower()))
java_home = os.environ.get("JAVA_HOME")
if java_home is None:
raise SystemError("The environment variable JAVA_HOME is not set")
java_path = os.path.join(java_home, "bin", "java")
_port = launch_gateway(classpath=__gmql_jar_path,
die_on_exit=True,
java_path=java_path,
javaopts=get_local_java_options(),
jarpath=__py4j_path)
__gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=_port, auto_convert=True))
python_api_package = get_python_api_package(__gateway)
__pythonManager = start_gmql_manager(python_api_package)
conf = get_configuration()
conf.set_master(master.lower())
_set_spark_configuration(conf)
_set_system_configuration(conf)
else:
# use spark-submit
logger.debug("Submitting backend to {}".format(master))
master = re.sub("^spark_", "", master.lower())
configs = get_spark_configs()
spark_location = find()
logger.debug("Found spark at location: {}".format(spark_location))
command = [
os.path.join(spark_location, 'bin', 'spark-submit'), '--master',
master, '--deploy-mode', "client"
]
for cname, c in configs.items():
command.extend(['--conf', '{}={}'.format(cname, c)])
command.append(__gmql_jar_path)
stderr = open(os.devnull, "w")
proc = Popen(command, stdout=PIPE, stdin=PIPE, stderr=stderr)
while True:
try:
_port = int(proc.stdout.readline())
break
except ValueError:
pass
logger.debug("Backend listening at port {}".format(_port))
redirect_stdout = open(os.devnull, "w")
OutputConsumer(redirect_stdout, proc.stdout, daemon=True).start()
ProcessConsumer(proc, [redirect_stdout], daemon=True).start()
quiet_close(stderr)
__gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=_port, auto_convert=True))
pm = __gateway.entry_point.getPythonManager()
pm.startEngine()
__pythonManager = pm
