def testGoodRetryFromJava(self):
"""Should retry from Java to Python.
Similar use case as testGoodRetry, but from Java: Python calls Java,
which calls Python back. Then Java waits for a while and calls Python
again.
Because Python Server has been waiting for too much time, the
receiving socket has closed so the call from Java to Python will fail
on send, and Java must retry by creating a new connection
(ClientServerConnection).
Because ClientServer reuses the same connection in each thread, we must
launch a new thread on the Java side to correctly test the Python
Server.
"""
client_server = ClientServer(
JavaParameters(), PythonParameters(read_timeout=0.250))
with clientserver_example_app_process():
try:
operator = WaitOperator(0)
opExample = client_server.jvm.py4j.examples.OperatorExample()
opExample.launchOperator(operator, 500)
sleep(0.1)
str_connection = str(
list(client_server._callback_server.connections)[0])
sleep(0.75)
str_connection2 = str(
list(client_server._callback_server.connections)[0])
self.assertNotEqual(str_connection, str_connection2)
except Py4JJavaError:
self.fail("Callbackserver did not retry.")
finally:
client_server.shutdown()
def testJavaGC(self):
# This will only work with some JVM.
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters())
example = client_server.entry_point.getNewExample()
impl = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl))
self.assertEqual(
"This is Hello;\n10MyMy!\n;",
example.callHello2(impl))
self.assertEqual(2, len(client_server.gateway_property.pool))
# Make sure that finalizers do not block by calling the Java
# finalizer again
impl2 = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl2))
self.assertEqual(3, len(client_server.gateway_property.pool))
# The two PythonProxies should be evicted on the Java side
# Java should tell python to release the references.
client_server.jvm.java.lang.System.gc()
# Leave time for sotimeout
sleep(3)
self.assertTrue(len(client_server.gateway_property.pool) < 2)
client_server.shutdown()
def testErrorInPythonCallback(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters(
propagate_java_exceptions=True))
example = client_server.entry_point.getNewExample()
try:
example.callHello(IHelloFailingImpl(
ValueError('My interesting Python exception')))
self.fail()
except Py4JJavaError as e:
self.assertTrue(is_instance_of(
client_server, e.java_exception,
'py4j.Py4JException'))
self.assertIn('interesting Python exception', str(e))
try:
example.callHello(IHelloFailingImpl(
Py4JJavaError(
'',
client_server.jvm.java.lang.IllegalStateException(
'My IllegalStateException'))))
self.fail()
except Py4JJavaError as e:
self.assertTrue(is_instance_of(
client_server, e.java_exception,
'java.lang.IllegalStateException'))
client_server.shutdown()
def testBasicJVM(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters())
ms = client_server.jvm.System.currentTimeMillis()
self.assertTrue(ms > 0)
client_server.shutdown()
def testBadRetryFromJava(self):
"""Should not retry from Java to Python.
Similar use case as testBadRetry, but from Java: Java calls a long
Python operation.
If there is a bug, Java will call Python, then read will fail, then it
will call Python again.
If there is no bug, Java will call Python, read will fail, then Java
will raise an Exception that will be received as a Py4JError on the
Python side.
"""
client_server = ClientServer(
JavaParameters(), PythonParameters())
with clientserver_example_app_process(False, True):
try:
operator = WaitOperator(0.5)
opExample = client_server.jvm.py4j.examples.OperatorExample()
opExample.randomBinaryOperator(operator)
self.fail(
"Should never retry once the first command went through."
" number of calls made: {0}".format(operator.callCount))
except Py4JError:
# XXX This occurs when WaitOperator tries to send a response to
# the Java side (this is slightly different then the
# GatewayServer equivalent where the Py4JError occurs on the
# clientserver, but the Java side can still send back an
# exception to the JavaGateway).
self.assertTrue(True)
finally:
client_server.shutdown()
def testRecursion(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters())
pingpong = client_server.jvm.py4j.examples.PingPong()
ping = PythonPing()
self.assertEqual(2, pingpong.start(ping))
pingpong = client_server.jvm.py4j.examples.PingPong(True)
try:
pingpong.start(ping)
self.fail()
except Py4JJavaError:
# TODO Make sure error are recursively propagated
# Also a problem with old threading model.
self.assertTrue(True)
ping = PythonPing(True)
pingpong = client_server.jvm.py4j.examples.PingPong(False)
try:
pingpong.start(ping)
self.fail()
except Py4JJavaError:
# TODO Make sure error are recursively propagated
# Also a problem with old threading model.
self.assertTrue(True)
client_server.shutdown()
def testJavaClientPythonServer(self):
hello_state = HelloState()
client_server = ClientServer(
JavaParameters(), PythonParameters(), hello_state)
with clientserver_example_app_process(True):
client_server.shutdown()
# Check that Java correctly called Python
self.assertEqual(2, len(hello_state.calls))
self.assertEqual((None, None), hello_state.calls[0])
self.assertEqual((2, "Hello World"), hello_state.calls[1])
def testErrorInPy4J(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters())
try:
client_server.jvm.java.lang.Math.abs(
3000000000000000000000000000000000000)
self.fail("Should not be able to convert overflowing long")
except Py4JError:
self.assertTrue(True)
# Check that the connection is not broken (refs #265)
val = client_server.jvm.java.lang.Math.abs(-4)
self.assertEqual(4, val)
client_server.shutdown()
def test_all_regular_signals(self):
listener = MockListener(self)
server_started.connect(listener.started)
hello_state = HelloState()
client_server = ClientServer(
JavaParameters(), PythonParameters(), hello_state)
server_stopped.connect(
listener.stopped, sender=client_server.get_callback_server())
server_connection_started.connect(
listener.connection_started,
sender=client_server.get_callback_server())
server_connection_stopped.connect(
listener.connection_stopped,
sender=client_server.get_callback_server())
pre_server_shutdown.connect(
listener.pre_shutdown, sender=client_server.get_callback_server())
post_server_shutdown.connect(
listener.post_shutdown, sender=client_server.get_callback_server())
with clientserver_example_app_process(True):
client_server.shutdown()
self.assertEqual(1, listener.received["started"])
self.assertEqual(1, listener.received["stopped"])
self.assertEqual(1, listener.received["pre_shutdown"])
self.assertEqual(1, listener.received["post_shutdown"])
self.assertEqual(1, listener.received["connection_started"])
self.assertEqual(1, listener.received["connection_stopped"])
def test_signals_started_from_python(self):
listener = MockListener(self)
with clientserver_example_app_process():
server_started.connect(listener.started)
client_server = ClientServer(
JavaParameters(), PythonParameters())
example = client_server.entry_point.getNewExample()
impl = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl))
server_stopped.connect(
listener.stopped, sender=client_server.get_callback_server())
server_connection_started.connect(
listener.connection_started,
sender=client_server.get_callback_server())
server_connection_stopped.connect(
listener.connection_stopped,
sender=client_server.get_callback_server())
pre_server_shutdown.connect(
listener.pre_shutdown,
sender=client_server.get_callback_server())
post_server_shutdown.connect(
listener.post_shutdown,
sender=client_server.get_callback_server())
client_server.shutdown()
self.assertEqual(1, listener.received["started"])
self.assertEqual(1, listener.received["stopped"])
self.assertEqual(1, listener.received["pre_shutdown"])
self.assertEqual(1, listener.received["post_shutdown"])
# Connection initiated from JavaClient, so no signal sent.
self.assertEqual(0, listener.received["connection_started"])
self.assertEqual(0, listener.received["connection_stopped"])
def testSendObjects(self):
"""This test receives 1000 calls creating object cycles.
Typically, the garbage collector will starts in the middle of
Py4J connection code, which will create an error if the garbage
collection is done on the same thread.
"""
hello = HelloObjects()
client_server = ClientServer(
JavaParameters(), PythonParameters(), hello)
with clientserver_example_app_process(
start_gc_test=True, join=False) as p:
p.join()
client_server.shutdown()
self.assertEquals(1000, hello.calls)
def testJavaToPythonToJavaNoGCNoShutdown(self):
def internal_work(clientserver):
hello_state = HelloState2()
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT+5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5),
python_server_entry_point=hello_state)
hello_state.gateway = clientserver2
sleep()
clientserver.entry_point.startServerWithPythonEntry(False)
sleep()
clientserver2.shutdown()
# Check that Java correctly called Python
self.assertEqual(2, len(hello_state.calls))
self.assertEqual((None, None), hello_state.calls[0])
self.assertEqual((2, "Hello World"), hello_state.calls[1])
with gateway_server_example_app_process(False):
clientserver = ClientServer()
# We disable gc to test whether a shut down on one side will
# garbage collect everything.
gc.disable()
internal_work(clientserver)
gc.enable()
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getCreatedObjectsKeySet()
finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getFinalizedObjectsKeySet()
# 7 objects: 2 InstrumentedObject (sayHello called twice), 1
# JavaServer, 1 PythonClient, 1 ClientServer, 3
# ClientServerConnection (1 to call sayHello,
# 1 that receives shutdown command)
self.assertEqual(7, len(createdSet))
self.assertEqual(7, len(finalizedSet))
self.assertEqual(createdSet, finalizedSet)
clientserver.shutdown()
# 8 objects: ClientServer (ok), PythonServer (ok), JavaClient,
# GatewayProperty, HelloState (ok), 3 ClientServer Connections (2)
assert_python_memory(self, 7)
def testGoodRetry(self):
"""Should retry from Python to Java.
Python calls Java twice in a row, then waits, then calls again.
Java fails when it does not receive calls quickly.
If there is a bug, Python will fail on the third call because the Java
connection was closed and it did not retry.
If there is a bug, Python might not fail because Java did not close the
connection on timeout. The connection used to call Java will be the
same one for all calls (and an assertion will fail).
If there is no bug, Python will call Java twice with the same
connection. On the third call, the write will fail, and a new
connection will be created.
"""
client_server = ClientServer(
JavaParameters(), PythonParameters())
connections = client_server._gateway_client.deque
with clientserver_example_app_process(False, True):
try:
# Call #1
client_server.jvm.System.currentTimeMillis()
str_connection = str(connections[0])
# Call #2
client_server.jvm.System.currentTimeMillis()
self.assertEqual(1, len(connections))
str_connection2 = str(connections[0])
self.assertEqual(str_connection, str_connection2)
sleep(0.5)
client_server.jvm.System.currentTimeMillis()
self.assertEqual(1, len(connections))
str_connection3 = str(connections[0])
# A new connection was automatically created.
self.assertNotEqual(str_connection, str_connection3)
except Py4JError:
self.fail("Should retry automatically by default.")
finally:
client_server.shutdown()
def testJavaToPythonToJavaCleanGC(self):
def internal_work(clientserver):
hello_state = HelloState2()
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT+5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5),
python_server_entry_point=hello_state)
hello_state.gateway = clientserver2
sleep()
clientserver.entry_point.startServerWithPythonEntry(True)
sleep()
clientserver2.shutdown()
# Check that Java correctly called Python
self.assertEqual(2, len(hello_state.calls))
self.assertEqual((None, None), hello_state.calls[0])
self.assertEqual((2, "Hello World"), hello_state.calls[1])
with gateway_server_example_app_process(False):
clientserver = ClientServer()
internal_work(clientserver)
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getCreatedObjectsKeySet()
finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getFinalizedObjectsKeySet()
# 7 objects: 2 InstrumentedObject (sayHello called twice), 1
# JavaServer, 1 PythonClient, 1 ClientServer, 2
# ClientServerConnection (1 to call sayHello, 1 that calls GC from
# Java to Python)
self.assertEqual(7, len(createdSet))
self.assertEqual(7, len(finalizedSet))
self.assertEqual(createdSet, finalizedSet)
clientserver.shutdown()
# 8 objects: ClientServer (ok), PythonServer (ok), JavaClient,
# GatewayProperty, HelloState (ok), 3 ClientServer Connections (2)
assert_python_memory(self, 8)
def testStream(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(), PythonParameters())
e = client_server.entry_point.getNewExample()
# not binary - just get the Java object
v1 = e.getStream()
self.assertTrue(
is_instance_of(
client_server, v1,
"java.nio.channels.ReadableByteChannel"))
# pull it as a binary stream
with e.getStream.stream() as conn:
self.assertTrue(isinstance(conn, GatewayConnectionGuard))
expected =\
u"Lorem ipsum dolor sit amet, consectetur adipiscing elit."
self.assertEqual(
expected, smart_decode(conn.read(len(expected))))
client_server.shutdown()
def testPythonToJava(self):
def work_with_object(clientserver):
obj = clientserver.jvm.py4j.\
instrumented.InstrumentedObject("test")
return str(obj)
def internal_work():
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT+5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
sleep()
work_with_object(clientserver2)
python_gc()
sleep()
clientserver2.shutdown()
with gateway_server_example_app_process(False):
clientserver = ClientServer()
clientserver.entry_point.startServer2()
internal_work()
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getCreatedObjectsKeySet()
finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getFinalizedObjectsKeySet()
# 5 objects: ClientServer, ClientServerConnection, PythonClient,
# JavaServer, InstrumentedObject
self.assertEqual(5, len(createdSet))
self.assertEqual(5, len(finalizedSet))
self.assertEqual(createdSet, finalizedSet)
clientserver.shutdown()
# 5 objects: ClientServer, ClientServerConnection, PythonClient,
# JavaServer, GatewayProperty
assert_python_memory(self, 5)
def testPythonGC(self):
def internal_function(client_server):
example = client_server.entry_point.getNewExample()
in_middle = len(
client_server.java_gateway_server.getGateway().getBindings())
self.assertEqual(1, example.method1())
# After this method is executed, Python no longer need
# a reference to example and it should tell Java to release
# the reference too
return in_middle
with clientserver_example_app_process():
# This will only work with some JVM.
client_server = ClientServer(
JavaParameters(), PythonParameters())
before = len(
client_server.java_gateway_server.getGateway().getBindings())
in_middle = internal_function(client_server)
# Force Python to run garbage collection in case it did
# not run yet.
sleep()
gc.collect()
# Long sleep to ensure that the finalizer worker received
# everything
sleep(2)
after = len(
client_server.java_gateway_server.getGateway().getBindings())
# Number of references on the Java side should be the same before
# and after
self.assertEqual(before, after)
# Number of references when we created a JavaObject should be
# higher than at the beginning.
self.assertGreater(in_middle, before)
client_server.shutdown()
def testBadRetry(self):
"""Should not retry from Python to Java.
Python calls a long Java method. The call goes through, but the
response takes a long time to get back.
If there is a bug, Python will fail on read and retry (sending the same
call twice).
If there is no bug, Python will fail on read and raise an Exception.
"""
client_server = ClientServer(
JavaParameters(read_timeout=0.250), PythonParameters())
with clientserver_example_app_process():
try:
example = client_server.jvm.py4j.examples.ExampleClass()
value = example.sleepFirstTimeOnly(500)
self.fail(
"Should never retry once the first command went through."
"number of calls made: {0}".format(value))
except Py4JError:
self.assertTrue(True)
finally:
client_server.shutdown()
def testPythonToJavaToPython(self):
def play_with_ping(clientserver):
ping = InstrumentedPythonPing()
pingpong = clientserver.jvm.py4j.examples.PingPong()
total = pingpong.start(ping)
return total
def internal_work():
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT+5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
sleep()
play_with_ping(clientserver2)
python_gc()
sleep()
clientserver2.shutdown()
with gateway_server_example_app_process(False):
clientserver = ClientServer()
clientserver.entry_point.startServer2()
internal_work()
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getCreatedObjectsKeySet()
finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
getFinalizedObjectsKeySet()
# 4 objects: ClientServer, ClientServerConnection, JavaServer,
# PythonClient
self.assertEqual(4, len(createdSet))
self.assertEqual(4, len(finalizedSet))
self.assertEqual(createdSet, finalizedSet)
clientserver.shutdown()
def testPythonToJavaToPythonClose(self):
def play_with_ping(clientserver):
ping = InstrumentedPythonPing()
pingpong = clientserver.jvm.py4j.examples.PingPong()
total = pingpong.start(ping)
return total
def internal_work(assert_memory):
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT+5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
sleep()
play_with_ping(clientserver2)
python_gc()
sleep()
clientserver2.close(
close_callback_server_connections=True,
keep_callback_server=True)
sleep()
assert_memory()
clientserver2.shutdown()
sleep()
with gateway_server_example_app_process(False):
clientserver = ClientServer()
clientserver.entry_point.startServer2()
def perform_memory_tests():
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
createdSet = clientserver.jvm.py4j.instrumented.\
MetricRegistry.getCreatedObjectsKeySet()
finalizedSet = clientserver.jvm.py4j.instrumented.\
MetricRegistry.getFinalizedObjectsKeySet()
# 6 objects: ClientServer, JavaServer,
# PythonClient, 3 ClientServerConnection.
self.assertEqual(6, len(createdSet))
# Should be 2: ClientServer, 1 ClientServerConnection
# But for some reasons, Java refuses to collect the
# clientserverconnection even though there are no strong
# references.
self.assertEqual(1, len(finalizedSet))
# 8 objects: ClientServer, PythonServer, JavaClient,
# GatewayProperty, PythonPing, 3 ClientServerConnection
self.assertEqual(8, len(CREATED))
# PythonPing + ClientServerConnection
self.assertEqual(2, len(FINALIZED))
internal_work(perform_memory_tests)
python_gc()
clientserver.jvm.py4j.instrumented.MetricRegistry.\
forceFinalization()
sleep()
clientserver.shutdown()
# 9 objects: ClientServer, PythonServer, JavaClient,
# GatewayProperty, PythonPing, 4 ClientServerConnection
assert_python_memory(self, 9)
pl.setX(0, earthp[0])
pl.setY(0, earthp[1])
pl.setZ(0, earthp[2])
pl.setX(1, moonp[0])
pl.setY(1, moonp[1])
pl.setZ(1, moonp[2])
self.polyline.markForUpdate()
def toString():
return "line-update-runnable"
class Java:
implements = ["java.lang.Runnable"]
gateway = ClientServer(java_parameters=JavaParameters(auto_convert=True),
python_parameters=PythonParameters())
gs = gateway.entry_point
gs.cameraStop()
gs.stopSimulationTime()
gs.setVisibility("element.orbits", True)
gs.setCameraLock(True)
gs.setCameraOrientationLock(False)
gs.setFov(49)
gs.goToObject("Earth", 91.38e-2)
print("We will now add a line between the Earth and Moon")
def testBasicJVM(self):
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(), PythonParameters())
ms = client_server.jvm.System.currentTimeMillis()
self.assertTrue(ms > 0)
client_server.shutdown()
def testMultiClientServerWithSharedJavaThread(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# ## Preconditions
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 1 Hop to Shared Java Thread
# Check that the shared Java thread is the same thread
# for both ClientServers
sharedJavaThreadId = entry0.getSharedJavaThreadId()
self.assertEqual(sharedJavaThreadId,
entry1.getSharedJavaThreadId())
# And that it is distinct from either corresponding to
# this Python thread
self.assertNotEqual(sharedJavaThreadId, entry0.getJavaThreadId())
self.assertNotEqual(sharedJavaThreadId, entry1.getJavaThreadId())
# ## 2 Hops via Shared Java Thread to Python Thread
# Check that the shared thread ends up as different threads
# in the Python side. This part may not be obvious as the
# top-level idea seems to be for Python thread to be pinned
# to Java thread. Consider that this case is a simplification
# of the real case. In the real case there are two
# ClientServers running in same JVM, but in Python side each
# ClientServer is in its own process. In that case it makes
# it obvious that the shared Java thread should indeed
# end up in different Python threads.
sharedPythonThreadId0 = entry0.getSharedPythonThreadId()
sharedPythonThreadId1 = entry1.getSharedPythonThreadId()
# three way assert to make sure all three python
# threads are distinct
self.assertNotEqual(thisThreadId, sharedPythonThreadId0)
self.assertNotEqual(thisThreadId, sharedPythonThreadId1)
self.assertNotEqual(sharedPythonThreadId0, sharedPythonThreadId1)
# Check that the Python thread id does not change between
# invocations
self.assertEquals(sharedPythonThreadId0,
entry0.getSharedPythonThreadId())
self.assertEquals(sharedPythonThreadId1,
entry1.getSharedPythonThreadId())
# ## 3 Hops to Shared Java Thread
# Check that the thread above after 2 hops calls back
# into the Java shared thread
self.assertEqual(sharedJavaThreadId,
entry0.getSharedViaPythonJavaThreadId())
self.assertEqual(sharedJavaThreadId,
entry1.getSharedViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
def launch_gateway(conf=None, popen_kwargs=None):
"""
launch jvm gateway
:param conf: spark configuration passed to spark-submit
:param popen_kwargs: Dictionary of kwargs to pass to Popen when spawning
the py4j JVM. This is a developer feature intended for use in
customizing how pyspark interacts with the py4j JVM (e.g., capturing
stdout/stderr).
:return:
"""
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
gateway_secret = os.environ["PYSPARK_GATEWAY_SECRET"]
# Process already exists
proc = None
else:
SPARK_HOME = _find_spark_home()
# Launch the Py4j gateway using Spark's run command so that we pick up the
# proper classpath and settings from spark-env.sh
on_windows = platform.system() == "Windows"
script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
command = [os.path.join(SPARK_HOME, script)]
if conf:
for k, v in conf.getAll():
command += ['--conf', '%s=%s' % (k, v)]
submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
if os.environ.get("SPARK_TESTING"):
submit_args = ' '.join(
["--conf spark.ui.enabled=false", submit_args])
command = command + shlex.split(submit_args)
# Create a temporary directory where the gateway server should write the connection
# information.
conn_info_dir = tempfile.mkdtemp()
try:
fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
os.close(fd)
os.unlink(conn_info_file)
env = dict(os.environ)
env["_PYSPARK_DRIVER_CONN_INFO_PATH"] = conn_info_file
# Launch the Java gateway.
popen_kwargs = {} if popen_kwargs is None else popen_kwargs
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
popen_kwargs['stdin'] = PIPE
# We always set the necessary environment variables.
popen_kwargs['env'] = env
if not on_windows:
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
popen_kwargs['preexec_fn'] = preexec_func
proc = Popen(command, **popen_kwargs)
else:
# preexec_fn not supported on Windows
proc = Popen(command, **popen_kwargs)
# Wait for the file to appear, or for the process to exit, whichever happens first.
while not proc.poll() and not os.path.isfile(conn_info_file):
time.sleep(0.1)
if not os.path.isfile(conn_info_file):
raise Exception(
"Java gateway process exited before sending its port number"
)
with open(conn_info_file, "rb") as info:
gateway_port = read_int(info)
gateway_secret = UTF8Deserializer().loads(info)
finally:
shutil.rmtree(conn_info_dir)
# In Windows, ensure the Java child processes do not linger after Python has exited.
# In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
# the parent process' stdin sends an EOF). In Windows, however, this is not possible
# because java.lang.Process reads directly from the parent process' stdin, contending
# with any opportunity to read an EOF from the parent. Note that this is only best
# effort and will not take effect if the python process is violently terminated.
if on_windows:
# In Windows, the child process here is "spark-submit.cmd", not the JVM itself
# (because the UNIX "exec" command is not available). This means we cannot simply
# call proc.kill(), which kills only the "spark-submit.cmd" process but not the
# JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
# child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
def killChild():
Popen([
"cmd", "/c", "taskkill", "/f", "/t", "/pid",
str(proc.pid)
])
atexit.register(killChild)
# Connect to the gateway (or client server to pin the thread between JVM and Python)
if os.environ.get("PYSPARK_PIN_THREAD", "false").lower() == "true":
gateway = ClientServer(
java_parameters=JavaParameters(port=gateway_port,
auth_token=gateway_secret,
auto_convert=True),
python_parameters=PythonParameters(port=0, eager_load=False))
else:
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=gateway_port, auth_token=gateway_secret, auto_convert=True))
# Store a reference to the Popen object for use by the caller (e.g., in reading stdout/stderr)
gateway.proc = proc
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.conda.*")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.ml.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
# TODO(davies): move into sql
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
class SimpleHello(object):
def sayHello(self, int_value=None, string_value=None):
print(int_value, string_value)
return "Said hello to {0}".format(string_value)
class Java:
implements = ["py4j.examples._3_5_JavaCallPy.ClientServer.IHello"]
# Make sure that the python code is started first.
# Then execute: java -cp py4j.jar py4j.examples._3_5_JavaCallPy.ClientServer.SingleThreadClientApplication
from py4j.clientserver import ClientServer, JavaParameters, PythonParameters
simple_hello = SimpleHello()
gateway = ClientServer(
java_parameters=JavaParameters(),
python_parameters=PythonParameters(),
python_server_entry_point=simple_hello)
# Output in cmd is as follows:
None None
2 Hello World
return self._covert_list(clips)
def _covert_list(self, py_list: list):
"""
covert py list to java list contain list in list
:param py_list: py list
:return: java list
"""
gateway = JavaGateway()
if type(py_list[0]) is list:
java_list = ListConverter().convert([], gateway._gateway_client)
for l in py_list:
j_l = ListConverter().convert(l, gateway._gateway_client)
java_list.append(j_l)
else:
java_list = ListConverter().convert(py_list,
gateway._gateway_client)
return java_list
def java_list_oppend(self, List):
pass
class Java:
implements = ["com.alan.Client"]
if __name__ == '__main__':
client_server = ClientServer(java_parameters=JavaParameters(),
python_parameters=PythonParameters(),
python_server_entry_point=JavaServer())
print("Server started")
def connect(
cls,
java_port: int = None,
python_port: int = None,
auto_load_java: bool = False,
client_server: bool = True,
) -> bool:
"""
connects to jnegmas
"""
if not java_port:
java_port = cls.DEFAULT_JAVA_PORT
if not python_port:
python_port = cls.DEFAULT_PYTHON_PORT
if auto_load_java:
if cls.gateway is None:
cls.start_java_side(java_port=java_port)
return True
if cls.gateway is None:
eager_load, auto_convert = True, True
auto_field, auto_gc = True, False
propagate_exceptions = False
if client_server:
cls.gateway = ClientServer(
java_parameters=JavaParameters(
port=java_port,
auto_convert=auto_convert,
eager_load=eager_load,
auto_close=True,
auto_gc=False,
auto_field=auto_field,
daemonize_memory_management=True,
),
python_parameters=PythonParameters(
port=python_port,
propagate_java_exceptions=propagate_exceptions,
daemonize=True,
eager_load=eager_load,
auto_gc=auto_gc,
daemonize_connections=True,
),
python_server_entry_point=PyEntryPoint(),
)
else:
pyparams = CallbackServerParameters(
port=python_port,
daemonize_connections=True,
daemonize=True,
eager_load=eager_load,
propagate_java_exceptions=propagate_exceptions,
)
cls.gateway = JavaGateway(
gateway_parameters=GatewayParameters(
port=java_port,
auto_convert=auto_convert,
auto_field=auto_field,
eager_load=eager_load,
auto_close=True,
),
callback_server_parameters=pyparams,
auto_convert=auto_convert,
start_callback_server=True,
eager_load=eager_load,
python_server_entry_point=PyEntryPoint(),
)
python_port = cls.gateway.get_callback_server().get_listening_port(
)
cls.gateway.java_gateway_server.resetCallbackClient(
cls.gateway.java_gateway_server.getCallbackClient().getAddress(
),
python_port,
)
return True
def testMultiClientServerWithSharedJavaThread(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(
JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# ## Preconditions
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 1 Hop to Shared Java Thread
# Check that the shared Java thread is the same thread
# for both ClientServers
sharedJavaThreadId = entry0.getSharedJavaThreadId()
self.assertEqual(sharedJavaThreadId,
entry1.getSharedJavaThreadId())
# And that it is distinct from either corresponding to
# this Python thread
self.assertNotEqual(sharedJavaThreadId, entry0.getJavaThreadId())
self.assertNotEqual(sharedJavaThreadId, entry1.getJavaThreadId())
# ## 2 Hops via Shared Java Thread to Python Thread
# Check that the shared thread ends up as different threads
# in the Python side. This part may not be obvious as the
# top-level idea seems to be for Python thread to be pinned
# to Java thread. Consider that this case is a simplification
# of the real case. In the real case there are two
# ClientServers running in same JVM, but in Python side each
# ClientServer is in its own process. In that case it makes
# it obvious that the shared Java thread should indeed
# end up in different Python threads.
sharedPythonThreadId0 = entry0.getSharedPythonThreadId()
sharedPythonThreadId1 = entry1.getSharedPythonThreadId()
# three way assert to make sure all three python
# threads are distinct
self.assertNotEqual(thisThreadId, sharedPythonThreadId0)
self.assertNotEqual(thisThreadId, sharedPythonThreadId1)
self.assertNotEqual(sharedPythonThreadId0, sharedPythonThreadId1)
# Check that the Python thread id does not change between
# invocations
self.assertEquals(sharedPythonThreadId0,
entry0.getSharedPythonThreadId())
self.assertEquals(sharedPythonThreadId1,
entry1.getSharedPythonThreadId())
# ## 3 Hops to Shared Java Thread
# Check that the thread above after 2 hops calls back
# into the Java shared thread
self.assertEqual(sharedJavaThreadId,
entry0.getSharedViaPythonJavaThreadId())
self.assertEqual(sharedJavaThreadId,
entry1.getSharedViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
def testMultiClientServer(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 0 Hops to Thread ID
# Check that the two thread getters get the same thread
self.assertEquals(thisThreadId, int(threadIdGetter0.getThreadId()))
self.assertEquals(thisThreadId, int(threadIdGetter1.getThreadId()))
# ## 1 Hop to Thread ID
# Check that ClientServers on Java side are on different threads
javaThreadId0 = entry0.getJavaThreadId()
javaThreadId1 = entry1.getJavaThreadId()
self.assertNotEqual(javaThreadId0, javaThreadId1)
# Check that ClientServers on Java side stay on same thread
# on subsequent calls to them
self.assertEqual(javaThreadId0, entry0.getJavaThreadId())
self.assertEqual(javaThreadId1, entry1.getJavaThreadId())
# Check alternate way of getting thread ids and that they match
self.assertEqual(javaThreadId0,
int(threadIdGetter0.getJavaThreadId()))
self.assertEqual(javaThreadId1,
int(threadIdGetter1.getJavaThreadId()))
# ## 2 Hops to Thread ID
# Check that round trips from Python to Java and Python
# end up back on this thread, regardless of which
# client server we use for the round trip
self.assertEqual(thisThreadId, entry0.getPythonThreadId())
self.assertEqual(thisThreadId, entry1.getPythonThreadId())
# ## 3 Hops to Thread ID
# Check that round trips from Python to Java to Python to Java
# end up on the same thread as 1 hop
self.assertEqual(javaThreadId0, entry0.getViaPythonJavaThreadId())
self.assertEqual(javaThreadId1, entry1.getViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
def testMultiClientServer(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(
JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 0 Hops to Thread ID
# Check that the two thread getters get the same thread
self.assertEquals(thisThreadId,
int(threadIdGetter0.getThreadId()))
self.assertEquals(thisThreadId,
int(threadIdGetter1.getThreadId()))
# ## 1 Hop to Thread ID
# Check that ClientServers on Java side are on different threads
javaThreadId0 = entry0.getJavaThreadId()
javaThreadId1 = entry1.getJavaThreadId()
self.assertNotEqual(javaThreadId0, javaThreadId1)
# Check that ClientServers on Java side stay on same thread
# on subsequent calls to them
self.assertEqual(javaThreadId0, entry0.getJavaThreadId())
self.assertEqual(javaThreadId1, entry1.getJavaThreadId())
# Check alternate way of getting thread ids and that they match
self.assertEqual(javaThreadId0,
int(threadIdGetter0.getJavaThreadId()))
self.assertEqual(javaThreadId1,
int(threadIdGetter1.getJavaThreadId()))
# ## 2 Hops to Thread ID
# Check that round trips from Python to Java and Python
# end up back on this thread, regardless of which
# client server we use for the round trip
self.assertEqual(thisThreadId,
entry0.getPythonThreadId())
self.assertEqual(thisThreadId,
entry1.getPythonThreadId())
# ## 3 Hops to Thread ID
# Check that round trips from Python to Java to Python to Java
# end up on the same thread as 1 hop
self.assertEqual(javaThreadId0,
entry0.getViaPythonJavaThreadId())
self.assertEqual(javaThreadId1,
entry1.getViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
outputs = {"status": "success", "response": list(model_features)}
return str(json.dumps(outputs))
except Exception as e:
raise e
msg = "Failure in executing {0}".format(sys.argv[0])
return str({"status": "error", "response": {"message": str(e)}})
class Java:
implements = ["com.apporchid.cloudseer.python.IPythonScript"]
from py4j.clientserver import ClientServer, JavaParameters, PythonParameters
# from AW_logger import awLogger
import sys
# logger = awLogger
import sys
pythonPort = int(sys.argv[1])
Demoops = DemoClassification()
#print(Demoops.executeScript({'input_dir':'C:/apporchid/prediction/'}))
gateway = ClientServer(
java_parameters=JavaParameters(),
python_parameters=PythonParameters(port = pythonPort),
python_server_entry_point = Demoops)
Demoops.clientserver = gateway