def testBadRetryFromJava(self):
"""Should not retry from Java to Python.
Similar use case as testBadRetry, but from Java: Java calls a long
Python operation.
If there is a bug, Java will call Python, then read will fail, then it
will call Python again.
If there is no bug, Java will call Python, read will fail, then Java
will raise an Exception that will be received as a Py4JError on the
Python side.
"""
client_server = ClientServer(JavaParameters(), PythonParameters())
with clientserver_example_app_process(False, True):
try:
operator = WaitOperator(0.5)
opExample = client_server.jvm.py4j.examples.OperatorExample()
opExample.randomBinaryOperator(operator)
self.fail(
"Should never retry once the first command went through."
" number of calls made: {0}".format(operator.callCount))
except Py4JError:
# XXX This occurs when WaitOperator tries to send a response to
# the Java side (this is slightly different then the
# GatewayServer equivalent where the Py4JError occurs on the
# clientserver, but the Java side can still send back an
# exception to the JavaGateway).
self.assertTrue(True)
finally:
client_server.shutdown()
def test_all_regular_signals(self):
listener = MockListener(self)
server_started.connect(listener.started)
hello_state = HelloState()
client_server = ClientServer(JavaParameters(), PythonParameters(),
hello_state)
server_stopped.connect(listener.stopped,
sender=client_server.get_callback_server())
server_connection_started.connect(
listener.connection_started,
sender=client_server.get_callback_server())
server_connection_stopped.connect(
listener.connection_stopped,
sender=client_server.get_callback_server())
pre_server_shutdown.connect(listener.pre_shutdown,
sender=client_server.get_callback_server())
post_server_shutdown.connect(
listener.post_shutdown, sender=client_server.get_callback_server())
with clientserver_example_app_process(True):
client_server.shutdown()
self.assertEqual(1, listener.received["started"])
self.assertEqual(1, listener.received["stopped"])
self.assertEqual(1, listener.received["pre_shutdown"])
self.assertEqual(1, listener.received["post_shutdown"])
self.assertEqual(1, listener.received["connection_started"])
self.assertEqual(1, listener.received["connection_stopped"])
def testRecursionWithAutoGC(self):
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(auto_gc=True),
PythonParameters(auto_gc=True))
pingpong = client_server.jvm.py4j.examples.PingPong()
ping = PythonPing()
self.assertEqual(2, pingpong.start(ping))
pingpong = client_server.jvm.py4j.examples.PingPong(True)
try:
pingpong.start(ping)
self.fail()
except Py4JJavaError:
# TODO Make sure error are recursively propagated
# Also a problem with old threading model.
self.assertTrue(True)
ping = PythonPing(True)
pingpong = client_server.jvm.py4j.examples.PingPong(False)
try:
pingpong.start(ping)
self.fail()
except Py4JJavaError:
# TODO Make sure error are recursively propagated
# Also a problem with old threading model.
self.assertTrue(True)
client_server.shutdown()
def connect_to_IDE():
java_port = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT
# print("initiating Python ClientServer, connecting to port " + str(java_port))
entry_point = PythonEntryPoint()
gateway = ClientServer(java_parameters=JavaParameters(
address="127.0.0.1",
port=java_port,
auto_field=True,
auto_convert=True,
auto_close=True,
auth_token=os.environ["PY4J_AUTH_TOKEN"]),
python_parameters=PythonParameters(
address="127.0.0.1",
port=0,
daemonize=True,
daemonize_connections=True),
python_server_entry_point=entry_point)
# Even though this won't erase it from the environment of the process itself at
# the system level, at least it will be more difficult to access from Python.
del os.environ["PY4J_AUTH_TOKEN"]
Gateway.gateway = gateway
python_port = gateway.get_callback_server().get_listening_port()
# telling Java which port we listen on
address = gateway.java_gateway_server.getCallbackClient().getAddress()
gateway.java_gateway_server.resetCallbackClient(address, python_port)
def testJavaGC(self):
# This will only work with some JVM.
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(), PythonParameters())
example = client_server.entry_point.getNewExample()
impl = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl))
self.assertEqual("This is Hello;\n10MyMy!\n;",
example.callHello2(impl))
self.assertEqual(2, len(client_server.gateway_property.pool))
# Make sure that finalizers do not block by calling the Java
# finalizer again
impl2 = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl2))
self.assertEqual(3, len(client_server.gateway_property.pool))
# The two PythonProxies should be evicted on the Java side
# Java should tell python to release the references.
client_server.jvm.java.lang.System.gc()
# Leave time for sotimeout
sleep(3)
self.assertTrue(len(client_server.gateway_property.pool) < 2)
client_server.shutdown()
def testErrorInPythonCallback(self):
with clientserver_example_app_process():
client_server = ClientServer(
JavaParameters(),
PythonParameters(propagate_java_exceptions=True))
example = client_server.entry_point.getNewExample()
try:
example.callHello(
IHelloFailingImpl(
ValueError('My interesting Python exception')))
self.fail()
except Py4JJavaError as e:
self.assertTrue(
is_instance_of(client_server, e.java_exception,
'py4j.Py4JException'))
self.assertTrue('interesting Python exception' in str(e))
try:
example.callHello(
IHelloFailingImpl(
Py4JJavaError(
'',
client_server.jvm.java.lang.IllegalStateException(
'My IllegalStateException'))))
self.fail()
except Py4JJavaError as e:
self.assertTrue(
is_instance_of(client_server, e.java_exception,
'java.lang.IllegalStateException'))
client_server.shutdown()
def testGoodRetryFromJava(self):
"""Should retry from Java to Python.
Similar use case as testGoodRetry, but from Java: Python calls Java,
which calls Python back. Then Java waits for a while and calls Python
again.
Because Python Server has been waiting for too much time, the
receiving socket has closed so the call from Java to Python will fail
on send, and Java must retry by creating a new connection
(ClientServerConnection).
Because ClientServer reuses the same connection in each thread, we must
launch a new thread on the Java side to correctly test the Python
Server.
"""
client_server = ClientServer(JavaParameters(),
PythonParameters(read_timeout=0.250))
with clientserver_example_app_process():
try:
operator = WaitOperator(0)
opExample = client_server.jvm.py4j.examples.OperatorExample()
opExample.launchOperator(operator, 500)
sleep(0.1)
str_connection = str(
list(client_server._callback_server.connections)[0])
sleep(0.75)
str_connection2 = str(
list(client_server._callback_server.connections)[0])
self.assertNotEqual(str_connection, str_connection2)
except Py4JJavaError:
self.fail("Callbackserver did not retry.")
finally:
client_server.shutdown()
def test_signals_started_from_python(self):
listener = MockListener(self)
with clientserver_example_app_process():
server_started.connect(listener.started)
client_server = ClientServer(JavaParameters(), PythonParameters())
example = client_server.entry_point.getNewExample()
impl = IHelloImpl()
self.assertEqual("This is Hello!", example.callHello(impl))
server_stopped.connect(listener.stopped,
sender=client_server.get_callback_server())
server_connection_started.connect(
listener.connection_started,
sender=client_server.get_callback_server())
server_connection_stopped.connect(
listener.connection_stopped,
sender=client_server.get_callback_server())
pre_server_shutdown.connect(
listener.pre_shutdown,
sender=client_server.get_callback_server())
post_server_shutdown.connect(
listener.post_shutdown,
sender=client_server.get_callback_server())
client_server.shutdown()
self.assertEqual(1, listener.received["started"])
self.assertEqual(1, listener.received["stopped"])
self.assertEqual(1, listener.received["pre_shutdown"])
self.assertEqual(1, listener.received["post_shutdown"])
# Connection initiated from JavaClient, so no signal sent.
self.assertEqual(0, listener.received["connection_started"])
self.assertEqual(0, listener.received["connection_stopped"])
def get_pinned_thread_gateway():
"""Get Py4J ClientServer that can work with both sides.
"""
from py4j.clientserver import (
ClientServer, PythonParameters, JavaParameters)
client_server = ClientServer(
java_parameters=JavaParameters(), python_parameters=PythonParameters())
return client_server
def internal_work():
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT + 5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5))
sleep()
play_with_ping(clientserver2)
python_gc()
sleep()
clientserver2.shutdown()
def __init__(self, port, pyport, kernel_name, context):
try:
pep = PythonEntryPoint(kernel_name, context)
except NoSuchKernel:
sys.exit(2)
ClientServer(java_parameters=JavaParameters(port=int(port)),
python_parameters=PythonParameters(port=int(pyport)),
python_server_entry_point=pep)
print('Py4j server is running')
def testJavaClientPythonServer(self):
hello_state = HelloState()
client_server = ClientServer(JavaParameters(), PythonParameters(),
hello_state)
with clientserver_example_app_process(True):
client_server.shutdown()
# Check that Java correctly called Python
self.assertEqual(2, len(hello_state.calls))
self.assertEqual((None, None), hello_state.calls[0])
self.assertEqual((2, "Hello World"), hello_state.calls[1])
def testErrorInPy4J(self):
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(), PythonParameters())
try:
client_server.jvm.java.lang.Math.abs(
3000000000000000000000000000000000000)
self.fail("Should not be able to convert overflowing long")
except Py4JError:
self.assertTrue(True)
# Check that the connection is not broken (refs #265)
val = client_server.jvm.java.lang.Math.abs(-4)
self.assertEqual(4, val)
client_server.shutdown()
def main(port: int):
initialize_logging()
register_input_converter(DateTimeConverter())
gateway = ClientServer(java_parameters=JavaParameters(eager_load=False,
auto_convert=True,
auto_field=True),
python_parameters=PythonParameters(port=port,
eager_load=True),
python_server_entry_point=Runner())
register_output_converter(
proto.REFERENCE_TYPE,
lambda target_id, gateway_client: convert_reference_type(
target_id, gateway_client, gateway))
def run_image_classifier_service(model):
gateway = None
try:
model_file = None
workspace_dir = None
# Read the arguments.
opts, args = getopt.getopt(sys.argv[1:], 'm:w:', ['modelfile=', 'workspacedir='])
for opt, arg in opts:
if opt in ('-m', '--modelfile'):
model_file = arg
elif opt in ('-w', '--workspacedir'):
workspace_dir = arg
if model_file is None:
raise Exception('Model file not provided')
print('Using model file: {}, workspace directory: {}.'.format(model_file,
workspace_dir), flush=True)
def sigterm_handler(_signo, _stack_frame):
if gateway:
gateway.shutdown()
sys.exit(0)
signal.signal(signal.SIGTERM, sigterm_handler)
service = ImageClassifierService(model, model_file, workspace_dir)
py4j_java_port = int(os.getenv('PY4J_JAVA_PORT', -1))
py4j_python_port = int(os.getenv('PY4J_PYTHON_PORT', -1))
py4j_auth_token = os.getenv('PY4J_AUTH_TOKEN')
print('Using ports {:d} for Java, {:d} for Python (-1 means the default port).'.format(py4j_java_port,
py4j_python_port), flush=True)
gateway = ClientServer(python_server_entry_point=service,
java_parameters=JavaParameters(
port=py4j_java_port if py4j_java_port > -1 else DEFAULT_PORT, auth_token=py4j_auth_token),
python_parameters=PythonParameters(
port=py4j_python_port if py4j_python_port > -1 else DEFAULT_PYTHON_PROXY_PORT, auth_token=py4j_auth_token))
service.startup(gateway)
print('The service has started.', flush=True)
except:
print('The service error.')
traceback.print_exc(file=sys.stdout)
print('The service has failed to start.', flush=True)
finally:
sys.stdout.flush()
sys.stderr.flush()
def internal_work(assert_memory):
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT + 5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5))
sleep()
play_with_ping(clientserver2)
python_gc()
sleep()
clientserver2.close(close_callback_server_connections=True,
keep_callback_server=True)
sleep()
assert_memory()
clientserver2.shutdown()
sleep()
def get_client_server():
global client_server
if client_server is None:
client_server = ClientServer(
java_parameters=JavaParameters(auth_token=os.environ["AUTH_TOKEN"],
port=3071 + 653,
auto_field=True,
auto_convert=True,
auto_close=True),
python_parameters=PythonParameters(port=0,
daemonize=True,
daemonize_connections=True),
python_server_entry_point=None)
return client_server
def testSendObjects(self):
"""This test receives 1000 calls creating object cycles.
Typically, the garbage collector will starts in the middle of
Py4J connection code, which will create an error if the garbage
collection is done on the same thread.
"""
hello = HelloObjects()
client_server = ClientServer(JavaParameters(), PythonParameters(),
hello)
with clientserver_example_app_process(start_gc_test=True,
join=False) as p:
p.join()
client_server.shutdown()
self.assertEquals(1000, hello.calls)
def connect(cls, auto_load_java: bool = False) -> bool:
"""
connects to jnegmas
"""
if auto_load_java:
if cls.gateway is None:
cls.start_java_side()
return True
if cls.gateway is None:
cls.gateway = ClientServer(
java_parameters=JavaParameters(
port=JNegmasGateway.DEFAULT_JAVA_PORT, auto_convert=True),
python_parameters=PythonParameters(
port=JNegmasGateway.DEFAULT_PYTHON_PORT,
propagate_java_exceptions=True))
return True
def internal_work(clientserver):
hello_state = HelloState2()
clientserver2 = InstrClientServer(
JavaParameters(port=DEFAULT_PORT + 5),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5),
python_server_entry_point=hello_state)
hello_state.gateway = clientserver2
sleep()
clientserver.entry_point.startServerWithPythonEntry(False)
sleep()
clientserver2.shutdown()
# Check that Java correctly called Python
self.assertEqual(2, len(hello_state.calls))
self.assertEqual((None, None), hello_state.calls[0])
self.assertEqual((2, "Hello World"), hello_state.calls[1])
def connect_to_IDE():
java_port = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT
# print("initiating Python ClientServer, connecting to port " + str(java_port))
entry_point = PythonEntryPoint()
gateway = ClientServer(
java_parameters=JavaParameters(port=java_port, auto_field=True, auto_convert=True, auto_close=True),
python_parameters=PythonParameters(port=0, daemonize=True, daemonize_connections=True),
python_server_entry_point=entry_point)
Gateway.gateway = gateway
python_port = gateway.get_callback_server().get_listening_port()
# telling Java which port we listen on
address = gateway.java_gateway_server.getCallbackClient().getAddress()
gateway.java_gateway_server.resetCallbackClient(address, python_port)
def testStream(self):
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(), PythonParameters())
e = client_server.entry_point.getNewExample()
# not binary - just get the Java object
v1 = e.getStream()
self.assertTrue(
is_instance_of(client_server, v1,
"java.nio.channels.ReadableByteChannel"))
# pull it as a binary stream
with e.getStream.stream() as conn:
self.assertTrue(isinstance(conn, GatewayConnectionGuard))
expected =\
u"Lorem ipsum dolor sit amet, consectetur adipiscing elit."
self.assertEqual(expected,
smart_decode(conn.read(len(expected))))
client_server.shutdown()
def testGoodRetry(self):
"""Should retry from Python to Java.
Python calls Java twice in a row, then waits, then calls again.
Java fails when it does not receive calls quickly.
If there is a bug, Python will fail on the third call because the Java
connection was closed and it did not retry.
If there is a bug, Python might not fail because Java did not close the
connection on timeout. The connection used to call Java will be the
same one for all calls (and an assertion will fail).
If there is no bug, Python will call Java twice with the same
connection. On the third call, the write will fail, and a new
connection will be created.
"""
client_server = ClientServer(JavaParameters(), PythonParameters())
connections = client_server._gateway_client.deque
with clientserver_example_app_process(False, True):
try:
# Call #1
client_server.jvm.System.currentTimeMillis()
str_connection = str(connections[0])
# Call #2
client_server.jvm.System.currentTimeMillis()
self.assertEqual(1, len(connections))
str_connection2 = str(connections[0])
self.assertEqual(str_connection, str_connection2)
sleep(0.5)
client_server.jvm.System.currentTimeMillis()
self.assertEqual(1, len(connections))
str_connection3 = str(connections[0])
# A new connection was automatically created.
self.assertNotEqual(str_connection, str_connection3)
except Py4JError:
self.fail("Should retry automatically by default.")
finally:
client_server.shutdown()
def testPythonGC(self):
def internal_function(client_server):
example = client_server.entry_point.getNewExample()
in_middle = len(
client_server.java_gateway_server.getGateway().getBindings())
self.assertEqual(1, example.method1())
# After this method is executed, Python no longer need
# a reference to example and it should tell Java to release
# the reference too
return in_middle
with clientserver_example_app_process():
# This will only work with some JVM.
client_server = ClientServer(JavaParameters(), PythonParameters())
before = len(
client_server.java_gateway_server.getGateway().getBindings())
in_middle = internal_function(client_server)
# Force Python to run garbage collection in case it did
# not run yet.
sleep()
gc.collect()
# Long sleep to ensure that the finalizer worker received
# everything
sleep(2)
after = len(
client_server.java_gateway_server.getGateway().getBindings())
# Number of references on the Java side should be the same before
# and after
self.assertEqual(before, after)
# Number of references when we created a JavaObject should be
# higher than at the beginning.
self.assertGreater(in_middle, before)
client_server.shutdown()
def testBadRetry(self):
"""Should not retry from Python to Java.
Python calls a long Java method. The call goes through, but the
response takes a long time to get back.
If there is a bug, Python will fail on read and retry (sending the same
call twice).
If there is no bug, Python will fail on read and raise an Exception.
"""
client_server = ClientServer(JavaParameters(read_timeout=0.250),
PythonParameters())
with clientserver_example_app_process():
try:
example = client_server.jvm.py4j.examples.ExampleClass()
value = example.sleepFirstTimeOnly(500)
self.fail(
"Should never retry once the first command went through."
"number of calls made: {0}".format(value))
except Py4JError:
self.assertTrue(True)
finally:
client_server.shutdown()
def launch_gateway(conf=None, popen_kwargs=None):
"""
launch jvm gateway
Parameters
----------
conf : :py:class:`pyspark.SparkConf`
spark configuration passed to spark-submit
popen_kwargs : dict
Dictionary of kwargs to pass to Popen when spawning
the py4j JVM. This is a developer feature intended for use in
customizing how pyspark interacts with the py4j JVM (e.g., capturing
stdout/stderr).
Returns
-------
ClientServer or JavaGateway
"""
if "PYSPARK_GATEWAY_PORT" in os.environ:
gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
gateway_secret = os.environ["PYSPARK_GATEWAY_SECRET"]
# Process already exists
proc = None
else:
SPARK_HOME = _find_spark_home()
# Launch the Py4j gateway using Spark's run command so that we pick up the
# proper classpath and settings from spark-env.sh
on_windows = platform.system() == "Windows"
script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
command = [os.path.join(SPARK_HOME, script)]
if conf:
for k, v in conf.getAll():
command += ["--conf", "%s=%s" % (k, v)]
submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
if os.environ.get("SPARK_TESTING"):
submit_args = " ".join(
["--conf spark.ui.enabled=false", submit_args])
command = command + shlex.split(submit_args)
# Create a temporary directory where the gateway server should write the connection
# information.
conn_info_dir = tempfile.mkdtemp()
try:
fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
os.close(fd)
os.unlink(conn_info_file)
env = dict(os.environ)
env["_PYSPARK_DRIVER_CONN_INFO_PATH"] = conn_info_file
# Launch the Java gateway.
popen_kwargs = {} if popen_kwargs is None else popen_kwargs
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
popen_kwargs["stdin"] = PIPE
# We always set the necessary environment variables.
popen_kwargs["env"] = env
if not on_windows:
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
popen_kwargs["preexec_fn"] = preexec_func
proc = Popen(command, **popen_kwargs)
else:
# preexec_fn not supported on Windows
proc = Popen(command, **popen_kwargs)
# Wait for the file to appear, or for the process to exit, whichever happens first.
while not proc.poll() and not os.path.isfile(conn_info_file):
time.sleep(0.1)
if not os.path.isfile(conn_info_file):
raise RuntimeError(
"Java gateway process exited before sending its port number"
)
with open(conn_info_file, "rb") as info:
gateway_port = read_int(info)
gateway_secret = UTF8Deserializer().loads(info)
finally:
shutil.rmtree(conn_info_dir)
# In Windows, ensure the Java child processes do not linger after Python has exited.
# In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
# the parent process' stdin sends an EOF). In Windows, however, this is not possible
# because java.lang.Process reads directly from the parent process' stdin, contending
# with any opportunity to read an EOF from the parent. Note that this is only best
# effort and will not take effect if the python process is violently terminated.
if on_windows:
# In Windows, the child process here is "spark-submit.cmd", not the JVM itself
# (because the UNIX "exec" command is not available). This means we cannot simply
# call proc.kill(), which kills only the "spark-submit.cmd" process but not the
# JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
# child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
def killChild():
Popen([
"cmd", "/c", "taskkill", "/f", "/t", "/pid",
str(proc.pid)
])
atexit.register(killChild)
# Connect to the gateway (or client server to pin the thread between JVM and Python)
if os.environ.get("PYSPARK_PIN_THREAD", "true").lower() == "true":
gateway = ClientServer(
java_parameters=JavaParameters(port=gateway_port,
auth_token=gateway_secret,
auto_convert=True),
python_parameters=PythonParameters(port=0, eager_load=False),
)
else:
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=gateway_port, auth_token=gateway_secret, auto_convert=True))
# Store a reference to the Popen object for use by the caller (e.g., in reading stdout/stderr)
gateway.proc = proc
# Import the classes used by PySpark
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.ml.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
java_import(gateway.jvm, "org.apache.spark.resource.*")
# TODO(davies): move into sql
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
return gateway
implements = ["main/java/jty.Predict_Proba"]
train_X = np.array(np.random.randint(0, 10, size=200000).reshape(1000, 200))
train_y = np.array(np.random.randint(0, 2, size=1000))
test_X = np.array(np.random.randint(0, 10, size=800).reshape(4, 200))
model = lgb.LGBMClassifier(
boosting_type='gbdt', # 提升树的类型
metric='auc', #
num_leaves=31, # 树的最大叶子数
reg_alpha=0.25, # L1正则化系数
reg_lambda=0.25, # L2正则化系数
max_depth=-1, # 最大树的深度
n_estimators=30000, # 拟合的树的棵树，相当于训练轮数
subsample=0.75, # 训练样本采样率 行
colsample_bytree=0.75, # 训练样本采样率 列
subsample_freq=1, # 子样本频率
learning_rate=0.03, # 学习率
min_child_weight=10, # 分支节点的最小权重
random_state=2019, # 随机种子数
n_jobs=8,
)
model.fit(train_X, train_y)
print("f")
demo = Demo()
gateway = ClientServer(java_parameters=JavaParameters(),
python_parameters=PythonParameters(),
python_server_entry_point=demo)
def testMultiClientServer(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 0 Hops to Thread ID
# Check that the two thread getters get the same thread
self.assertEquals(thisThreadId, int(threadIdGetter0.getThreadId()))
self.assertEquals(thisThreadId, int(threadIdGetter1.getThreadId()))
# ## 1 Hop to Thread ID
# Check that ClientServers on Java side are on different threads
javaThreadId0 = entry0.getJavaThreadId()
javaThreadId1 = entry1.getJavaThreadId()
self.assertNotEqual(javaThreadId0, javaThreadId1)
# Check that ClientServers on Java side stay on same thread
# on subsequent calls to them
self.assertEqual(javaThreadId0, entry0.getJavaThreadId())
self.assertEqual(javaThreadId1, entry1.getJavaThreadId())
# Check alternate way of getting thread ids and that they match
self.assertEqual(javaThreadId0,
int(threadIdGetter0.getJavaThreadId()))
self.assertEqual(javaThreadId1,
int(threadIdGetter1.getJavaThreadId()))
# ## 2 Hops to Thread ID
# Check that round trips from Python to Java and Python
# end up back on this thread, regardless of which
# client server we use for the round trip
self.assertEqual(thisThreadId, entry0.getPythonThreadId())
self.assertEqual(thisThreadId, entry1.getPythonThreadId())
# ## 3 Hops to Thread ID
# Check that round trips from Python to Java to Python to Java
# end up on the same thread as 1 hop
self.assertEqual(javaThreadId0, entry0.getViaPythonJavaThreadId())
self.assertEqual(javaThreadId1, entry1.getViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
def testMultiClientServerWithSharedJavaThread(self):
with java_multi_client_server_app_process():
client_server0 = ClientServer(JavaParameters(), PythonParameters())
client_server1 = ClientServer(
JavaParameters(port=DEFAULT_PORT + 2),
PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))
entry0 = client_server0.entry_point
entry1 = client_server1.entry_point
# set up the ability for Java to get the Python thread ids
threadIdGetter0 = PythonGetThreadId(client_server0)
threadIdGetter1 = PythonGetThreadId(client_server1)
entry0.setPythonThreadIdGetter(threadIdGetter0)
entry1.setPythonThreadIdGetter(threadIdGetter1)
thisThreadId = threading.current_thread().ident
# ## Preconditions
# Make sure we are talking to two different Entry points on
# Java side
self.assertEqual(0, entry0.getEntryId())
self.assertEqual(1, entry1.getEntryId())
# ## 1 Hop to Shared Java Thread
# Check that the shared Java thread is the same thread
# for both ClientServers
sharedJavaThreadId = entry0.getSharedJavaThreadId()
self.assertEqual(sharedJavaThreadId,
entry1.getSharedJavaThreadId())
# And that it is distinct from either corresponding to
# this Python thread
self.assertNotEqual(sharedJavaThreadId, entry0.getJavaThreadId())
self.assertNotEqual(sharedJavaThreadId, entry1.getJavaThreadId())
# ## 2 Hops via Shared Java Thread to Python Thread
# Check that the shared thread ends up as different threads
# in the Python side. This part may not be obvious as the
# top-level idea seems to be for Python thread to be pinned
# to Java thread. Consider that this case is a simplification
# of the real case. In the real case there are two
# ClientServers running in same JVM, but in Python side each
# ClientServer is in its own process. In that case it makes
# it obvious that the shared Java thread should indeed
# end up in different Python threads.
sharedPythonThreadId0 = entry0.getSharedPythonThreadId()
sharedPythonThreadId1 = entry1.getSharedPythonThreadId()
# three way assert to make sure all three python
# threads are distinct
self.assertNotEqual(thisThreadId, sharedPythonThreadId0)
self.assertNotEqual(thisThreadId, sharedPythonThreadId1)
self.assertNotEqual(sharedPythonThreadId0, sharedPythonThreadId1)
# Check that the Python thread id does not change between
# invocations
self.assertEquals(sharedPythonThreadId0,
entry0.getSharedPythonThreadId())
self.assertEquals(sharedPythonThreadId1,
entry1.getSharedPythonThreadId())
# ## 3 Hops to Shared Java Thread
# Check that the thread above after 2 hops calls back
# into the Java shared thread
self.assertEqual(sharedJavaThreadId,
entry0.getSharedViaPythonJavaThreadId())
self.assertEqual(sharedJavaThreadId,
entry1.getSharedViaPythonJavaThreadId())
client_server0.shutdown()
client_server1.shutdown()
def testBasicJVM(self):
with clientserver_example_app_process():
client_server = ClientServer(JavaParameters(), PythonParameters())
ms = client_server.jvm.System.currentTimeMillis()
self.assertTrue(ms > 0)
client_server.shutdown()
