def testGoodRetryFromJava(self): """Should retry from Java to Python. Similar use case as testGoodRetry, but from Java: Python calls Java, which calls Python back. Then Java waits for a while and calls Python again. Because Python Server has been waiting for too much time, the receiving socket has closed so the call from Java to Python will fail on send, and Java must retry by creating a new connection (ClientServerConnection). Because ClientServer reuses the same connection in each thread, we must launch a new thread on the Java side to correctly test the Python Server. """ client_server = ClientServer(JavaParameters(), PythonParameters(read_timeout=0.250)) with clientserver_example_app_process(): try: operator = WaitOperator(0) opExample = client_server.jvm.py4j.examples.OperatorExample() opExample.launchOperator(operator, 500) sleep(0.1) str_connection = str( list(client_server._callback_server.connections)[0]) sleep(0.75) str_connection2 = str( list(client_server._callback_server.connections)[0]) self.assertNotEqual(str_connection, str_connection2) except Py4JJavaError: self.fail("Callbackserver did not retry.") finally: client_server.shutdown()
def testGoodRetryFromJava(self): """Should retry from Java to Python. Similar use case as testGoodRetry, but from Java: Python calls Java, which calls Python back. Then Java waits for a while and calls Python again. Because Python Server has been waiting for too much time, the receiving socket has closed so the call from Java to Python will fail on send, and Java must retry by creating a new connection (ClientServerConnection). Because ClientServer reuses the same connection in each thread, we must launch a new thread on the Java side to correctly test the Python Server. """ client_server = ClientServer( JavaParameters(), PythonParameters(read_timeout=0.250)) with clientserver_example_app_process(): try: operator = WaitOperator(0) opExample = client_server.jvm.py4j.examples.OperatorExample() opExample.launchOperator(operator, 500) sleep(0.1) str_connection = str( list(client_server._callback_server.connections)[0]) sleep(0.75) str_connection2 = str( list(client_server._callback_server.connections)[0]) self.assertNotEqual(str_connection, str_connection2) except Py4JJavaError: self.fail("Callbackserver did not retry.") finally: client_server.shutdown()
def start_clientserver_example_app_process( start_java_client=False, start_short_timeout=False, start_gc_test=False, auth_token=None): args = () gw_params = GatewayParameters() if auth_token: args = ("--auth-token", auth_token) gw_params = GatewayParameters(auth_token=auth_token) # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if start_short_timeout: p = Process(target=start_short_timeout_clientserver_example_server, args=args) elif start_java_client: p = Process(target=start_java_clientserver_example_server, args=args) elif start_gc_test: p = Process(target=start_java_clientserver_gc_example_server, args=args) else: p = Process(target=start_clientserver_example_server, args=args) p.start() sleep() check_connection(gateway_parameters=gw_params) return p
def testJavaGC(self): # This will only work with some JVM. with clientserver_example_app_process(): client_server = ClientServer(JavaParameters(), PythonParameters()) example = client_server.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl)) self.assertEqual(2, len(client_server.gateway_property.pool)) # Make sure that finalizers do not block by calling the Java # finalizer again impl2 = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl2)) self.assertEqual(3, len(client_server.gateway_property.pool)) # The two PythonProxies should be evicted on the Java side # Java should tell python to release the references. client_server.jvm.java.lang.System.gc() # Leave time for sotimeout sleep(3) self.assertTrue(len(client_server.gateway_property.pool) < 2) client_server.shutdown()
def example_app_process(): p = start_example_app_process() try: yield p finally: p.join() sleep()
def setUp(self): key_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "selfsigned.pem") client_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) client_ssl_context.verify_mode = ssl.CERT_REQUIRED client_ssl_context.check_hostname = True client_ssl_context.load_verify_locations(cafile=key_file) server_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) server_ssl_context.load_cert_chain(key_file, password='******') callback_server_parameters = CallbackServerParameters( ssl_context=server_ssl_context) # address must match cert, because we're checking hostnames gateway_parameters = GatewayParameters( address='localhost', ssl_context=client_ssl_context) self.p = start_example_tls_process() self.gateway = JavaGateway( gateway_parameters=gateway_parameters, callback_server_parameters=callback_server_parameters) # It seems SecureServerSocket may need a little more time to # initialize on some platforms/slow machines. sleep(0.500)
def perform_memory_tests(): python_gc() clientserver.jvm.py4j.instrumented.MetricRegistry.\ forceFinalization() sleep() createdSet = clientserver.jvm.py4j.instrumented.\ MetricRegistry.getCreatedObjectsKeySet() finalizedSet = clientserver.jvm.py4j.instrumented.\ MetricRegistry.getFinalizedObjectsKeySet() # 6 objects: ClientServer, JavaServer, # PythonClient, 3 ClientServerConnection. self.assertEqual(6, len(createdSet)) # Should be 2: ClientServer, 1 ClientServerConnection # But for some reasons, Java refuses to collect the # clientserverconnection even though there are no strong # references. self.assertEqual(1, len(finalizedSet)) # 8 objects: ClientServer, PythonServer, JavaClient, # GatewayProperty, PythonPing, 3 ClientServerConnection self.assertEqual(8, len(CREATED)) # PythonPing + ClientServerConnection self.assertEqual(2, len(FINALIZED))
def start_clientserver_example_app_process(start_java_client=False, start_short_timeout=False, start_gc_test=False, auth_token=None): args = () gw_params = GatewayParameters() if auth_token: args = ("--auth-token", auth_token) gw_params = GatewayParameters(auth_token=auth_token) # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if start_short_timeout: p = Process(target=start_short_timeout_clientserver_example_server, args=args) elif start_java_client: p = Process(target=start_java_clientserver_example_server, args=args) elif start_gc_test: p = Process(target=start_java_clientserver_gc_example_server, args=args) else: p = Process(target=start_clientserver_example_server, args=args) p.start() sleep() check_connection(gateway_parameters=gw_params) return p
def start_example_app_process2(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_example_server2) p.start() sleep() check_connection() return p
def testBytes(self): sleep() operator = CustomBytesOperator() returnbytes = self.gateway.entry_point.callBytesOperator(operator) self.assertEqual(2, returnbytes[0]) self.assertEqual(6, returnbytes[-1]) sleep()
def start_example_app_process3(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_example_server3) p.start() sleep() test_gateway_connection() return p
def testGC(self): with gateway_example_app_process("nomem"): # This will only work with some JVM. gateway = JavaGateway( callback_server_parameters=CallbackServerParameters()) sleep() example = gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual( "This is Hello;\n10MyMy!\n;", example.callHello2(impl)) self.assertEqual(2, len(gateway.gateway_property.pool)) # Make sure that finalizers do not block impl2 = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl2)) self.assertEqual(3, len(gateway.gateway_property.pool)) gateway.jvm.java.lang.System.gc() # Leave time for sotimeout sleep(3) # Make sure the three objects have not been removed from the pool # because the Java side should not send gc request. self.assertEqual(len(gateway.gateway_property.pool), 3) gateway.shutdown()
def testProxy(self): sleep() example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl))
def testProxyReturnerFloatErrorTypeConversion(self): sleep() example = self.gateway.jvm.py4j.examples.ReturnerExample() returner = Returner(bad_type=True) self.assertRaises( Py4JJavaError, example.computeFloat, returner)
def testProxyReturnerIntOverflow(self): sleep() example = self.gateway.jvm.py4j.examples.ReturnerExample() returner = Returner(bad_type=True) self.assertRaises( Py4JJavaError, example.computeInt, returner)
def testProxy(self): # self.gateway.jvm.py4j.GatewayServer.turnLoggingOn() sleep() example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl))
def testGC(self): with gateway_example_app_process("nomem"): # This will only work with some JVM. gateway = JavaGateway( callback_server_parameters=CallbackServerParameters()) sleep() example = gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl)) self.assertEqual(2, len(gateway.gateway_property.pool)) # Make sure that finalizers do not block impl2 = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl2)) self.assertEqual(3, len(gateway.gateway_property.pool)) gateway.jvm.java.lang.System.gc() # Leave time for sotimeout sleep(3) # Make sure the three objects have not been removed from the pool # because the Java side should not send gc request. self.assertEqual(len(gateway.gateway_property.pool), 3) gateway.shutdown()
def testJavaGC(self): # This will only work with some JVM. with clientserver_example_app_process(): client_server = ClientServer( JavaParameters(), PythonParameters()) example = client_server.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual( "This is Hello;\n10MyMy!\n;", example.callHello2(impl)) self.assertEqual(2, len(client_server.gateway_property.pool)) # Make sure that finalizers do not block by calling the Java # finalizer again impl2 = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl2)) self.assertEqual(3, len(client_server.gateway_property.pool)) # The two PythonProxies should be evicted on the Java side # Java should tell python to release the references. client_server.jvm.java.lang.System.gc() # Leave time for sotimeout sleep(3) self.assertTrue(len(client_server.gateway_property.pool) < 2) client_server.shutdown()
def setUp(self): key_file = os.path.join( os.path.dirname(os.path.realpath(__file__)), "selfsigned.pem") client_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) client_ssl_context.verify_mode = ssl.CERT_REQUIRED client_ssl_context.check_hostname = True client_ssl_context.load_verify_locations(cafile=key_file) server_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) server_ssl_context.load_cert_chain(key_file, password='******') callback_server_parameters = CallbackServerParameters( ssl_context=server_ssl_context) # address must match cert, because we're checking hostnames gateway_parameters = GatewayParameters( address='localhost', ssl_context=client_ssl_context) self.p = start_example_tls_process() self.gateway = JavaGateway( gateway_parameters=gateway_parameters, callback_server_parameters=callback_server_parameters) sleep()
def internal_work(): gateway2 = InstrJavaGateway(gateway_parameters=GatewayParameters( port=DEFAULT_PORT+5)) sleep() work_with_object(gateway2) python_gc() sleep() gateway2.shutdown()
def internal_work(): gateway2 = InstrJavaGateway(gateway_parameters=GatewayParameters( port=DEFAULT_PORT + 5)) sleep() work_with_object(gateway2) python_gc() sleep() gateway2.shutdown()
def testProxy(self): sleep() example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual( "This is Hello;\n10MyMy!\n;", example.callHello2(impl))
def testProxy(self): # self.gateway.jvm.py4j.GatewayServer.turnLoggingOn() sleep() example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual('This is Hello!', example.callHello(impl)) self.assertEqual('This is Hello;\n10MyMy!\n;', example.callHello2(impl))
def internal_work(): clientserver2 = InstrClientServer( JavaParameters(port=DEFAULT_PORT + 5), PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5)) sleep() play_with_ping(clientserver2) python_gc() sleep() clientserver2.shutdown()
def testMethodConstructor(self): sleep() adder = Adder() oe1 = self.gateway.jvm.py4j.examples.OperatorExample() # Test method oe1.randomBinaryOperator(adder) # Test constructor oe2 = self.gateway.jvm.py4j.examples.OperatorExample(adder) self.assertTrue(oe2 is not None)
def internal_work(): clientserver2 = InstrClientServer( JavaParameters(port=DEFAULT_PORT+5), PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5)) sleep() play_with_ping(clientserver2) python_gc() sleep() clientserver2.shutdown()
def testMethodConstructor(self): sleep() goodAddition = GoodAddition() oe1 = self.gateway.jvm.py4j.examples.OperatorExample() # Test method oe1.randomBinaryOperator(goodAddition) # Test constructor oe2 = self.gateway.jvm.py4j.examples.OperatorExample(goodAddition) self.assertTrue(oe2 is not None)
def start_clientserver_example_app_process(start_java_client=False): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if not start_java_client: p = Process(target=start_clientserver_example_server) else: p = Process(target=start_java_clientserver_example_server) p.start() sleep() test_gateway_connection() return p
def testPeriodicCleanup(self): operator = FalseAddition() self.assertRaises(Exception, self.gateway.entry_point.randomTernaryOperator, operator) # Time for periodic cleanup sleep(2) self.assertRaises(Exception, self.gateway.entry_point.randomTernaryOperator, operator)
def start_gateway_server_example_app_process(start_gateway_server=True): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if start_gateway_server: p = Process(target=start_instrumented_gateway_server) else: p = Process(target=start_instrumented_clientserver) p.start() sleep() check_connection() return p
def start_java_multi_client_server_app_process(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_java_multi_client_server_app) p.start() sleep() # test both gateways... check_connection(gateway_parameters=GatewayParameters(port=DEFAULT_PORT)) check_connection(gateway_parameters=GatewayParameters(port=DEFAULT_PORT + 2)) return p
def testPeriodicCleanup(self): operator = FalseAddition() self.assertRaises( Exception, self.gateway.entry_point.randomTernaryOperator, operator) # Time for periodic cleanup sleep(2) self.assertRaises( Exception, self.gateway.entry_point.randomTernaryOperator, operator)
def internal_work(): gateway2 = InstrJavaGateway( gateway_parameters=GatewayParameters(port=DEFAULT_PORT + 5), callback_server_parameters=CallbackServerParameters( port=DEFAULT_PYTHON_PROXY_PORT + 5)) sleep() play_with_ping(gateway2) python_gc() sleep() gateway2.shutdown()
def start_java_multi_client_server_app_process(): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED p = Process(target=start_java_multi_client_server_app) p.start() sleep() # test both gateways... check_connection( gateway_parameters=GatewayParameters(port=DEFAULT_PORT)) check_connection( gateway_parameters=GatewayParameters(port=DEFAULT_PORT + 2)) return p
def internal_work(): gateway2 = InstrJavaGateway( gateway_parameters=GatewayParameters( port=DEFAULT_PORT+5), callback_server_parameters=CallbackServerParameters( port=DEFAULT_PYTHON_PROXY_PORT+5)) sleep() play_with_ping(gateway2) python_gc() sleep() gateway2.shutdown()
def testGC(self): # This will only work with some JVM. sleep() example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl)) self.assertEqual(2, len(self.gateway.gateway_property.pool)) self.gateway.jvm.java.lang.System.gc() sleep(1) self.assertTrue(len(self.gateway.gateway_property.pool) < 2)
def testProxy(self): sleep() pythonAddress = self.gateway.java_gateway_server.getPythonAddress() port = self.gateway.get_callback_server().get_listening_port() self.gateway.java_gateway_server.resetCallbackClient( pythonAddress, port) example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual("This is Hello;\n10MyMy!\n;", example.callHello2(impl))
def start_clientserver_example_app_process(start_java_client=False, start_short_timeout=False): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if not start_java_client and not start_short_timeout: p = Process(target=start_clientserver_example_server) elif start_short_timeout: p = Process(target=start_short_timeout_clientserver_example_server) else: p = Process(target=start_java_clientserver_example_server) p.start() sleep() check_connection() return p
def start_example_app_process(app=None): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if not app: target = start_example_server elif app == "nomem": target = start_no_mem_example_server elif app == "pythonentrypoint": target = start_python_entry_point_server p = Process(target=target) p.start() sleep() check_connection() return p
def start_example_app_process(app=None, args=()): # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED if not app: target = start_example_server elif app == "nomem": target = start_no_mem_example_server elif app == "pythonentrypoint": target = start_python_entry_point_server p = Process(target=target, args=args) p.start() sleep() check_connection() return p
def testProxy(self): sleep() pythonAddress = self.gateway.java_gateway_server.getPythonAddress() port = self.gateway.get_callback_server().get_listening_port() self.gateway.java_gateway_server.resetCallbackClient( pythonAddress, port) example = self.gateway.entry_point.getNewExample() impl = IHelloImpl() self.assertEqual("This is Hello!", example.callHello(impl)) self.assertEqual( "This is Hello;\n10MyMy!\n;", example.callHello2(impl))
def testProxyError(self): sleep() example = self.gateway.entry_point.getNewExample() try: example.callHello(IHelloFailingImpl( Py4JJavaError( '', self.gateway.jvm.java.lang.IllegalStateException( 'My IllegalStateException')))) self.fail() except Py4JJavaError as e: self.assertTrue(is_instance_of( self.gateway, e.java_exception, 'py4j.Py4JException')) self.assertIn('My IllegalStateException', str(e))
def testJavaToPythonToJavaNoGCNoShutdown(self): def internal_work(gateway): hello_state = HelloState2(run_gc=False) gateway2 = InstrJavaGateway( gateway_parameters=GatewayParameters( port=DEFAULT_PORT+5), callback_server_parameters=CallbackServerParameters( port=DEFAULT_PYTHON_PROXY_PORT+5), python_server_entry_point=hello_state) hello_state.gateway = gateway2 sleep() gateway.entry_point.startServerWithPythonEntry(False) sleep() gateway2.shutdown() # Check that Java correctly called Python self.assertEqual(2, len(hello_state.calls)) self.assertEqual((None, None), hello_state.calls[0]) self.assertEqual((2, "Hello World"), hello_state.calls[1]) with gateway_server_example_app_process(): gateway = JavaGateway() # We disable gc to test whether a shut down on one side will # garbage collect everything. gc.disable() internal_work(gateway) gc.enable() python_gc() gateway.jvm.py4j.instrumented.MetricRegistry.forceFinalization() sleep() createdSet = gateway.jvm.py4j.instrumented.MetricRegistry.\ getCreatedObjectsKeySet() finalizedSet = gateway.jvm.py4j.instrumented.MetricRegistry.\ getFinalizedObjectsKeySet() # 6 objects: 2 InstrumentedObject (sayHello called twice), 1 # InstrGatewayServer, 1 CallbackClient, 1 CallbackConnection, 1 # GatewayConnection self.assertEqual(6, len(createdSet)) self.assertEqual(6, len(finalizedSet)) self.assertEqual(createdSet, finalizedSet) gateway.shutdown() # 7 objects: JavaGateway, GatewayClient, CallbackServer, # GatewayProperty, HelloState, GatewayConnection, # CallbackConnection assert_python_memory(self, 7)
def internal_work(clientserver): hello_state = HelloState2() clientserver2 = InstrClientServer( JavaParameters(port=DEFAULT_PORT + 5), PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5), python_server_entry_point=hello_state) hello_state.gateway = clientserver2 sleep() clientserver.entry_point.startServerWithPythonEntry(False) sleep() clientserver2.shutdown() # Check that Java correctly called Python self.assertEqual(2, len(hello_state.calls)) self.assertEqual((None, None), hello_state.calls[0]) self.assertEqual((2, "Hello World"), hello_state.calls[1])
def testJavaToPythonToJavaNoGCNoShutdown(self): def internal_work(clientserver): hello_state = HelloState2() clientserver2 = InstrClientServer( JavaParameters(port=DEFAULT_PORT + 5), PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 5), python_server_entry_point=hello_state) hello_state.gateway = clientserver2 sleep() clientserver.entry_point.startServerWithPythonEntry(False) sleep() clientserver2.shutdown() # Check that Java correctly called Python self.assertEqual(2, len(hello_state.calls)) self.assertEqual((None, None), hello_state.calls[0]) self.assertEqual((2, "Hello World"), hello_state.calls[1]) with gateway_server_example_app_process(False): clientserver = ClientServer() # We disable gc to test whether a shut down on one side will # garbage collect everything. gc.disable() internal_work(clientserver) gc.enable() python_gc() clientserver.jvm.py4j.instrumented.MetricRegistry.\ forceFinalization() sleep() createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\ getCreatedObjectsKeySet() finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\ getFinalizedObjectsKeySet() # 7 objects: 2 InstrumentedObject (sayHello called twice), 1 # JavaServer, 1 PythonClient, 1 ClientServer, 3 # ClientServerConnection (1 to call sayHello, 1 that calls GC from # Java to Python, 1 that receives shutdown command) self.assertEqual(8, len(createdSet)) self.assertEqual(8, len(finalizedSet)) self.assertEqual(createdSet, finalizedSet) clientserver.shutdown() # 8 objects: ClientServer (ok), PythonServer (ok), JavaClient, # GatewayProperty, HelloState (ok), 3 ClientServer Connections (2) assert_python_memory(self, 8)
def internal_work(clientserver): hello_state = HelloState2() clientserver2 = InstrClientServer( JavaParameters(port=DEFAULT_PORT+5), PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5), python_server_entry_point=hello_state) hello_state.gateway = clientserver2 sleep() clientserver.entry_point.startServerWithPythonEntry(False) sleep() clientserver2.shutdown() # Check that Java correctly called Python self.assertEqual(2, len(hello_state.calls)) self.assertEqual((None, None), hello_state.calls[0]) self.assertEqual((2, "Hello World"), hello_state.calls[1])