Ejemplo n.º 1
0
    def testGoodRetryFromJava(self):
        """Should retry from Java to Python.
        Similar use case as testGoodRetry, but from Java: Python calls Java,
        which calls Python back. Then Java waits for a while and calls Python
        again.

        Because Python Server has been waiting for too much time, the
        receiving socket has closed so the call from Java to Python will fail
        on send, and Java must retry by creating a new connection
        (ClientServerConnection).

        Because ClientServer reuses the same connection in each thread, we must
        launch a new thread on the Java side to correctly test the Python
        Server.
        """
        client_server = ClientServer(
            JavaParameters(), PythonParameters(read_timeout=0.250))
        with clientserver_example_app_process():
            try:
                operator = WaitOperator(0)
                opExample = client_server.jvm.py4j.examples.OperatorExample()
                opExample.launchOperator(operator, 500)
                sleep(0.1)
                str_connection = str(
                    list(client_server._callback_server.connections)[0])

                sleep(0.75)
                str_connection2 = str(
                    list(client_server._callback_server.connections)[0])
                self.assertNotEqual(str_connection, str_connection2)
            except Py4JJavaError:
                self.fail("Callbackserver did not retry.")
            finally:
                client_server.shutdown()
Ejemplo n.º 2
0
    def testJavaGC(self):
        # This will only work with some JVM.
        with clientserver_example_app_process():
            client_server = ClientServer(
                JavaParameters(), PythonParameters())
            example = client_server.entry_point.getNewExample()
            impl = IHelloImpl()
            self.assertEqual("This is Hello!", example.callHello(impl))
            self.assertEqual(
                "This is Hello;\n10MyMy!\n;",
                example.callHello2(impl))
            self.assertEqual(2, len(client_server.gateway_property.pool))

            # Make sure that finalizers do not block by calling the Java
            # finalizer again
            impl2 = IHelloImpl()
            self.assertEqual("This is Hello!", example.callHello(impl2))
            self.assertEqual(3, len(client_server.gateway_property.pool))

            # The two PythonProxies should be evicted on the Java side
            # Java should tell python to release the references.
            client_server.jvm.java.lang.System.gc()

            # Leave time for sotimeout
            sleep(3)
            self.assertTrue(len(client_server.gateway_property.pool) < 2)
            client_server.shutdown()
Ejemplo n.º 3
0
    def testErrorInPythonCallback(self):
        with clientserver_example_app_process():
            client_server = ClientServer(
                JavaParameters(), PythonParameters(
                    propagate_java_exceptions=True))
            example = client_server.entry_point.getNewExample()

            try:
                example.callHello(IHelloFailingImpl(
                    ValueError('My interesting Python exception')))
                self.fail()
            except Py4JJavaError as e:
                self.assertTrue(is_instance_of(
                    client_server, e.java_exception,
                    'py4j.Py4JException'))
                self.assertIn('interesting Python exception', str(e))

            try:
                example.callHello(IHelloFailingImpl(
                    Py4JJavaError(
                        '',
                        client_server.jvm.java.lang.IllegalStateException(
                            'My IllegalStateException'))))
                self.fail()
            except Py4JJavaError as e:
                self.assertTrue(is_instance_of(
                    client_server, e.java_exception,
                    'java.lang.IllegalStateException'))

            client_server.shutdown()
Ejemplo n.º 4
0
 def testBasicJVM(self):
     with clientserver_example_app_process():
         client_server = ClientServer(
             JavaParameters(), PythonParameters())
         ms = client_server.jvm.System.currentTimeMillis()
         self.assertTrue(ms > 0)
         client_server.shutdown()
Ejemplo n.º 5
0
    def testBadRetryFromJava(self):
        """Should not retry from Java to Python.
        Similar use case as testBadRetry, but from Java: Java calls a long
        Python operation.

        If there is a bug, Java will call Python, then read will fail, then it
        will call Python again.

        If there is no bug, Java will call Python, read will fail, then Java
        will raise an Exception that will be received as a Py4JError on the
        Python side.
        """

        client_server = ClientServer(
            JavaParameters(), PythonParameters())

        with clientserver_example_app_process(False, True):
            try:
                operator = WaitOperator(0.5)
                opExample = client_server.jvm.py4j.examples.OperatorExample()

                opExample.randomBinaryOperator(operator)
                self.fail(
                    "Should never retry once the first command went through."
                    " number of calls made: {0}".format(operator.callCount))
            except Py4JError:
                # XXX This occurs when WaitOperator tries to send a response to
                # the Java side (this is slightly different then the
                # GatewayServer equivalent where the Py4JError occurs on the
                # clientserver, but the Java side can still send back an
                # exception to the JavaGateway).
                self.assertTrue(True)
            finally:
                client_server.shutdown()
Ejemplo n.º 6
0
    def testRecursion(self):
        with clientserver_example_app_process():
            client_server = ClientServer(
                JavaParameters(), PythonParameters())
            pingpong = client_server.jvm.py4j.examples.PingPong()
            ping = PythonPing()
            self.assertEqual(2, pingpong.start(ping))

            pingpong = client_server.jvm.py4j.examples.PingPong(True)

            try:
                pingpong.start(ping)
                self.fail()
            except Py4JJavaError:
                # TODO Make sure error are recursively propagated
                # Also a problem with old threading model.
                self.assertTrue(True)

            ping = PythonPing(True)
            pingpong = client_server.jvm.py4j.examples.PingPong(False)

            try:
                pingpong.start(ping)
                self.fail()
            except Py4JJavaError:
                # TODO Make sure error are recursively propagated
                # Also a problem with old threading model.
                self.assertTrue(True)

            client_server.shutdown()
Ejemplo n.º 7
0
    def testJavaClientPythonServer(self):
        hello_state = HelloState()
        client_server = ClientServer(
            JavaParameters(), PythonParameters(), hello_state)

        with clientserver_example_app_process(True):
            client_server.shutdown()

        # Check that Java correctly called Python
        self.assertEqual(2, len(hello_state.calls))
        self.assertEqual((None, None), hello_state.calls[0])
        self.assertEqual((2, "Hello World"), hello_state.calls[1])
Ejemplo n.º 8
0
 def testErrorInPy4J(self):
     with clientserver_example_app_process():
         client_server = ClientServer(
             JavaParameters(), PythonParameters())
         try:
             client_server.jvm.java.lang.Math.abs(
                 3000000000000000000000000000000000000)
             self.fail("Should not be able to convert overflowing long")
         except Py4JError:
             self.assertTrue(True)
         # Check that the connection is not broken (refs #265)
         val = client_server.jvm.java.lang.Math.abs(-4)
         self.assertEqual(4, val)
         client_server.shutdown()
Ejemplo n.º 9
0
    def test_all_regular_signals(self):
        listener = MockListener(self)

        server_started.connect(listener.started)

        hello_state = HelloState()
        client_server = ClientServer(
            JavaParameters(), PythonParameters(), hello_state)
        server_stopped.connect(
            listener.stopped, sender=client_server.get_callback_server())
        server_connection_started.connect(
            listener.connection_started,
            sender=client_server.get_callback_server())
        server_connection_stopped.connect(
            listener.connection_stopped,
            sender=client_server.get_callback_server())
        pre_server_shutdown.connect(
            listener.pre_shutdown, sender=client_server.get_callback_server())
        post_server_shutdown.connect(
            listener.post_shutdown, sender=client_server.get_callback_server())
        with clientserver_example_app_process(True):
            client_server.shutdown()

        self.assertEqual(1, listener.received["started"])
        self.assertEqual(1, listener.received["stopped"])
        self.assertEqual(1, listener.received["pre_shutdown"])
        self.assertEqual(1, listener.received["post_shutdown"])
        self.assertEqual(1, listener.received["connection_started"])
        self.assertEqual(1, listener.received["connection_stopped"])
Ejemplo n.º 10
0
    def test_signals_started_from_python(self):
        listener = MockListener(self)

        with clientserver_example_app_process():
            server_started.connect(listener.started)

            client_server = ClientServer(
                JavaParameters(), PythonParameters())
            example = client_server.entry_point.getNewExample()
            impl = IHelloImpl()
            self.assertEqual("This is Hello!", example.callHello(impl))

            server_stopped.connect(
                listener.stopped, sender=client_server.get_callback_server())
            server_connection_started.connect(
                listener.connection_started,
                sender=client_server.get_callback_server())
            server_connection_stopped.connect(
                listener.connection_stopped,
                sender=client_server.get_callback_server())
            pre_server_shutdown.connect(
                listener.pre_shutdown,
                sender=client_server.get_callback_server())
            post_server_shutdown.connect(
                listener.post_shutdown,
                sender=client_server.get_callback_server())
            client_server.shutdown()

        self.assertEqual(1, listener.received["started"])
        self.assertEqual(1, listener.received["stopped"])
        self.assertEqual(1, listener.received["pre_shutdown"])
        self.assertEqual(1, listener.received["post_shutdown"])
        # Connection initiated from JavaClient, so no signal sent.
        self.assertEqual(0, listener.received["connection_started"])
        self.assertEqual(0, listener.received["connection_stopped"])
Ejemplo n.º 11
0
    def testSendObjects(self):
        """This test receives 1000 calls creating object cycles.
        Typically, the garbage collector will starts in the middle of
        Py4J connection code, which will create an error if the garbage
        collection is done on the same thread.
        """
        hello = HelloObjects()
        client_server = ClientServer(
            JavaParameters(), PythonParameters(), hello)

        with clientserver_example_app_process(
                start_gc_test=True, join=False) as p:
            p.join()
            client_server.shutdown()
            self.assertEquals(1000, hello.calls)
Ejemplo n.º 12
0
    def testJavaToPythonToJavaNoGCNoShutdown(self):
        def internal_work(clientserver):
            hello_state = HelloState2()
            clientserver2 = InstrClientServer(
                JavaParameters(port=DEFAULT_PORT+5),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5),
                python_server_entry_point=hello_state)
            hello_state.gateway = clientserver2
            sleep()

            clientserver.entry_point.startServerWithPythonEntry(False)
            sleep()
            clientserver2.shutdown()

            # Check that Java correctly called Python
            self.assertEqual(2, len(hello_state.calls))
            self.assertEqual((None, None), hello_state.calls[0])
            self.assertEqual((2, "Hello World"), hello_state.calls[1])

        with gateway_server_example_app_process(False):
            clientserver = ClientServer()
            # We disable gc to test whether a shut down on one side will
            # garbage collect everything.
            gc.disable()
            internal_work(clientserver)
            gc.enable()
            python_gc()
            clientserver.jvm.py4j.instrumented.MetricRegistry.\
                forceFinalization()
            sleep()
            createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getCreatedObjectsKeySet()
            finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getFinalizedObjectsKeySet()
            # 7 objects: 2 InstrumentedObject (sayHello called twice), 1
            # JavaServer, 1 PythonClient, 1 ClientServer, 3
            # ClientServerConnection (1 to call sayHello,
            # 1 that receives shutdown command)
            self.assertEqual(7, len(createdSet))
            self.assertEqual(7, len(finalizedSet))
            self.assertEqual(createdSet, finalizedSet)
            clientserver.shutdown()

            # 8 objects: ClientServer (ok), PythonServer (ok), JavaClient,
            # GatewayProperty, HelloState (ok), 3 ClientServer Connections (2)
            assert_python_memory(self, 7)
Ejemplo n.º 13
0
    def testGoodRetry(self):
        """Should retry from Python to Java.
        Python calls Java twice in a row, then waits, then calls again.

        Java fails when it does not receive calls quickly.

        If there is a bug, Python will fail on the third call because the Java
        connection was closed and it did not retry.

        If there is a bug, Python might not fail because Java did not close the
        connection on timeout. The connection used to call Java will be the
        same one for all calls (and an assertion will fail).

        If there is no bug, Python will call Java twice with the same
        connection. On the third call, the write will fail, and a new
        connection will be created.
        """
        client_server = ClientServer(
            JavaParameters(), PythonParameters())
        connections = client_server._gateway_client.deque

        with clientserver_example_app_process(False, True):
            try:
                # Call #1
                client_server.jvm.System.currentTimeMillis()
                str_connection = str(connections[0])

                # Call #2
                client_server.jvm.System.currentTimeMillis()
                self.assertEqual(1, len(connections))
                str_connection2 = str(connections[0])
                self.assertEqual(str_connection, str_connection2)

                sleep(0.5)
                client_server.jvm.System.currentTimeMillis()
                self.assertEqual(1, len(connections))
                str_connection3 = str(connections[0])
                # A new connection was automatically created.
                self.assertNotEqual(str_connection, str_connection3)

            except Py4JError:
                self.fail("Should retry automatically by default.")
            finally:
                client_server.shutdown()
Ejemplo n.º 14
0
    def testJavaToPythonToJavaCleanGC(self):
        def internal_work(clientserver):
            hello_state = HelloState2()
            clientserver2 = InstrClientServer(
                JavaParameters(port=DEFAULT_PORT+5),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5),
                python_server_entry_point=hello_state)
            hello_state.gateway = clientserver2
            sleep()

            clientserver.entry_point.startServerWithPythonEntry(True)
            sleep()
            clientserver2.shutdown()

            # Check that Java correctly called Python
            self.assertEqual(2, len(hello_state.calls))
            self.assertEqual((None, None), hello_state.calls[0])
            self.assertEqual((2, "Hello World"), hello_state.calls[1])

        with gateway_server_example_app_process(False):
            clientserver = ClientServer()
            internal_work(clientserver)
            python_gc()
            clientserver.jvm.py4j.instrumented.MetricRegistry.\
                forceFinalization()
            sleep()
            createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getCreatedObjectsKeySet()
            finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getFinalizedObjectsKeySet()
            # 7 objects: 2 InstrumentedObject (sayHello called twice), 1
            # JavaServer, 1 PythonClient, 1 ClientServer, 2
            # ClientServerConnection (1 to call sayHello, 1 that calls GC from
            # Java to Python)
            self.assertEqual(7, len(createdSet))
            self.assertEqual(7, len(finalizedSet))
            self.assertEqual(createdSet, finalizedSet)
            clientserver.shutdown()

            # 8 objects: ClientServer (ok), PythonServer (ok), JavaClient,
            # GatewayProperty, HelloState (ok), 3 ClientServer Connections (2)
            assert_python_memory(self, 8)
Ejemplo n.º 15
0
    def testStream(self):
        with clientserver_example_app_process():
            client_server = ClientServer(
                JavaParameters(), PythonParameters())
            e = client_server.entry_point.getNewExample()

            # not binary - just get the Java object
            v1 = e.getStream()
            self.assertTrue(
                is_instance_of(
                    client_server, v1,
                    "java.nio.channels.ReadableByteChannel"))

            # pull it as a binary stream
            with e.getStream.stream() as conn:
                self.assertTrue(isinstance(conn, GatewayConnectionGuard))
                expected =\
                    u"Lorem ipsum dolor sit amet, consectetur adipiscing elit."
                self.assertEqual(
                    expected, smart_decode(conn.read(len(expected))))
            client_server.shutdown()
Ejemplo n.º 16
0
    def testPythonToJava(self):
        def work_with_object(clientserver):
            obj = clientserver.jvm.py4j.\
                instrumented.InstrumentedObject("test")
            return str(obj)

        def internal_work():
            clientserver2 = InstrClientServer(
                JavaParameters(port=DEFAULT_PORT+5),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
            sleep()
            work_with_object(clientserver2)
            python_gc()
            sleep()
            clientserver2.shutdown()

        with gateway_server_example_app_process(False):
            clientserver = ClientServer()
            clientserver.entry_point.startServer2()
            internal_work()
            python_gc()
            clientserver.jvm.py4j.instrumented.MetricRegistry.\
                forceFinalization()
            sleep()
            createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getCreatedObjectsKeySet()
            finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getFinalizedObjectsKeySet()

            # 5 objects: ClientServer, ClientServerConnection, PythonClient,
            # JavaServer, InstrumentedObject
            self.assertEqual(5, len(createdSet))
            self.assertEqual(5, len(finalizedSet))
            self.assertEqual(createdSet, finalizedSet)
            clientserver.shutdown()

            # 5 objects: ClientServer, ClientServerConnection, PythonClient,
            # JavaServer, GatewayProperty
            assert_python_memory(self, 5)
Ejemplo n.º 17
0
    def testPythonGC(self):
        def internal_function(client_server):
            example = client_server.entry_point.getNewExample()
            in_middle = len(
                client_server.java_gateway_server.getGateway().getBindings())
            self.assertEqual(1, example.method1())

            # After this method is executed, Python no longer need
            # a reference to example and it should tell Java to release
            # the reference too
            return in_middle

        with clientserver_example_app_process():
            # This will only work with some JVM.
            client_server = ClientServer(
                JavaParameters(), PythonParameters())
            before = len(
                client_server.java_gateway_server.getGateway().getBindings())
            in_middle = internal_function(client_server)

            # Force Python to run garbage collection in case it did
            # not run yet.
            sleep()
            gc.collect()
            # Long sleep to ensure that the finalizer worker received
            # everything
            sleep(2)

            after = len(
                client_server.java_gateway_server.getGateway().getBindings())

            # Number of references on the Java side should be the same before
            # and after
            self.assertEqual(before, after)

            # Number of references when we created a JavaObject should be
            # higher than at the beginning.
            self.assertGreater(in_middle, before)
            client_server.shutdown()
Ejemplo n.º 18
0
    def testBadRetry(self):
        """Should not retry from Python to Java.
        Python calls a long Java method. The call goes through, but the
        response takes a long time to get back.

        If there is a bug, Python will fail on read and retry (sending the same
        call twice).

        If there is no bug, Python will fail on read and raise an Exception.
        """
        client_server = ClientServer(
            JavaParameters(read_timeout=0.250), PythonParameters())
        with clientserver_example_app_process():
            try:
                example = client_server.jvm.py4j.examples.ExampleClass()
                value = example.sleepFirstTimeOnly(500)
                self.fail(
                    "Should never retry once the first command went through."
                    "number of calls made: {0}".format(value))
            except Py4JError:
                self.assertTrue(True)
            finally:
                client_server.shutdown()
Ejemplo n.º 19
0
    def testPythonToJavaToPython(self):
        def play_with_ping(clientserver):
            ping = InstrumentedPythonPing()
            pingpong = clientserver.jvm.py4j.examples.PingPong()
            total = pingpong.start(ping)
            return total

        def internal_work():
            clientserver2 = InstrClientServer(
                JavaParameters(port=DEFAULT_PORT+5),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
            sleep()
            play_with_ping(clientserver2)
            python_gc()
            sleep()
            clientserver2.shutdown()

        with gateway_server_example_app_process(False):
            clientserver = ClientServer()
            clientserver.entry_point.startServer2()
            internal_work()
            python_gc()
            clientserver.jvm.py4j.instrumented.MetricRegistry.\
                forceFinalization()
            sleep()
            createdSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getCreatedObjectsKeySet()
            finalizedSet = clientserver.jvm.py4j.instrumented.MetricRegistry.\
                getFinalizedObjectsKeySet()

            # 4 objects: ClientServer, ClientServerConnection, JavaServer,
            # PythonClient
            self.assertEqual(4, len(createdSet))
            self.assertEqual(4, len(finalizedSet))
            self.assertEqual(createdSet, finalizedSet)
            clientserver.shutdown()
Ejemplo n.º 20
0
    def testPythonToJavaToPythonClose(self):
        def play_with_ping(clientserver):
            ping = InstrumentedPythonPing()
            pingpong = clientserver.jvm.py4j.examples.PingPong()
            total = pingpong.start(ping)
            return total

        def internal_work(assert_memory):
            clientserver2 = InstrClientServer(
                JavaParameters(port=DEFAULT_PORT+5),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT+5))
            sleep()
            play_with_ping(clientserver2)
            python_gc()
            sleep()
            clientserver2.close(
                close_callback_server_connections=True,
                keep_callback_server=True)
            sleep()
            assert_memory()
            clientserver2.shutdown()
            sleep()

        with gateway_server_example_app_process(False):
            clientserver = ClientServer()
            clientserver.entry_point.startServer2()

            def perform_memory_tests():
                python_gc()
                clientserver.jvm.py4j.instrumented.MetricRegistry.\
                    forceFinalization()
                sleep()

                createdSet = clientserver.jvm.py4j.instrumented.\
                    MetricRegistry.getCreatedObjectsKeySet()
                finalizedSet = clientserver.jvm.py4j.instrumented.\
                    MetricRegistry.getFinalizedObjectsKeySet()

                # 6 objects: ClientServer, JavaServer,
                # PythonClient, 3 ClientServerConnection.
                self.assertEqual(6, len(createdSet))

                # Should be 2: ClientServer, 1 ClientServerConnection
                # But for some reasons, Java refuses to collect the
                # clientserverconnection even though there are no strong
                # references.
                self.assertEqual(1, len(finalizedSet))

                # 8 objects: ClientServer, PythonServer, JavaClient,
                # GatewayProperty, PythonPing, 3 ClientServerConnection
                self.assertEqual(8, len(CREATED))

                # PythonPing + ClientServerConnection
                self.assertEqual(2, len(FINALIZED))

            internal_work(perform_memory_tests)
            python_gc()
            clientserver.jvm.py4j.instrumented.MetricRegistry.\
                forceFinalization()
            sleep()

            clientserver.shutdown()

            # 9 objects: ClientServer, PythonServer, JavaClient,
            # GatewayProperty, PythonPing, 4 ClientServerConnection
            assert_python_memory(self, 9)
Ejemplo n.º 21
0
        pl.setX(0, earthp[0])
        pl.setY(0, earthp[1])
        pl.setZ(0, earthp[2])
        pl.setX(1, moonp[0])
        pl.setY(1, moonp[1])
        pl.setZ(1, moonp[2])

        self.polyline.markForUpdate()

    def toString():
        return "line-update-runnable"

    class Java:
        implements = ["java.lang.Runnable"]

gateway = ClientServer(java_parameters=JavaParameters(auto_convert=True),
                      python_parameters=PythonParameters())
gs = gateway.entry_point

gs.cameraStop()

gs.stopSimulationTime()
gs.setVisibility("element.orbits", True)
gs.setCameraLock(True)
gs.setCameraOrientationLock(False)

gs.setFov(49)

gs.goToObject("Earth", 91.38e-2)

print("We will now add a line between the Earth and Moon")
Ejemplo n.º 22
0
 def testBasicJVM(self):
     with clientserver_example_app_process():
         client_server = ClientServer(JavaParameters(), PythonParameters())
         ms = client_server.jvm.System.currentTimeMillis()
         self.assertTrue(ms > 0)
         client_server.shutdown()
Ejemplo n.º 23
0
    def testMultiClientServerWithSharedJavaThread(self):
        with java_multi_client_server_app_process():
            client_server0 = ClientServer(JavaParameters(), PythonParameters())
            client_server1 = ClientServer(
                JavaParameters(port=DEFAULT_PORT + 2),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))

            entry0 = client_server0.entry_point
            entry1 = client_server1.entry_point

            # set up the ability for Java to get the Python thread ids
            threadIdGetter0 = PythonGetThreadId(client_server0)
            threadIdGetter1 = PythonGetThreadId(client_server1)
            entry0.setPythonThreadIdGetter(threadIdGetter0)
            entry1.setPythonThreadIdGetter(threadIdGetter1)
            thisThreadId = threading.current_thread().ident

            # ## Preconditions

            # Make sure we are talking to two different Entry points on
            # Java side
            self.assertEqual(0, entry0.getEntryId())
            self.assertEqual(1, entry1.getEntryId())

            # ## 1 Hop to Shared Java Thread

            # Check that the shared Java thread is the same thread
            # for both ClientServers
            sharedJavaThreadId = entry0.getSharedJavaThreadId()
            self.assertEqual(sharedJavaThreadId,
                             entry1.getSharedJavaThreadId())
            # And that it is distinct from either corresponding to
            # this Python thread
            self.assertNotEqual(sharedJavaThreadId, entry0.getJavaThreadId())
            self.assertNotEqual(sharedJavaThreadId, entry1.getJavaThreadId())

            # ## 2 Hops via Shared Java Thread to Python Thread

            # Check that the shared thread ends up as different threads
            # in the Python side. This part may not be obvious as the
            # top-level idea seems to be for Python thread to be pinned
            # to Java thread. Consider that this case is a simplification
            # of the real case. In the real case there are two
            # ClientServers running in same JVM, but in Python side each
            # ClientServer is in its own process. In that case it makes
            # it obvious that the shared Java thread should indeed
            # end up in different Python threads.
            sharedPythonThreadId0 = entry0.getSharedPythonThreadId()
            sharedPythonThreadId1 = entry1.getSharedPythonThreadId()
            # three way assert to make sure all three python
            # threads are distinct
            self.assertNotEqual(thisThreadId, sharedPythonThreadId0)
            self.assertNotEqual(thisThreadId, sharedPythonThreadId1)
            self.assertNotEqual(sharedPythonThreadId0, sharedPythonThreadId1)
            # Check that the Python thread id does not change between
            # invocations
            self.assertEquals(sharedPythonThreadId0,
                              entry0.getSharedPythonThreadId())
            self.assertEquals(sharedPythonThreadId1,
                              entry1.getSharedPythonThreadId())

            # ## 3 Hops to Shared Java Thread

            # Check that the thread above after 2 hops calls back
            # into the Java shared thread
            self.assertEqual(sharedJavaThreadId,
                             entry0.getSharedViaPythonJavaThreadId())
            self.assertEqual(sharedJavaThreadId,
                             entry1.getSharedViaPythonJavaThreadId())

            client_server0.shutdown()
            client_server1.shutdown()
Ejemplo n.º 24
0
def launch_gateway(conf=None, popen_kwargs=None):
    """
    launch jvm gateway
    :param conf: spark configuration passed to spark-submit
    :param popen_kwargs: Dictionary of kwargs to pass to Popen when spawning
        the py4j JVM. This is a developer feature intended for use in
        customizing how pyspark interacts with the py4j JVM (e.g., capturing
        stdout/stderr).
    :return:
    """
    if "PYSPARK_GATEWAY_PORT" in os.environ:
        gateway_port = int(os.environ["PYSPARK_GATEWAY_PORT"])
        gateway_secret = os.environ["PYSPARK_GATEWAY_SECRET"]
        # Process already exists
        proc = None
    else:
        SPARK_HOME = _find_spark_home()
        # Launch the Py4j gateway using Spark's run command so that we pick up the
        # proper classpath and settings from spark-env.sh
        on_windows = platform.system() == "Windows"
        script = "./bin/spark-submit.cmd" if on_windows else "./bin/spark-submit"
        command = [os.path.join(SPARK_HOME, script)]
        if conf:
            for k, v in conf.getAll():
                command += ['--conf', '%s=%s' % (k, v)]
        submit_args = os.environ.get("PYSPARK_SUBMIT_ARGS", "pyspark-shell")
        if os.environ.get("SPARK_TESTING"):
            submit_args = ' '.join(
                ["--conf spark.ui.enabled=false", submit_args])
        command = command + shlex.split(submit_args)

        # Create a temporary directory where the gateway server should write the connection
        # information.
        conn_info_dir = tempfile.mkdtemp()
        try:
            fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
            os.close(fd)
            os.unlink(conn_info_file)

            env = dict(os.environ)
            env["_PYSPARK_DRIVER_CONN_INFO_PATH"] = conn_info_file

            # Launch the Java gateway.
            popen_kwargs = {} if popen_kwargs is None else popen_kwargs
            # We open a pipe to stdin so that the Java gateway can die when the pipe is broken
            popen_kwargs['stdin'] = PIPE
            # We always set the necessary environment variables.
            popen_kwargs['env'] = env
            if not on_windows:
                # Don't send ctrl-c / SIGINT to the Java gateway:
                def preexec_func():
                    signal.signal(signal.SIGINT, signal.SIG_IGN)

                popen_kwargs['preexec_fn'] = preexec_func
                proc = Popen(command, **popen_kwargs)
            else:
                # preexec_fn not supported on Windows
                proc = Popen(command, **popen_kwargs)

            # Wait for the file to appear, or for the process to exit, whichever happens first.
            while not proc.poll() and not os.path.isfile(conn_info_file):
                time.sleep(0.1)

            if not os.path.isfile(conn_info_file):
                raise Exception(
                    "Java gateway process exited before sending its port number"
                )

            with open(conn_info_file, "rb") as info:
                gateway_port = read_int(info)
                gateway_secret = UTF8Deserializer().loads(info)
        finally:
            shutil.rmtree(conn_info_dir)

        # In Windows, ensure the Java child processes do not linger after Python has exited.
        # In UNIX-based systems, the child process can kill itself on broken pipe (i.e. when
        # the parent process' stdin sends an EOF). In Windows, however, this is not possible
        # because java.lang.Process reads directly from the parent process' stdin, contending
        # with any opportunity to read an EOF from the parent. Note that this is only best
        # effort and will not take effect if the python process is violently terminated.
        if on_windows:
            # In Windows, the child process here is "spark-submit.cmd", not the JVM itself
            # (because the UNIX "exec" command is not available). This means we cannot simply
            # call proc.kill(), which kills only the "spark-submit.cmd" process but not the
            # JVMs. Instead, we use "taskkill" with the tree-kill option "/t" to terminate all
            # child processes in the tree (http://technet.microsoft.com/en-us/library/bb491009.aspx)
            def killChild():
                Popen([
                    "cmd", "/c", "taskkill", "/f", "/t", "/pid",
                    str(proc.pid)
                ])

            atexit.register(killChild)

    # Connect to the gateway (or client server to pin the thread between JVM and Python)
    if os.environ.get("PYSPARK_PIN_THREAD", "false").lower() == "true":
        gateway = ClientServer(
            java_parameters=JavaParameters(port=gateway_port,
                                           auth_token=gateway_secret,
                                           auto_convert=True),
            python_parameters=PythonParameters(port=0, eager_load=False))
    else:
        gateway = JavaGateway(gateway_parameters=GatewayParameters(
            port=gateway_port, auth_token=gateway_secret, auto_convert=True))

    # Store a reference to the Popen object for use by the caller (e.g., in reading stdout/stderr)
    gateway.proc = proc

    # Import the classes used by PySpark
    java_import(gateway.jvm, "org.apache.spark.SparkConf")
    java_import(gateway.jvm, "org.apache.spark.api.conda.*")
    java_import(gateway.jvm, "org.apache.spark.api.java.*")
    java_import(gateway.jvm, "org.apache.spark.api.python.*")
    java_import(gateway.jvm, "org.apache.spark.ml.python.*")
    java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
    # TODO(davies): move into sql
    java_import(gateway.jvm, "org.apache.spark.sql.*")
    java_import(gateway.jvm, "org.apache.spark.sql.api.python.*")
    java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
    java_import(gateway.jvm, "scala.Tuple2")

    return gateway
Ejemplo n.º 25
0
class SimpleHello(object):

    def sayHello(self, int_value=None, string_value=None):
        print(int_value, string_value)
        return "Said hello to {0}".format(string_value)

    class Java:
        implements = ["py4j.examples._3_5_JavaCallPy.ClientServer.IHello"]

# Make sure that the python code is started first.
# Then execute: java -cp py4j.jar py4j.examples._3_5_JavaCallPy.ClientServer.SingleThreadClientApplication
from py4j.clientserver import ClientServer, JavaParameters, PythonParameters
simple_hello = SimpleHello()
gateway = ClientServer(
    java_parameters=JavaParameters(),
    python_parameters=PythonParameters(),
    python_server_entry_point=simple_hello)

# Output in cmd is as follows:
None None
2 Hello World
Ejemplo n.º 26
0
        return self._covert_list(clips)

    def _covert_list(self, py_list: list):
        """
        covert py list to java list contain list in list
        :param py_list: py list
        :return: java list
        """
        gateway = JavaGateway()
        if type(py_list[0]) is list:
            java_list = ListConverter().convert([], gateway._gateway_client)
            for l in py_list:
                j_l = ListConverter().convert(l, gateway._gateway_client)
                java_list.append(j_l)
        else:
            java_list = ListConverter().convert(py_list,
                                                gateway._gateway_client)
        return java_list

    def java_list_oppend(self, List):
        pass

    class Java:
        implements = ["com.alan.Client"]


if __name__ == '__main__':
    client_server = ClientServer(java_parameters=JavaParameters(),
                                 python_parameters=PythonParameters(),
                                 python_server_entry_point=JavaServer())
    print("Server started")
Ejemplo n.º 27
0
 def connect(
     cls,
     java_port: int = None,
     python_port: int = None,
     auto_load_java: bool = False,
     client_server: bool = True,
 ) -> bool:
     """
     connects to jnegmas
     """
     if not java_port:
         java_port = cls.DEFAULT_JAVA_PORT
     if not python_port:
         python_port = cls.DEFAULT_PYTHON_PORT
     if auto_load_java:
         if cls.gateway is None:
             cls.start_java_side(java_port=java_port)
         return True
     if cls.gateway is None:
         eager_load, auto_convert = True, True
         auto_field, auto_gc = True, False
         propagate_exceptions = False
         if client_server:
             cls.gateway = ClientServer(
                 java_parameters=JavaParameters(
                     port=java_port,
                     auto_convert=auto_convert,
                     eager_load=eager_load,
                     auto_close=True,
                     auto_gc=False,
                     auto_field=auto_field,
                     daemonize_memory_management=True,
                 ),
                 python_parameters=PythonParameters(
                     port=python_port,
                     propagate_java_exceptions=propagate_exceptions,
                     daemonize=True,
                     eager_load=eager_load,
                     auto_gc=auto_gc,
                     daemonize_connections=True,
                 ),
                 python_server_entry_point=PyEntryPoint(),
             )
         else:
             pyparams = CallbackServerParameters(
                 port=python_port,
                 daemonize_connections=True,
                 daemonize=True,
                 eager_load=eager_load,
                 propagate_java_exceptions=propagate_exceptions,
             )
             cls.gateway = JavaGateway(
                 gateway_parameters=GatewayParameters(
                     port=java_port,
                     auto_convert=auto_convert,
                     auto_field=auto_field,
                     eager_load=eager_load,
                     auto_close=True,
                 ),
                 callback_server_parameters=pyparams,
                 auto_convert=auto_convert,
                 start_callback_server=True,
                 eager_load=eager_load,
                 python_server_entry_point=PyEntryPoint(),
             )
         python_port = cls.gateway.get_callback_server().get_listening_port(
         )
         cls.gateway.java_gateway_server.resetCallbackClient(
             cls.gateway.java_gateway_server.getCallbackClient().getAddress(
             ),
             python_port,
         )
     return True
Ejemplo n.º 28
0
    def testMultiClientServerWithSharedJavaThread(self):
        with java_multi_client_server_app_process():
            client_server0 = ClientServer(
                JavaParameters(), PythonParameters())
            client_server1 = ClientServer(
                JavaParameters(port=DEFAULT_PORT + 2),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))

            entry0 = client_server0.entry_point
            entry1 = client_server1.entry_point

            # set up the ability for Java to get the Python thread ids
            threadIdGetter0 = PythonGetThreadId(client_server0)
            threadIdGetter1 = PythonGetThreadId(client_server1)
            entry0.setPythonThreadIdGetter(threadIdGetter0)
            entry1.setPythonThreadIdGetter(threadIdGetter1)
            thisThreadId = threading.current_thread().ident

            # ## Preconditions

            # Make sure we are talking to two different Entry points on
            # Java side
            self.assertEqual(0, entry0.getEntryId())
            self.assertEqual(1, entry1.getEntryId())

            # ## 1 Hop to Shared Java Thread

            # Check that the shared Java thread is the same thread
            # for both ClientServers
            sharedJavaThreadId = entry0.getSharedJavaThreadId()
            self.assertEqual(sharedJavaThreadId,
                             entry1.getSharedJavaThreadId())
            # And that it is distinct from either corresponding to
            # this Python thread
            self.assertNotEqual(sharedJavaThreadId, entry0.getJavaThreadId())
            self.assertNotEqual(sharedJavaThreadId, entry1.getJavaThreadId())

            # ## 2 Hops via Shared Java Thread to Python Thread

            # Check that the shared thread ends up as different threads
            # in the Python side. This part may not be obvious as the
            # top-level idea seems to be for Python thread to be pinned
            # to Java thread. Consider that this case is a simplification
            # of the real case. In the real case there are two
            # ClientServers running in same JVM, but in Python side each
            # ClientServer is in its own process. In that case it makes
            # it obvious that the shared Java thread should indeed
            # end up in different Python threads.
            sharedPythonThreadId0 = entry0.getSharedPythonThreadId()
            sharedPythonThreadId1 = entry1.getSharedPythonThreadId()
            # three way assert to make sure all three python
            # threads are distinct
            self.assertNotEqual(thisThreadId, sharedPythonThreadId0)
            self.assertNotEqual(thisThreadId, sharedPythonThreadId1)
            self.assertNotEqual(sharedPythonThreadId0, sharedPythonThreadId1)
            # Check that the Python thread id does not change between
            # invocations
            self.assertEquals(sharedPythonThreadId0,
                              entry0.getSharedPythonThreadId())
            self.assertEquals(sharedPythonThreadId1,
                              entry1.getSharedPythonThreadId())

            # ## 3 Hops to Shared Java Thread

            # Check that the thread above after 2 hops calls back
            # into the Java shared thread
            self.assertEqual(sharedJavaThreadId,
                             entry0.getSharedViaPythonJavaThreadId())
            self.assertEqual(sharedJavaThreadId,
                             entry1.getSharedViaPythonJavaThreadId())

            client_server0.shutdown()
            client_server1.shutdown()
Ejemplo n.º 29
0
    def testMultiClientServer(self):
        with java_multi_client_server_app_process():
            client_server0 = ClientServer(JavaParameters(), PythonParameters())
            client_server1 = ClientServer(
                JavaParameters(port=DEFAULT_PORT + 2),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))

            entry0 = client_server0.entry_point
            entry1 = client_server1.entry_point

            # set up the ability for Java to get the Python thread ids
            threadIdGetter0 = PythonGetThreadId(client_server0)
            threadIdGetter1 = PythonGetThreadId(client_server1)
            entry0.setPythonThreadIdGetter(threadIdGetter0)
            entry1.setPythonThreadIdGetter(threadIdGetter1)
            thisThreadId = threading.current_thread().ident

            # Make sure we are talking to two different Entry points on
            # Java side
            self.assertEqual(0, entry0.getEntryId())
            self.assertEqual(1, entry1.getEntryId())

            # ## 0 Hops to Thread ID

            # Check that the two thread getters get the same thread
            self.assertEquals(thisThreadId, int(threadIdGetter0.getThreadId()))
            self.assertEquals(thisThreadId, int(threadIdGetter1.getThreadId()))

            # ## 1 Hop to Thread ID

            # Check that ClientServers on Java side are on different threads
            javaThreadId0 = entry0.getJavaThreadId()
            javaThreadId1 = entry1.getJavaThreadId()
            self.assertNotEqual(javaThreadId0, javaThreadId1)
            # Check that ClientServers on Java side stay on same thread
            # on subsequent calls to them
            self.assertEqual(javaThreadId0, entry0.getJavaThreadId())
            self.assertEqual(javaThreadId1, entry1.getJavaThreadId())
            # Check alternate way of getting thread ids and that they match
            self.assertEqual(javaThreadId0,
                             int(threadIdGetter0.getJavaThreadId()))
            self.assertEqual(javaThreadId1,
                             int(threadIdGetter1.getJavaThreadId()))

            # ## 2 Hops to Thread ID

            # Check that round trips from Python to Java and Python
            # end up back on this thread, regardless of which
            # client server we use for the round trip
            self.assertEqual(thisThreadId, entry0.getPythonThreadId())
            self.assertEqual(thisThreadId, entry1.getPythonThreadId())

            # ## 3 Hops to Thread ID

            # Check that round trips from Python to Java to Python to Java
            # end up on the same thread as 1 hop
            self.assertEqual(javaThreadId0, entry0.getViaPythonJavaThreadId())
            self.assertEqual(javaThreadId1, entry1.getViaPythonJavaThreadId())

            client_server0.shutdown()
            client_server1.shutdown()
Ejemplo n.º 30
0
    def testMultiClientServer(self):
        with java_multi_client_server_app_process():
            client_server0 = ClientServer(
                JavaParameters(), PythonParameters())
            client_server1 = ClientServer(
                JavaParameters(port=DEFAULT_PORT + 2),
                PythonParameters(port=DEFAULT_PYTHON_PROXY_PORT + 2))

            entry0 = client_server0.entry_point
            entry1 = client_server1.entry_point

            # set up the ability for Java to get the Python thread ids
            threadIdGetter0 = PythonGetThreadId(client_server0)
            threadIdGetter1 = PythonGetThreadId(client_server1)
            entry0.setPythonThreadIdGetter(threadIdGetter0)
            entry1.setPythonThreadIdGetter(threadIdGetter1)
            thisThreadId = threading.current_thread().ident

            # Make sure we are talking to two different Entry points on
            # Java side
            self.assertEqual(0, entry0.getEntryId())
            self.assertEqual(1, entry1.getEntryId())

            # ## 0 Hops to Thread ID

            # Check that the two thread getters get the same thread
            self.assertEquals(thisThreadId,
                              int(threadIdGetter0.getThreadId()))
            self.assertEquals(thisThreadId,
                              int(threadIdGetter1.getThreadId()))

            # ## 1 Hop to Thread ID

            # Check that ClientServers on Java side are on different threads
            javaThreadId0 = entry0.getJavaThreadId()
            javaThreadId1 = entry1.getJavaThreadId()
            self.assertNotEqual(javaThreadId0, javaThreadId1)
            # Check that ClientServers on Java side stay on same thread
            # on subsequent calls to them
            self.assertEqual(javaThreadId0, entry0.getJavaThreadId())
            self.assertEqual(javaThreadId1, entry1.getJavaThreadId())
            # Check alternate way of getting thread ids and that they match
            self.assertEqual(javaThreadId0,
                             int(threadIdGetter0.getJavaThreadId()))
            self.assertEqual(javaThreadId1,
                             int(threadIdGetter1.getJavaThreadId()))

            # ## 2 Hops to Thread ID

            # Check that round trips from Python to Java and Python
            # end up back on this thread, regardless of which
            # client server we use for the round trip
            self.assertEqual(thisThreadId,
                             entry0.getPythonThreadId())
            self.assertEqual(thisThreadId,
                             entry1.getPythonThreadId())

            # ## 3 Hops to Thread ID

            # Check that round trips from Python to Java to Python to Java
            # end up on the same thread as 1 hop
            self.assertEqual(javaThreadId0,
                             entry0.getViaPythonJavaThreadId())
            self.assertEqual(javaThreadId1,
                             entry1.getViaPythonJavaThreadId())

            client_server0.shutdown()
            client_server1.shutdown()
            outputs = {"status": "success", "response": list(model_features)}

            return str(json.dumps(outputs))

        except Exception as e:
            raise e
            msg = "Failure in executing {0}".format(sys.argv[0])

            return str({"status": "error", "response": {"message": str(e)}})

        class Java:
            implements = ["com.apporchid.cloudseer.python.IPythonScript"]


from py4j.clientserver import ClientServer, JavaParameters, PythonParameters

# from AW_logger import awLogger
import sys
# logger = awLogger
import sys

pythonPort = int(sys.argv[1])
Demoops = DemoClassification()

#print(Demoops.executeScript({'input_dir':'C:/apporchid/prediction/'}))

gateway = ClientServer(
    java_parameters=JavaParameters(),
    python_parameters=PythonParameters(port = pythonPort),
    python_server_entry_point = Demoops)
Demoops.clientserver = gateway