def _ensure_initialized(cls): SparkContext._ensure_initialized() gw = SparkContext._gateway java_import(gw.jvm, "org.apache.spark.streaming.*") java_import(gw.jvm, "org.apache.spark.streaming.api.java.*") java_import(gw.jvm, "org.apache.spark.streaming.api.python.*") # start callback server # getattr will fallback to JVM, so we cannot test by hasattr() if "_callback_server" not in gw.__dict__ or gw._callback_server is None: gw.callback_server_parameters.eager_load = True gw.callback_server_parameters.daemonize = True gw.callback_server_parameters.daemonize_connections = True gw.callback_server_parameters.port = 0 gw.start_callback_server(gw.callback_server_parameters) cbport = gw._callback_server.server_socket.getsockname()[1] gw._callback_server.port = cbport # gateway with real port gw._python_proxy_port = gw._callback_server.port # get the GatewayServer object in JVM by ID jgws = JavaObject("GATEWAY_SERVER", gw._gateway_client) # update the port of CallbackClient with real port jgws.resetCallbackClient(jgws.getCallbackClient().getAddress(), gw._python_proxy_port) # register serializer for TransformFunction # it happens before creating SparkContext when loading from checkpointing cls._transformerSerializer = TransformFunctionSerializer( SparkContext._active_spark_context, CloudPickleSerializer(), gw)
def testJavaErrorGC(self): id = self.doError() java_object = JavaObject(id, self.gateway._gateway_client) try: # Should fail because it should have been garbage collected... java_object.getCause() self.fail() except Py4JError: self.assertTrue(True)
def ensure_callback_server_started(gw): """ Start callback server if not already started. The callback server is needed if the Java driver process needs to callback into the Python driver process to execute Python code. """ # getattr will fallback to JVM, so we cannot test by hasattr() if "_callback_server" not in gw.__dict__ or gw._callback_server is None: gw.callback_server_parameters.eager_load = True gw.callback_server_parameters.daemonize = True gw.callback_server_parameters.daemonize_connections = True gw.callback_server_parameters.port = 0 gw.start_callback_server(gw.callback_server_parameters) cbport = gw._callback_server.server_socket.getsockname()[1] gw._callback_server.port = cbport # gateway with real port gw._python_proxy_port = gw._callback_server.port # get the GatewayServer object in JVM by ID jgws = JavaObject("GATEWAY_SERVER", gw._gateway_client) # update the port of CallbackClient with real port jgws.resetCallbackClient(jgws.getCallbackClient().getAddress(), gw._python_proxy_port)
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client) self._get = get_method(self, "get")
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client) self._next_name = "next"
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client) self.java_remove = get_method(self, "remove")
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client)
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client) self._add = get_method(self, "add") self._clear = get_method(self, "clear") self._remove = get_method(self, "remove")
def __init__(self, target_id, gateway_client): JavaObject.__init__(self, target_id, gateway_client) self._add = get_method(self, 'add') self._clear = get_method(self, 'clear') self._remove = get_method(self, 'remove')
def _get_python_id(java_vertex: JavaObject) -> Tuple[int, ...]: return tuple(java_vertex.getId().getValue())
def _to_ndarray(java_tensor: JavaObject) -> Any: if (java_tensor.getRank() == 0): return np.array(java_tensor.scalar()) else: return Tensor.__get_ndarray_from_tensor(java_tensor).reshape( java_tensor.getShape())