def _ensure_initialized(cls):
SparkContext._ensure_initialized()
gw = SparkContext._gateway
java_import(gw.jvm, "org.apache.spark.streaming.*")
java_import(gw.jvm, "org.apache.spark.streaming.api.java.*")
java_import(gw.jvm, "org.apache.spark.streaming.api.python.*")
# start callback server
# getattr will fallback to JVM, so we cannot test by hasattr()
if "_callback_server" not in gw.__dict__ or gw._callback_server is None:
gw.callback_server_parameters.eager_load = True
gw.callback_server_parameters.daemonize = True
gw.callback_server_parameters.daemonize_connections = True
gw.callback_server_parameters.port = 0
gw.start_callback_server(gw.callback_server_parameters)
cbport = gw._callback_server.server_socket.getsockname()[1]
gw._callback_server.port = cbport
# gateway with real port
gw._python_proxy_port = gw._callback_server.port
# get the GatewayServer object in JVM by ID
jgws = JavaObject("GATEWAY_SERVER", gw._gateway_client)
# update the port of CallbackClient with real port
jgws.resetCallbackClient(jgws.getCallbackClient().getAddress(), gw._python_proxy_port)
# register serializer for TransformFunction
# it happens before creating SparkContext when loading from checkpointing
cls._transformerSerializer = TransformFunctionSerializer(
SparkContext._active_spark_context, CloudPickleSerializer(), gw)
def testJavaErrorGC(self):
id = self.doError()
java_object = JavaObject(id, self.gateway._gateway_client)
try:
# Should fail because it should have been garbage collected...
java_object.getCause()
self.fail()
except Py4JError:
self.assertTrue(True)
def ensure_callback_server_started(gw):
"""
Start callback server if not already started. The callback server is needed if the Java
driver process needs to callback into the Python driver process to execute Python code.
"""
# getattr will fallback to JVM, so we cannot test by hasattr()
if "_callback_server" not in gw.__dict__ or gw._callback_server is None:
gw.callback_server_parameters.eager_load = True
gw.callback_server_parameters.daemonize = True
gw.callback_server_parameters.daemonize_connections = True
gw.callback_server_parameters.port = 0
gw.start_callback_server(gw.callback_server_parameters)
cbport = gw._callback_server.server_socket.getsockname()[1]
gw._callback_server.port = cbport
# gateway with real port
gw._python_proxy_port = gw._callback_server.port
# get the GatewayServer object in JVM by ID
jgws = JavaObject("GATEWAY_SERVER", gw._gateway_client)
# update the port of CallbackClient with real port
jgws.resetCallbackClient(jgws.getCallbackClient().getAddress(), gw._python_proxy_port)
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
self._get = get_method(self, "get")
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
self._next_name = "next"
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
self.java_remove = get_method(self, "remove")
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
self._add = get_method(self, "add")
self._clear = get_method(self, "clear")
self._remove = get_method(self, "remove")
def __init__(self, target_id, gateway_client):
JavaObject.__init__(self, target_id, gateway_client)
self._add = get_method(self, 'add')
self._clear = get_method(self, 'clear')
self._remove = get_method(self, 'remove')
def _get_python_id(java_vertex: JavaObject) -> Tuple[int, ...]:
return tuple(java_vertex.getId().getValue())
def _to_ndarray(java_tensor: JavaObject) -> Any:
if (java_tensor.getRank() == 0):
return np.array(java_tensor.scalar())
else:
return Tensor.__get_ndarray_from_tensor(java_tensor).reshape(
java_tensor.getShape())
