def _start_gateway() -> JavaGateway:
# launch Java side with dynamic port and get back the port on which the
# server was bound to.
port = launch_gateway(port=0,
classpath=Context._find_jar_path('net-1.3.0-all.jar'),
javaopts=['-javaagent:' + Context._find_jar_path('quasar-core-0.7.9-jdk8.jar') + '=mb',
'-Xmx2048m'],
die_on_exit=True)
# connect python side to Java side with Java dynamic port and start python
# callback server with a dynamic port
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True),
callback_server_parameters=CallbackServerParameters(port=0))
# retrieve the port on which the python callback server was bound to.
python_port = gateway.get_callback_server().get_listening_port()
# tell the Java side to connect to the python callback server with the new
# python port. Note that we use the java_gateway_server attribute that
# retrieves the GatewayServer instance.
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
return gateway
def _start_gateway(port=0) -> JavaGateway:
# launch Java side with dynamic port and get back the port on which the
# server was bound to.
hearthstone_jar_path = Context.find_resource_path(
'hearthstone-0.8.67.jar')
net_jar_path = Context.find_resource_path('net-0.8.67-all.jar')
port = launch_gateway(port=port,
classpath=os.pathsep.join(
(hearthstone_jar_path, net_jar_path)),
die_on_exit=True)
# connect python side to Java side with Java dynamic port and start python
# callback server with a dynamic port
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True),
callback_server_parameters=CallbackServerParameters(port=0))
# retrieve the port on which the python callback server was bound to.
python_port = gateway.get_callback_server().get_listening_port()
# tell the Java side to connect to the python callback server with the new
# python port. Note that we use the java_gateway_server attribute that
# retrieves the GatewayServer instance.
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
return gateway
def __init__(self, classpath='', redirect_stdout=None):
"""Create and return a gateway to a Java object with SpatialKappa in
the classpath using py4j
The distribution of SpatialKappa should contain the
SpatialKappa jar file, so there should be no need to download
it separately.
Keyword arguments:
classpath -- classpath to override the default values
redirect_stdout -- Redirect stdout to this connection - see
py4j.java_gateway.launch_gateway()
"""
skjar_file = 'SpatialKappa-v2.1.5.jar'
antlrjar_file = 'ant-antlr-3.2.jar'
skjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', skjar_file)
antlrjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', antlrjar_file)
classpath = os.pathsep.join((classpath, skjar_path, antlrjar_path))
self.id = SpatialKappa.count
SpatialKappa.count = SpatialKappa.count + 1
self.redirect_stdout = redirect_stdout
self.port = jg.launch_gateway(classpath=classpath, die_on_exit=False, redirect_stdout=self.redirect_stdout)
self.gateway_client = jg.GatewayClient(port=self.port)
self.gateway = jg.JavaGateway(self.gateway_client, auto_convert=True)
def init_jvm(jar_path="./libs", max_heap=1024):
global jvm_gateway
if jvm_gateway is not None:
return
libraries = [
'{0}{1}klay-python-wrapper-0.3.jar'
]
jar_path = './libs'
max_heap = 1024
classpath = os.pathsep.join([lib.format(jar_path, os.sep) for lib in libraries])
py4j_path = '{0}{1}py4j0.10.8.1.jar'.format(jar_path, os.sep)
port = launch_gateway(jarpath=py4j_path,
classpath=classpath,
javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx{}m'.format(max_heap)],
die_on_exit=True)
try:
jvm_gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_convert=True))
except Exception as e:
jvm_gateway = None
logging.debug('fail')
logging.debug('success')
return jvm_gateway.jvm
def _start_gateway(port=0) -> JavaGateway:
# launch Java side with dynamic port and get back the port on which the
# server was bound to.
port = launch_gateway(
port=port,
classpath=Context.find_resource_path('net-0.8.22.jar'),
javaopts=[
"--add-modules", "java.se", "--add-exports",
"java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens",
"java.base/java.lang=ALL-UNNAMED", "--add-opens",
"java.base/java.nio=ALL-UNNAMED", "--add-opens",
"java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens",
"java.management/sun.management=ALL-UNNAMED", "--add-opens",
"jdk.management/com.sun.management.internal=ALL-UNNAMED"
],
die_on_exit=True)
# connect python side to Java side with Java dynamic port and start python
# callback server with a dynamic port
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True),
callback_server_parameters=CallbackServerParameters(port=0))
# retrieve the port on which the python callback server was bound to.
python_port = gateway.get_callback_server().get_listening_port()
# tell the Java side to connect to the python callback server with the new
# python port. Note that we use the java_gateway_server attribute that
# retrieves the GatewayServer instance.
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
return gateway
async def on_startup(app):
thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16)
app['thread_pool'] = thread_pool
port = launch_gateway(
jarpath='/spark-2.4.0-bin-hadoop2.7/jars/py4j-0.10.7.jar',
classpath='/spark-2.4.0-bin-hadoop2.7/jars/*:/hail.jar',
die_on_exit=True)
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port),
auto_convert=True)
app['gateway'] = gateway
hail_pkg = getattr(gateway.jvm, 'is').hail
app['hail_pkg'] = hail_pkg
jbackend = hail_pkg.backend.service.ServiceBackend.apply()
app['jbackend'] = jbackend
jhc = hail_pkg.HailContext.apply(jbackend, 'hail.log', False, False, 50,
False, 3)
app['jhc'] = jhc
app['users'] = set()
kube.config.load_incluster_config()
k8s_client = kube.client.CoreV1Api()
app['k8s_client'] = k8s_client
def __gateway(cls, classpath: str) -> JVMView:
from py4j.java_gateway import launch_gateway # ISSUE: ambient
port = launch_gateway(die_on_exit=True, classpath=classpath)
gw = JavaGateway(gateway_parameters=GatewayParameters(port=port))
jvm = gw.jvm
jvm = JavaGateway(gateway_parameters=GatewayParameters(port=port)).jvm
return jvm
def __init__(self, *args, **kwargs):
""" """
# path = os.path.realpath(__file__)
path = os.path.dirname(os.path.realpath(__file__))
port = launch_gateway(classpath='{path}/libs/*:{path}'.format(
**locals()))
super(JavaGateway,
self).__init__(gateway_parameters=GatewayParameters(port=port))
def newLocalStack(conf=DEFAULT_LOCAL_LAUNCH_CONF, classpath=get_Warp10_jar_path()):
"""Creates a new stack locally (not connected by a Warp 10 platform)
"""
port, auth_token = launch_gateway(enable_auth=True,die_on_exit=True,classpath=classpath)
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_convert=True, auth_token=auth_token))
entry_point = gateway.jvm.io.warp10.Py4JEntryPoint(conf)
return entry_point.newStack()
def _init_registration():
from py4j.java_gateway import (launch_gateway, JavaGateway, GatewayParameters)
global _java, _java_port
if _java is not None: return
_java_port = launch_gateway(
classpath=os.path.join(library_path(), 'nben', 'target', 'nben-standalone.jar'),
javaopts=['-Xmx2g'],
die_on_exit=True)
_java = JavaGateway(gateway_parameters=GatewayParameters(port=_java_port))
def launch_gateway(cls):
jars_dir = os.environ["PYPMML_JARS_DIR"] if "PYPMML_JARS_DIR" in os.environ else \
path.join(path.dirname(path.abspath(__file__)), 'jars')
launch_classpath = path.join(jars_dir, "*")
_port = launch_gateway(classpath=launch_classpath, die_on_exit=True)
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=_port,
auto_convert=True))
return gateway
def start_api_server():
port = launch_gateway(
classpath=
"/home/nithin/Desktop/Georgia Tech/Spring 2019/Adaptive Control and Reinforcement Learning/ACRL/Tetris/src",
die_on_exit=True)
global gateway
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port))
player_tetris = gateway.jvm.PlayerSkeletonCommunication().getFeatures()
return player_tetris
def __init__(self, classpath=''):
skjar_file = 'SpatialKappa-v2.1.1.jar'
antlrjar_file = 'ant-antlr-3.2.jar'
skjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', skjar_file)
antlrjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', antlrjar_file)
classpath = os.pathsep.join((classpath, skjar_path, antlrjar_path))
print classpath
self.port = jg.launch_gateway(classpath=classpath, die_on_exit=True)
self.gateway_client = jg.GatewayClient(port=self.port)
self.gateway = jg.JavaGateway(self.gateway_client)
def _init_registration():
from py4j.java_gateway import (launch_gateway, JavaGateway, GatewayParameters)
global _java, _java_port
if _java is not None: return
_java_port = launch_gateway(
classpath=os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
'lib', 'nben', 'target', 'nben-standalone.jar'),
javaopts=['-Xmx2g'],
die_on_exit=True)
_java = JavaGateway(gateway_parameters=GatewayParameters(port=_java_port))
def __init__(self, tmpdir, log, quiet, append, branching_factor,
skip_logging_configuration, optimizer_iterations):
super(LocalBackend, self).__init__()
spark_home = find_spark_home()
hail_jar_path = os.environ.get('HAIL_JAR')
if hail_jar_path is None:
if pkg_resources.resource_exists(__name__, "hail-all-spark.jar"):
hail_jar_path = pkg_resources.resource_filename(
__name__, "hail-all-spark.jar")
else:
raise RuntimeError(
'local backend requires a packaged jar or HAIL_JAR to be set'
)
port = launch_gateway(redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr,
jarpath=f'{spark_home}/jars/py4j-0.10.9.jar',
classpath=f'{spark_home}/jars/*:{hail_jar_path}',
die_on_exit=True)
self._gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True))
self._jvm = self._gateway.jvm
hail_package = getattr(self._jvm, 'is').hail
self._hail_package = hail_package
self._utils_package_object = scala_package_object(hail_package.utils)
self._jbackend = hail_package.backend.local.LocalBackend.apply(tmpdir)
self._jhc = hail_package.HailContext.apply(self._jbackend, log, True,
append, branching_factor,
skip_logging_configuration,
optimizer_iterations)
# This has to go after creating the SparkSession. Unclear why.
# Maybe it does its own patch?
install_exception_handler()
from hail.context import version
py_version = version()
jar_version = self._jhc.version()
if jar_version != py_version:
raise RuntimeError(
f"Hail version mismatch between JAR and Python library\n"
f" JAR: {jar_version}\n"
f" Python: {py_version}")
self._fs = LocalFS()
self._logger = None
if not quiet:
connect_logger(self._utils_package_object, 'localhost', 12888)
def __launch_java_gateway(cls) -> JavaGateway:
"""Launch java gateway."""
LOGGER.info('Initializing Farasa..')
port = launch_gateway(classpath=CLASS_PATH, die_on_exit=True)
params = GatewayParameters(port=port,
auto_convert=True,
auto_field=True,
eager_load=True)
return JavaGateway(gateway_parameters=params)
def __init__(self):
if not os.path.isfile(JAR_PATH):
raise ImportError("Could not find the JAR file associated with "
"javac_parser")
java_port = launch_gateway(jarpath=JAR_PATH,
die_on_exit=True,
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr)
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=java_port)
)
self.app = self.gateway.jvm.ca.ualberta.cs.App()
assert self.app.getNumParseErrors("") == 0
async def start(self, ctx: Context):
for (resource_name, context_attr, launch_jvm, gateway_params, callback_server_params,
classpath, javaopts) in self.gateways:
if launch_jvm:
gateway_params.port = launch_gateway(classpath=classpath, javaopts=javaopts)
gateway = JavaGateway(gateway_parameters=gateway_params,
callback_server_parameters=callback_server_params)
ctx.finished.connect(partial(self.shutdown_gateway, gateway=gateway,
resource_name=resource_name, shutdown_jvm=launch_jvm))
ctx.publish_resource(gateway, resource_name, context_attr)
logger.info('Configured Py4J gateway (%s / ctx.%s; address=%s, port=%d)',
resource_name, context_attr, gateway_params.address, gateway_params.port)
def init_jvm():
global gateway
# global javaChild
"""Initializes the Java virtual machine (JVM).
"""
if gateway != None:
return
folder_suffix = [
u'{0}', u'{0}{1}bin',
u'{0}{1}jhannanum-0.8.4.jar',
u'{0}{1}kkma-2.0.jar',
u'{0}{1}komoran-2.4-e.jar',
u'{0}{1}shineware-common-2.0.jar', u'{0}{1}shineware-ds-1.0.jar',
u'{0}{1}snakeyaml-1.12.jar', u'{0}{1}scala-library-2.11.4.jar', u'{0}{1}twitter-korean-text-2.4.3.jar', u'{0}{1}twitter-text-1.10.1.jar',
u'{0}{1}*']
javadir = u'%s%sjava' % (utils.installpath, os.sep)
args = [javadir, os.sep]
classpath = os.pathsep.join(f.format(*args) for f in folder_suffix)
py4jpath = u'{0}{1}py4j-0.9.2.jar'.format(*args)
port = launch_gateway(jarpath=py4jpath, classpath=classpath, javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx768m'])
# javaChild = Popen(['java',
# '-cp', classpath,
# '-Dfile.encoding=UTF8',
# '-ea', '-Xmx768m',
# 'kr.lucypark.py4j.KonlpyGateway']);
print "Initializing JAVA frameworks..."
time.sleep(1)
jvm_e = None
for i in range(0, 10):
try:
time.sleep(1)
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port))
except Exception as e:
gateway = None
jvm_e = e;
continue
break
if gateway == None:
if jvm_e != None:
raise jvm_e
raise JvmError("Could not connect to JVM. unknown error");
print "JAVA frameworks initialized."
atexit.register(shutdown_jvm);
def init_jvm():
global gateway
# global javaChild
"""Initializes the Java virtual machine (JVM).
"""
if gateway != None:
return
folder_suffix = [
u'{0}', u'{0}{1}bin', u'{0}{1}jhannanum-0.8.4.jar',
u'{0}{1}kkma-2.0.jar', u'{0}{1}komoran-2.4-e.jar',
u'{0}{1}shineware-common-2.0.jar', u'{0}{1}shineware-ds-1.0.jar',
u'{0}{1}snakeyaml-1.12.jar', u'{0}{1}scala-library-2.11.4.jar',
u'{0}{1}twitter-korean-text-2.4.3.jar',
u'{0}{1}twitter-text-1.10.1.jar', u'{0}{1}*'
]
javadir = u'%s%sjava' % (utils.installpath, os.sep)
args = [javadir, os.sep]
classpath = os.pathsep.join(f.format(*args) for f in folder_suffix)
py4jpath = u'{0}{1}py4j-0.9.2.jar'.format(*args)
port = launch_gateway(jarpath=py4jpath,
classpath=classpath,
javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx768m'])
# javaChild = Popen(['java',
# '-cp', classpath,
# '-Dfile.encoding=UTF8',
# '-ea', '-Xmx768m',
# 'kr.lucypark.py4j.KonlpyGateway']);
print "Initializing JAVA frameworks..."
time.sleep(1)
jvm_e = None
for i in range(0, 10):
try:
time.sleep(1)
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=port))
except Exception as e:
gateway = None
jvm_e = e
continue
break
if gateway == None:
if jvm_e != None:
raise jvm_e
raise JvmError("Could not connect to JVM. unknown error")
print "JAVA frameworks initialized."
atexit.register(shutdown_jvm)
def start_jvm(option, classpath):
import os
global GATEWAY
jarpath = None
for path in classpath:
if 'py4j' in path:
jarpath = path
break
port = launch_gateway(jarpath=jarpath, classpath=os.pathsep.join(classpath), javaopts=option)
GATEWAY = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_close=True),
callback_server_parameters=CallbackServerParameters())
return is_jvm_running()
def init_jvm(max_heap=1024, jar_path="./libs"):
"""KOMORAN Jar를 포함한 JVM을 초기화한 후, 반환합니다.
Args:
max_heap (int): JVM 실행 시 Max Heap Size (기본값: ``1024``, 단위: ``MB``)
jar_path (str): JVM 실행 시 포함할 Jar Path 지정 (기본값: ``"./libs"``)
Returns:
py4j.java_gateway.JavaGateway: 초기화된 JVM 객체
"""
base_path = os.path.dirname(os.path.realpath(__file__))
jar_path = os.path.abspath(os.path.join(base_path, jar_path))
assert os.path.exists(jar_path)
global jvm_gateway
if jvm_gateway is not None:
return
libraries = [
'{0}{1}KOMORAN-4e7b5ef.jar',
'{0}{1}KOMORANEntryPoint-0.1.0.jar',
# '{0}{1}*'
]
classpath = os.pathsep.join(
[lib.format(jar_path, os.sep) for lib in libraries])
py4j_path = "{0}{1}py4j0.10.8.1.jar".format(jar_path, os.sep)
port = launch_gateway(
jarpath=py4j_path,
classpath=classpath,
javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx{}m'.format(max_heap)],
die_on_exit=True)
logging.debug("initializing JVM... ")
try:
jvm_gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True))
# # for debugging with Java-side
# jvm_gateway = JavaGateway(gateway_parameters=GatewayParameters(port=25335, auto_convert=True))
except Exception as e:
jvm_gateway = None
logging.debug("fail")
logging.debug("success")
return jvm_gateway.jvm
def __init__(self, tmpdir, log, quiet, append, branching_factor,
skip_logging_configuration, optimizer_iterations):
SPARK_HOME = os.environ['SPARK_HOME']
HAIL_HOME = os.environ['HAIL_HOME']
port = launch_gateway(
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr,
jarpath=f'{SPARK_HOME}/jars/py4j-0.10.7.jar',
classpath=
f'{SPARK_HOME}/jars/*:{HAIL_HOME}/hail/build/libs/hail-all-spark.jar',
die_on_exit=True)
self._gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port, auto_convert=True))
self._jvm = self._gateway.jvm
hail_package = getattr(self._jvm, 'is').hail
self._hail_package = hail_package
self._utils_package_object = scala_package_object(hail_package.utils)
self._jbackend = hail_package.backend.local.LocalBackend.apply(tmpdir)
self._jhc = hail_package.HailContext.apply(self._jbackend, log, True,
append, branching_factor,
skip_logging_configuration,
optimizer_iterations)
# This has to go after creating the SparkSession. Unclear why.
# Maybe it does its own patch?
install_exception_handler()
from hail.context import version
py_version = version()
jar_version = self._jhc.version()
if jar_version != py_version:
raise RuntimeError(
f"Hail version mismatch between JAR and Python library\n"
f" JAR: {jar_version}\n"
f" Python: {py_version}")
self._fs = LocalFS()
self._logger = None
if not quiet:
connect_logger(self._utils_package_object, 'localhost', 12888)
def start_jvm(option, classpath, port: int = None):
import os
global GATEWAY
jarpath = None
for path in classpath:
if 'py4j' in path:
jarpath = path
break
gateway_port = launch_gateway(jarpath=jarpath, classpath=os.pathsep.join(classpath), javaopts=option)
logging.info("Java gateway started with port number %s", gateway_port)
if port is None:
port = DEFAULT_PYTHON_PROXY_PORT
logging.info("Callback server will use port number %s", port)
GATEWAY = JavaGateway(gateway_parameters=GatewayParameters(port=gateway_port, auto_close=True),
callback_server_parameters=CallbackServerParameters(port=port))
return is_jvm_running()
def get_gateway(extra_jar=None):
if extra_jar is None:
extra_jar = find_jar_path()
port = launch_gateway(
die_on_exit=True, classpath=extra_jar,
# redirect_stderr=sys.stderr, redirect_stdout=sys.stdout
)
gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port),
callback_server_parameters=CallbackServerParameters(port=0))
# retrieve the port on which the python callback server was bound to.
python_port = gateway.get_callback_server().get_listening_port()
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
return gateway
def start():
global pythonManager, gateway
java_home = os.environ.get("JAVA_HOME")
if java_home is None:
raise SystemError("The environment variable JAVA_HOME is not set")
java_path = os.path.join(java_home, "bin", "java")
gmql_jar_fn = resource_filename("gmql",
os.path.join("resources", __gmql_jar))
gmql_jar_fn = __check_backend(gmql_jar_fn)
libs_folder = resource_filename("gmql", os.path.join("resources", "lib"))
if sys.platform.startswith("win32"):
classpath_separator = ";"
else:
classpath_separator = ":"
classpath = gmql_jar_fn + classpath_separator + os.path.join(
libs_folder, "*")
_port = launch_gateway(classpath=classpath,
die_on_exit=True,
java_path=java_path,
javaopts=['-Xmx4096m'])
# gateway = JavaGateway.launch_gateway(classpath=gmql_jar_fn, die_on_exit=True,
# java_path=java_path)
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=_port, auto_convert=True))
python_api_package = __get_python_api_package(gateway)
pythonManager = __start_gmql_manager(python_api_package)
if sys.platform.startswith("win32"):
# if we are on windows set the hadoop home to winutils.exe
hadoopFolder_fn = resource_filename(
"gmql", os.path.join("resources", "hadoop"))
pythonManager.setHadoopHomeDir(hadoopFolder_fn)
# setting the spark tmp folder
pythonManager.setSparkLocalDir(folders['spark'])
def create_jvm(logger):
cp = {}
port = None
cp.update({'lib': ['*.jar'], 'jars': ['*.jar']})
if cp:
final_cp = []
for path, exprs in cp.items():
for expr in exprs:
final_cp.extend(glob.glob(os.path.join(path, expr)))
if not final_cp:
logger.warn('Hadoop JARs not found. Data source upload will not '
'work. Set HADOOP_HOME and/or SPARK_HOME environment '
'variables to the correct path.')
else:
final_cp = [os.path.abspath(p) for p in final_cp]
port = launch_gateway(classpath=":".join(final_cp),
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr,
die_on_exit=True)
print '>>>>>>>>>>>>>>>>>', port
return port
def start_java(py4j_path=None, accumulo_path=None, graphulo_path=None, dieonexit=None):
"""
Starts JVM along with a Py4J JavaGateway instance, with classpath exposing
Accumulo and Graphulo to the JVM.
Inputs:
py4j_path = full path for py4j jar
accumulo_path = full path of directory containing Accumulo jars
graphulo_path = full path for graphulo jar
dieonexit = option to have JVM close upon exiting of python
Note:
- By default, uses packaged JARs
"""
if py4j_path is None:
py4j_path = resource_filename(__name__, '/jars/py4j/share/py4j/py4j.0.1.jar')
if accumulo_path is None:
accumulo_path = resource_filename(__name__, '/jars/libext/*')
if graphulo_path is None:
graphulo_path = resource_filename(__name__, '/jars/lib/graphulo-3.0.0.jar')
cp = '.:'+accumulo_path+':'+graphulo_path
if dieonexit is None:
dieonexit = True
port = launch_gateway(jarpath=py4j_path, classpath=cp, die_on_exit=dieonexit)
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port),
callback_server_parameters=CallbackServerParameters(port=0))
print('JavaGateway started in Port '+str(port))
JavaConnector.jvm_port = port
JavaConnector.jvm_gateway = gateway
return gateway
def get_instance(self):
if not (self.instance is None):
return self.instance
if self.launch:
self.port, token = launch_gateway(enable_auth=True,
die_on_exit=True,
classpath=get_Warp10_jar_path())
if self.verbose:
print('Local gateway launched on port ' + str(self.port))
instance = JavaGateway(gateway_parameters=GatewayParameters(
port=self.port, auto_convert=True, auth_token=token))
else:
instance = JavaGateway(gateway_parameters=GatewayParameters(
self.addr,
self.port,
auto_convert=True,
auth_token=self.auth_token))
if self.verbose:
print('Establish connection with a gateway at ' + self.addr +
":" + str(self.port))
return instance
def get_gateway(extra_jar=None):
if extra_jar is None:
extra_jar = find_jar_path()
port = launch_gateway(
die_on_exit=True,
classpath=extra_jar,
# redirect_stderr=sys.stderr, redirect_stdout=sys.stdout
)
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port),
callback_server_parameters=CallbackServerParameters(port=0))
# retrieve the port on which the python callback server was bound to.
python_port = gateway.get_callback_server().get_listening_port()
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
return gateway
def create_jvm(logger):
port = None
dist_dirs = ['.', os.getenv('HADOOP_HOME'), os.getenv('SPARK_HOME')]
cp = {'lib': ['*.jar'], 'jars': ['*.jar']}
final_cp = []
for dist_dir in dist_dirs:
if dist_dir:
for path, exprs in list(cp.items()):
for expr in exprs:
# print(os.path.join(dist_dir, path, expr))
final_cp.extend(
glob.glob(os.path.join(dist_dir, path, expr)))
if not final_cp:
logger.warn('Hadoop JARs not found. Data source upload will not '
'work. Set HADOOP_HOME and/or SPARK_HOME environment '
'variables to the correct path.')
else:
final_cp = [os.path.abspath(p) for p in final_cp]
port = launch_gateway(classpath=":".join(final_cp),
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr,
die_on_exit=True)
return port
def launch_gateway(self):
self.update_log_files()
java_sock = socket()
java_sock.bind(('', 0))
java_port = java_sock.getsockname()[1]
python_sock = socket()
python_sock.bind(('', 0))
python_port = python_sock.getsockname()[1]
java_sock.close()
python_sock.close()
port = launch_gateway(classpath=self.classpath,
redirect_stderr=self.err,
redirect_stdout=self.out,
die_on_exit=True,
jarpath=self.jarpath,
port=java_port)
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port),
callback_server_parameters=CallbackServerParameters(
port=python_port))
python_port = self.gateway.get_callback_server().get_listening_port()
self.gateway.java_gateway_server.resetCallbackClient(
self.gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
process_name = self.gateway.jvm.java.lang.management.ManagementFactory.getRuntimeMXBean(
).getName()
self.java_pid = int(process_name.split('@')[0])
def launch_gateway(cls, javaopts=[], java_path=None):
"""Launch a `Gateway` in a new Java process.
:param javaopts: an array of extra options to pass to Java (the classpath
should be specified using the `classpath` parameter, not `javaopts`.)
:param java_path: If None, Py4J will use $JAVA_HOME/bin/java if $JAVA_HOME
is defined, otherwise it will use "java".
:return: An object of `Gateway`
"""
jars_dir = os.environ["PYPMML_JARS_DIR"] if "PYPMML_JARS_DIR" in os.environ else \
path.join(path.dirname(path.abspath(__file__)), 'jars')
launch_classpath = path.join(jars_dir, "*")
if not javaopts:
java_opts = os.environ.get("JAVA_OPTS")
if java_opts:
javaopts = java_opts.split()
_port = launch_gateway(classpath=launch_classpath,
javaopts=javaopts,
java_path=java_path,
die_on_exit=True)
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=_port, auto_convert=True))
return gateway
def run(self, parent_data_objs):
# Run the java py4j entry point
comp_dir = self._dag_node.comp_root_path()
self._logger.info("comp_dir: {}".format(comp_dir))
jar_files = glob.glob(os.path.join(comp_dir, "*.jar"))
self._logger.info("Java classpath files: {}".format(jar_files))
component_class = self._dag_node.comp_class()
java_jars = [self._mlcomp_jar] + jar_files
class_path = ":".join(java_jars)
java_gateway = None
all_ok = False
monitor_proc = None
try:
total_phys_mem_size_mb = ByteConv.from_bytes(
psutil.virtual_memory().total).mbytes
jvm_heap_size_option = "-Xmx{}m".format(
int(math.ceil(total_phys_mem_size_mb)))
java_opts = [jvm_heap_size_option]
self._logger.info("JVM options: {}".format(java_opts))
# Note: the jarpath is set to be the path to the mlcomp jar since the launch_gateway code is checking
# for the existence of the jar. The py4j jar is packed inside the mlcomp jar.
java_port = launch_gateway(port=0,
javaopts=java_opts,
die_on_exit=True,
jarpath=self._mlcomp_jar,
classpath=class_path,
redirect_stdout=sys.stdout,
redirect_stderr=sys.stderr)
java_gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=java_port),
callback_server_parameters=CallbackServerParameters(port=0),
python_server_entry_point=MLOpsPY4JWrapper())
python_port = java_gateway.get_callback_server(
).get_listening_port()
self._logger.debug("Python port: {}".format(python_port))
java_gateway.java_gateway_server.resetCallbackClient(
java_gateway.java_gateway_server.getCallbackClient().
getAddress(), python_port)
mlops_wrapper = MLOpsPY4JWrapper()
entry_point = java_gateway.jvm.com.parallelm.mlcomp.ComponentEntryPoint(
component_class)
component_via_py4j = entry_point.getComponent()
component_via_py4j.setMLOps(mlops_wrapper)
# Configure
m = java_gateway.jvm.java.util.HashMap()
for key in self._params.keys():
# py4j does not handle nested structures. So the configs which is a dict will not be passed to the java
# layer now.
if isinstance(self._params[key], dict):
continue
m[key] = self._params[key]
component_via_py4j.configure(m)
# Materialized
l = java_gateway.jvm.java.util.ArrayList()
for obj in parent_data_objs:
l.append(obj)
self._logger.info("Parent obj: {} type {}".format(
obj, type(obj)))
self._logger.info("Parent objs: {}".format(l))
if mlops_loaded:
monitor_proc = ProcessMonitor(mlops, self._ml_engine)
monitor_proc.start()
py4j_out_objs = component_via_py4j.materialize(l)
self._logger.debug(type(py4j_out_objs))
self._logger.debug(len(py4j_out_objs))
python_out_objs = []
for obj in py4j_out_objs:
self._logger.debug("Obj:")
self._logger.debug(obj)
python_out_objs.append(obj)
self._logger.info(
"Done running of materialize and getting output objects")
all_ok = True
except Py4JJavaError as e:
self._logger.error("Error in java code: {}".format(e))
raise MLCompException(str(e))
except Exception as e:
self._logger.error("General error: {}".format(e))
raise MLCompException(str(e))
finally:
self._logger.info("In finally block: all_ok {}".format(all_ok))
if java_gateway:
java_gateway.close_callback_server()
java_gateway.shutdown()
if mlops_loaded and monitor_proc:
monitor_proc.stop_gracefully()
return python_out_objs
def gateway(self):
if self._gateway is None:
self._gateway = launch_gateway()
return self._gateway
from django.conf import settings
from py4j.java_gateway import launch_gateway, JavaGateway,\
GatewayParameters, CallbackServerParameters, is_instance_of
logging.basicConfig(level=logging.INFO, format="%(message)s")
# Launch a gateway to the all Java JAR dependencies that we have
classpath = ':'.join([
os.path.join(settings.DEPS_DIR, jar)
for jar in os.listdir('deps')
])
port_number = launch_gateway(
classpath=classpath,
die_on_exit=True,
)
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port_number),
callback_server_parameters=CallbackServerParameters(port=0),
)
python_port = gateway.get_callback_server().get_listening_port()
gateway.java_gateway_server.resetCallbackClient(
gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port,
)
def java_isinstance(java_object, java_class):
'''
Convenience method to abstract away the gateway server to the
def parse_method(text: str) -> Tuple[WarcRecord]:
parsed_raw_warc = parse_raw_warc(text)
# crasher = 5 / 0 # ToDo: Uncomment
# print(crasher) # ToDo: Uncomment
return parsed_raw_warc
if __name__ == "__main__":
session: SparkSession = SparkSession.builder \
.master('local[3, 3]') \
.appName('Failure Exploration') \
.getOrCreate()
session.sparkContext.setLogLevel('ERROR')
session.sparkContext._gateway.start_callback_server()
java_process = launch_gateway()
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=java_process),
callback_server_parameters=CallbackServerParameters(port=0))
listener = RetryListener()
session.sparkContext._jsc.sc().addSparkListener(listener)
input_warc = '/Users/a/CC-MAIN-20191013195541-20191013222541-00000.warc' # ToDo: Change path
raw_records = extract_raw_records(input_warc, session)
warc_records = raw_records.flatMap(parse_method)
print(warc_records.count())
gateway.shutdown_callback_server()
