def _start_gateway() -> JavaGateway: # launch Java side with dynamic port and get back the port on which the # server was bound to. port = launch_gateway(port=0, classpath=Context._find_jar_path('net-1.3.0-all.jar'), javaopts=['-javaagent:' + Context._find_jar_path('quasar-core-0.7.9-jdk8.jar') + '=mb', '-Xmx2048m'], die_on_exit=True) # connect python side to Java side with Java dynamic port and start python # callback server with a dynamic port gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True), callback_server_parameters=CallbackServerParameters(port=0)) # retrieve the port on which the python callback server was bound to. python_port = gateway.get_callback_server().get_listening_port() # tell the Java side to connect to the python callback server with the new # python port. Note that we use the java_gateway_server attribute that # retrieves the GatewayServer instance. gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) return gateway
def _start_gateway(port=0) -> JavaGateway: # launch Java side with dynamic port and get back the port on which the # server was bound to. hearthstone_jar_path = Context.find_resource_path( 'hearthstone-0.8.67.jar') net_jar_path = Context.find_resource_path('net-0.8.67-all.jar') port = launch_gateway(port=port, classpath=os.pathsep.join( (hearthstone_jar_path, net_jar_path)), die_on_exit=True) # connect python side to Java side with Java dynamic port and start python # callback server with a dynamic port gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True), callback_server_parameters=CallbackServerParameters(port=0)) # retrieve the port on which the python callback server was bound to. python_port = gateway.get_callback_server().get_listening_port() # tell the Java side to connect to the python callback server with the new # python port. Note that we use the java_gateway_server attribute that # retrieves the GatewayServer instance. gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) return gateway
def __init__(self, classpath='', redirect_stdout=None): """Create and return a gateway to a Java object with SpatialKappa in the classpath using py4j The distribution of SpatialKappa should contain the SpatialKappa jar file, so there should be no need to download it separately. Keyword arguments: classpath -- classpath to override the default values redirect_stdout -- Redirect stdout to this connection - see py4j.java_gateway.launch_gateway() """ skjar_file = 'SpatialKappa-v2.1.5.jar' antlrjar_file = 'ant-antlr-3.2.jar' skjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', skjar_file) antlrjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', antlrjar_file) classpath = os.pathsep.join((classpath, skjar_path, antlrjar_path)) self.id = SpatialKappa.count SpatialKappa.count = SpatialKappa.count + 1 self.redirect_stdout = redirect_stdout self.port = jg.launch_gateway(classpath=classpath, die_on_exit=False, redirect_stdout=self.redirect_stdout) self.gateway_client = jg.GatewayClient(port=self.port) self.gateway = jg.JavaGateway(self.gateway_client, auto_convert=True)
def init_jvm(jar_path="./libs", max_heap=1024): global jvm_gateway if jvm_gateway is not None: return libraries = [ '{0}{1}klay-python-wrapper-0.3.jar' ] jar_path = './libs' max_heap = 1024 classpath = os.pathsep.join([lib.format(jar_path, os.sep) for lib in libraries]) py4j_path = '{0}{1}py4j0.10.8.1.jar'.format(jar_path, os.sep) port = launch_gateway(jarpath=py4j_path, classpath=classpath, javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx{}m'.format(max_heap)], die_on_exit=True) try: jvm_gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_convert=True)) except Exception as e: jvm_gateway = None logging.debug('fail') logging.debug('success') return jvm_gateway.jvm
def _start_gateway(port=0) -> JavaGateway: # launch Java side with dynamic port and get back the port on which the # server was bound to. port = launch_gateway( port=port, classpath=Context.find_resource_path('net-0.8.22.jar'), javaopts=[ "--add-modules", "java.se", "--add-exports", "java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.management/sun.management=ALL-UNNAMED", "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED" ], die_on_exit=True) # connect python side to Java side with Java dynamic port and start python # callback server with a dynamic port gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True), callback_server_parameters=CallbackServerParameters(port=0)) # retrieve the port on which the python callback server was bound to. python_port = gateway.get_callback_server().get_listening_port() # tell the Java side to connect to the python callback server with the new # python port. Note that we use the java_gateway_server attribute that # retrieves the GatewayServer instance. gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) return gateway
async def on_startup(app): thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16) app['thread_pool'] = thread_pool port = launch_gateway( jarpath='/spark-2.4.0-bin-hadoop2.7/jars/py4j-0.10.7.jar', classpath='/spark-2.4.0-bin-hadoop2.7/jars/*:/hail.jar', die_on_exit=True) gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port), auto_convert=True) app['gateway'] = gateway hail_pkg = getattr(gateway.jvm, 'is').hail app['hail_pkg'] = hail_pkg jbackend = hail_pkg.backend.service.ServiceBackend.apply() app['jbackend'] = jbackend jhc = hail_pkg.HailContext.apply(jbackend, 'hail.log', False, False, 50, False, 3) app['jhc'] = jhc app['users'] = set() kube.config.load_incluster_config() k8s_client = kube.client.CoreV1Api() app['k8s_client'] = k8s_client
def __gateway(cls, classpath: str) -> JVMView: from py4j.java_gateway import launch_gateway # ISSUE: ambient port = launch_gateway(die_on_exit=True, classpath=classpath) gw = JavaGateway(gateway_parameters=GatewayParameters(port=port)) jvm = gw.jvm jvm = JavaGateway(gateway_parameters=GatewayParameters(port=port)).jvm return jvm
def __init__(self, *args, **kwargs): """ """ # path = os.path.realpath(__file__) path = os.path.dirname(os.path.realpath(__file__)) port = launch_gateway(classpath='{path}/libs/*:{path}'.format( **locals())) super(JavaGateway, self).__init__(gateway_parameters=GatewayParameters(port=port))
def newLocalStack(conf=DEFAULT_LOCAL_LAUNCH_CONF, classpath=get_Warp10_jar_path()): """Creates a new stack locally (not connected by a Warp 10 platform) """ port, auth_token = launch_gateway(enable_auth=True,die_on_exit=True,classpath=classpath) gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_convert=True, auth_token=auth_token)) entry_point = gateway.jvm.io.warp10.Py4JEntryPoint(conf) return entry_point.newStack()
def _init_registration(): from py4j.java_gateway import (launch_gateway, JavaGateway, GatewayParameters) global _java, _java_port if _java is not None: return _java_port = launch_gateway( classpath=os.path.join(library_path(), 'nben', 'target', 'nben-standalone.jar'), javaopts=['-Xmx2g'], die_on_exit=True) _java = JavaGateway(gateway_parameters=GatewayParameters(port=_java_port))
def launch_gateway(cls): jars_dir = os.environ["PYPMML_JARS_DIR"] if "PYPMML_JARS_DIR" in os.environ else \ path.join(path.dirname(path.abspath(__file__)), 'jars') launch_classpath = path.join(jars_dir, "*") _port = launch_gateway(classpath=launch_classpath, die_on_exit=True) gateway = JavaGateway( gateway_parameters=GatewayParameters(port=_port, auto_convert=True)) return gateway
def start_api_server(): port = launch_gateway( classpath= "/home/nithin/Desktop/Georgia Tech/Spring 2019/Adaptive Control and Reinforcement Learning/ACRL/Tetris/src", die_on_exit=True) global gateway gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port)) player_tetris = gateway.jvm.PlayerSkeletonCommunication().getFeatures() return player_tetris
def __init__(self, classpath=''): skjar_file = 'SpatialKappa-v2.1.1.jar' antlrjar_file = 'ant-antlr-3.2.jar' skjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', skjar_file) antlrjar_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'share', 'SpatialKappa', antlrjar_file) classpath = os.pathsep.join((classpath, skjar_path, antlrjar_path)) print classpath self.port = jg.launch_gateway(classpath=classpath, die_on_exit=True) self.gateway_client = jg.GatewayClient(port=self.port) self.gateway = jg.JavaGateway(self.gateway_client)
def _init_registration(): from py4j.java_gateway import (launch_gateway, JavaGateway, GatewayParameters) global _java, _java_port if _java is not None: return _java_port = launch_gateway( classpath=os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'lib', 'nben', 'target', 'nben-standalone.jar'), javaopts=['-Xmx2g'], die_on_exit=True) _java = JavaGateway(gateway_parameters=GatewayParameters(port=_java_port))
def __init__(self, tmpdir, log, quiet, append, branching_factor, skip_logging_configuration, optimizer_iterations): super(LocalBackend, self).__init__() spark_home = find_spark_home() hail_jar_path = os.environ.get('HAIL_JAR') if hail_jar_path is None: if pkg_resources.resource_exists(__name__, "hail-all-spark.jar"): hail_jar_path = pkg_resources.resource_filename( __name__, "hail-all-spark.jar") else: raise RuntimeError( 'local backend requires a packaged jar or HAIL_JAR to be set' ) port = launch_gateway(redirect_stdout=sys.stdout, redirect_stderr=sys.stderr, jarpath=f'{spark_home}/jars/py4j-0.10.9.jar', classpath=f'{spark_home}/jars/*:{hail_jar_path}', die_on_exit=True) self._gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True)) self._jvm = self._gateway.jvm hail_package = getattr(self._jvm, 'is').hail self._hail_package = hail_package self._utils_package_object = scala_package_object(hail_package.utils) self._jbackend = hail_package.backend.local.LocalBackend.apply(tmpdir) self._jhc = hail_package.HailContext.apply(self._jbackend, log, True, append, branching_factor, skip_logging_configuration, optimizer_iterations) # This has to go after creating the SparkSession. Unclear why. # Maybe it does its own patch? install_exception_handler() from hail.context import version py_version = version() jar_version = self._jhc.version() if jar_version != py_version: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n" f" JAR: {jar_version}\n" f" Python: {py_version}") self._fs = LocalFS() self._logger = None if not quiet: connect_logger(self._utils_package_object, 'localhost', 12888)
def __launch_java_gateway(cls) -> JavaGateway: """Launch java gateway.""" LOGGER.info('Initializing Farasa..') port = launch_gateway(classpath=CLASS_PATH, die_on_exit=True) params = GatewayParameters(port=port, auto_convert=True, auto_field=True, eager_load=True) return JavaGateway(gateway_parameters=params)
def __init__(self): if not os.path.isfile(JAR_PATH): raise ImportError("Could not find the JAR file associated with " "javac_parser") java_port = launch_gateway(jarpath=JAR_PATH, die_on_exit=True, redirect_stdout=sys.stdout, redirect_stderr=sys.stderr) self.gateway = JavaGateway( gateway_parameters=GatewayParameters(port=java_port) ) self.app = self.gateway.jvm.ca.ualberta.cs.App() assert self.app.getNumParseErrors("") == 0
async def start(self, ctx: Context): for (resource_name, context_attr, launch_jvm, gateway_params, callback_server_params, classpath, javaopts) in self.gateways: if launch_jvm: gateway_params.port = launch_gateway(classpath=classpath, javaopts=javaopts) gateway = JavaGateway(gateway_parameters=gateway_params, callback_server_parameters=callback_server_params) ctx.finished.connect(partial(self.shutdown_gateway, gateway=gateway, resource_name=resource_name, shutdown_jvm=launch_jvm)) ctx.publish_resource(gateway, resource_name, context_attr) logger.info('Configured Py4J gateway (%s / ctx.%s; address=%s, port=%d)', resource_name, context_attr, gateway_params.address, gateway_params.port)
def init_jvm(): global gateway # global javaChild """Initializes the Java virtual machine (JVM). """ if gateway != None: return folder_suffix = [ u'{0}', u'{0}{1}bin', u'{0}{1}jhannanum-0.8.4.jar', u'{0}{1}kkma-2.0.jar', u'{0}{1}komoran-2.4-e.jar', u'{0}{1}shineware-common-2.0.jar', u'{0}{1}shineware-ds-1.0.jar', u'{0}{1}snakeyaml-1.12.jar', u'{0}{1}scala-library-2.11.4.jar', u'{0}{1}twitter-korean-text-2.4.3.jar', u'{0}{1}twitter-text-1.10.1.jar', u'{0}{1}*'] javadir = u'%s%sjava' % (utils.installpath, os.sep) args = [javadir, os.sep] classpath = os.pathsep.join(f.format(*args) for f in folder_suffix) py4jpath = u'{0}{1}py4j-0.9.2.jar'.format(*args) port = launch_gateway(jarpath=py4jpath, classpath=classpath, javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx768m']) # javaChild = Popen(['java', # '-cp', classpath, # '-Dfile.encoding=UTF8', # '-ea', '-Xmx768m', # 'kr.lucypark.py4j.KonlpyGateway']); print "Initializing JAVA frameworks..." time.sleep(1) jvm_e = None for i in range(0, 10): try: time.sleep(1) gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port)) except Exception as e: gateway = None jvm_e = e; continue break if gateway == None: if jvm_e != None: raise jvm_e raise JvmError("Could not connect to JVM. unknown error"); print "JAVA frameworks initialized." atexit.register(shutdown_jvm);
def init_jvm(): global gateway # global javaChild """Initializes the Java virtual machine (JVM). """ if gateway != None: return folder_suffix = [ u'{0}', u'{0}{1}bin', u'{0}{1}jhannanum-0.8.4.jar', u'{0}{1}kkma-2.0.jar', u'{0}{1}komoran-2.4-e.jar', u'{0}{1}shineware-common-2.0.jar', u'{0}{1}shineware-ds-1.0.jar', u'{0}{1}snakeyaml-1.12.jar', u'{0}{1}scala-library-2.11.4.jar', u'{0}{1}twitter-korean-text-2.4.3.jar', u'{0}{1}twitter-text-1.10.1.jar', u'{0}{1}*' ] javadir = u'%s%sjava' % (utils.installpath, os.sep) args = [javadir, os.sep] classpath = os.pathsep.join(f.format(*args) for f in folder_suffix) py4jpath = u'{0}{1}py4j-0.9.2.jar'.format(*args) port = launch_gateway(jarpath=py4jpath, classpath=classpath, javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx768m']) # javaChild = Popen(['java', # '-cp', classpath, # '-Dfile.encoding=UTF8', # '-ea', '-Xmx768m', # 'kr.lucypark.py4j.KonlpyGateway']); print "Initializing JAVA frameworks..." time.sleep(1) jvm_e = None for i in range(0, 10): try: time.sleep(1) gateway = JavaGateway(gateway_parameters=GatewayParameters( port=port)) except Exception as e: gateway = None jvm_e = e continue break if gateway == None: if jvm_e != None: raise jvm_e raise JvmError("Could not connect to JVM. unknown error") print "JAVA frameworks initialized." atexit.register(shutdown_jvm)
def start_jvm(option, classpath): import os global GATEWAY jarpath = None for path in classpath: if 'py4j' in path: jarpath = path break port = launch_gateway(jarpath=jarpath, classpath=os.pathsep.join(classpath), javaopts=option) GATEWAY = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_close=True), callback_server_parameters=CallbackServerParameters()) return is_jvm_running()
def init_jvm(max_heap=1024, jar_path="./libs"): """KOMORAN Jar를 포함한 JVM을 초기화한 후, 반환합니다. Args: max_heap (int): JVM 실행 시 Max Heap Size (기본값: ``1024``, 단위: ``MB``) jar_path (str): JVM 실행 시 포함할 Jar Path 지정 (기본값: ``"./libs"``) Returns: py4j.java_gateway.JavaGateway: 초기화된 JVM 객체 """ base_path = os.path.dirname(os.path.realpath(__file__)) jar_path = os.path.abspath(os.path.join(base_path, jar_path)) assert os.path.exists(jar_path) global jvm_gateway if jvm_gateway is not None: return libraries = [ '{0}{1}KOMORAN-4e7b5ef.jar', '{0}{1}KOMORANEntryPoint-0.1.0.jar', # '{0}{1}*' ] classpath = os.pathsep.join( [lib.format(jar_path, os.sep) for lib in libraries]) py4j_path = "{0}{1}py4j0.10.8.1.jar".format(jar_path, os.sep) port = launch_gateway( jarpath=py4j_path, classpath=classpath, javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx{}m'.format(max_heap)], die_on_exit=True) logging.debug("initializing JVM... ") try: jvm_gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True)) # # for debugging with Java-side # jvm_gateway = JavaGateway(gateway_parameters=GatewayParameters(port=25335, auto_convert=True)) except Exception as e: jvm_gateway = None logging.debug("fail") logging.debug("success") return jvm_gateway.jvm
def __init__(self, tmpdir, log, quiet, append, branching_factor, skip_logging_configuration, optimizer_iterations): SPARK_HOME = os.environ['SPARK_HOME'] HAIL_HOME = os.environ['HAIL_HOME'] port = launch_gateway( redirect_stdout=sys.stdout, redirect_stderr=sys.stderr, jarpath=f'{SPARK_HOME}/jars/py4j-0.10.7.jar', classpath= f'{SPARK_HOME}/jars/*:{HAIL_HOME}/hail/build/libs/hail-all-spark.jar', die_on_exit=True) self._gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port, auto_convert=True)) self._jvm = self._gateway.jvm hail_package = getattr(self._jvm, 'is').hail self._hail_package = hail_package self._utils_package_object = scala_package_object(hail_package.utils) self._jbackend = hail_package.backend.local.LocalBackend.apply(tmpdir) self._jhc = hail_package.HailContext.apply(self._jbackend, log, True, append, branching_factor, skip_logging_configuration, optimizer_iterations) # This has to go after creating the SparkSession. Unclear why. # Maybe it does its own patch? install_exception_handler() from hail.context import version py_version = version() jar_version = self._jhc.version() if jar_version != py_version: raise RuntimeError( f"Hail version mismatch between JAR and Python library\n" f" JAR: {jar_version}\n" f" Python: {py_version}") self._fs = LocalFS() self._logger = None if not quiet: connect_logger(self._utils_package_object, 'localhost', 12888)
def start_jvm(option, classpath, port: int = None): import os global GATEWAY jarpath = None for path in classpath: if 'py4j' in path: jarpath = path break gateway_port = launch_gateway(jarpath=jarpath, classpath=os.pathsep.join(classpath), javaopts=option) logging.info("Java gateway started with port number %s", gateway_port) if port is None: port = DEFAULT_PYTHON_PROXY_PORT logging.info("Callback server will use port number %s", port) GATEWAY = JavaGateway(gateway_parameters=GatewayParameters(port=gateway_port, auto_close=True), callback_server_parameters=CallbackServerParameters(port=port)) return is_jvm_running()
def get_gateway(extra_jar=None): if extra_jar is None: extra_jar = find_jar_path() port = launch_gateway( die_on_exit=True, classpath=extra_jar, # redirect_stderr=sys.stderr, redirect_stdout=sys.stdout ) gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port), callback_server_parameters=CallbackServerParameters(port=0)) # retrieve the port on which the python callback server was bound to. python_port = gateway.get_callback_server().get_listening_port() gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) return gateway
def start(): global pythonManager, gateway java_home = os.environ.get("JAVA_HOME") if java_home is None: raise SystemError("The environment variable JAVA_HOME is not set") java_path = os.path.join(java_home, "bin", "java") gmql_jar_fn = resource_filename("gmql", os.path.join("resources", __gmql_jar)) gmql_jar_fn = __check_backend(gmql_jar_fn) libs_folder = resource_filename("gmql", os.path.join("resources", "lib")) if sys.platform.startswith("win32"): classpath_separator = ";" else: classpath_separator = ":" classpath = gmql_jar_fn + classpath_separator + os.path.join( libs_folder, "*") _port = launch_gateway(classpath=classpath, die_on_exit=True, java_path=java_path, javaopts=['-Xmx4096m']) # gateway = JavaGateway.launch_gateway(classpath=gmql_jar_fn, die_on_exit=True, # java_path=java_path) gateway = JavaGateway( gateway_parameters=GatewayParameters(port=_port, auto_convert=True)) python_api_package = __get_python_api_package(gateway) pythonManager = __start_gmql_manager(python_api_package) if sys.platform.startswith("win32"): # if we are on windows set the hadoop home to winutils.exe hadoopFolder_fn = resource_filename( "gmql", os.path.join("resources", "hadoop")) pythonManager.setHadoopHomeDir(hadoopFolder_fn) # setting the spark tmp folder pythonManager.setSparkLocalDir(folders['spark'])
def create_jvm(logger): cp = {} port = None cp.update({'lib': ['*.jar'], 'jars': ['*.jar']}) if cp: final_cp = [] for path, exprs in cp.items(): for expr in exprs: final_cp.extend(glob.glob(os.path.join(path, expr))) if not final_cp: logger.warn('Hadoop JARs not found. Data source upload will not ' 'work. Set HADOOP_HOME and/or SPARK_HOME environment ' 'variables to the correct path.') else: final_cp = [os.path.abspath(p) for p in final_cp] port = launch_gateway(classpath=":".join(final_cp), redirect_stdout=sys.stdout, redirect_stderr=sys.stderr, die_on_exit=True) print '>>>>>>>>>>>>>>>>>', port return port
def start_java(py4j_path=None, accumulo_path=None, graphulo_path=None, dieonexit=None): """ Starts JVM along with a Py4J JavaGateway instance, with classpath exposing Accumulo and Graphulo to the JVM. Inputs: py4j_path = full path for py4j jar accumulo_path = full path of directory containing Accumulo jars graphulo_path = full path for graphulo jar dieonexit = option to have JVM close upon exiting of python Note: - By default, uses packaged JARs """ if py4j_path is None: py4j_path = resource_filename(__name__, '/jars/py4j/share/py4j/py4j.0.1.jar') if accumulo_path is None: accumulo_path = resource_filename(__name__, '/jars/libext/*') if graphulo_path is None: graphulo_path = resource_filename(__name__, '/jars/lib/graphulo-3.0.0.jar') cp = '.:'+accumulo_path+':'+graphulo_path if dieonexit is None: dieonexit = True port = launch_gateway(jarpath=py4j_path, classpath=cp, die_on_exit=dieonexit) gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port), callback_server_parameters=CallbackServerParameters(port=0)) print('JavaGateway started in Port '+str(port)) JavaConnector.jvm_port = port JavaConnector.jvm_gateway = gateway return gateway
def get_instance(self): if not (self.instance is None): return self.instance if self.launch: self.port, token = launch_gateway(enable_auth=True, die_on_exit=True, classpath=get_Warp10_jar_path()) if self.verbose: print('Local gateway launched on port ' + str(self.port)) instance = JavaGateway(gateway_parameters=GatewayParameters( port=self.port, auto_convert=True, auth_token=token)) else: instance = JavaGateway(gateway_parameters=GatewayParameters( self.addr, self.port, auto_convert=True, auth_token=self.auth_token)) if self.verbose: print('Establish connection with a gateway at ' + self.addr + ":" + str(self.port)) return instance
def get_gateway(extra_jar=None): if extra_jar is None: extra_jar = find_jar_path() port = launch_gateway( die_on_exit=True, classpath=extra_jar, # redirect_stderr=sys.stderr, redirect_stdout=sys.stdout ) gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port), callback_server_parameters=CallbackServerParameters(port=0)) # retrieve the port on which the python callback server was bound to. python_port = gateway.get_callback_server().get_listening_port() gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) return gateway
def create_jvm(logger): port = None dist_dirs = ['.', os.getenv('HADOOP_HOME'), os.getenv('SPARK_HOME')] cp = {'lib': ['*.jar'], 'jars': ['*.jar']} final_cp = [] for dist_dir in dist_dirs: if dist_dir: for path, exprs in list(cp.items()): for expr in exprs: # print(os.path.join(dist_dir, path, expr)) final_cp.extend( glob.glob(os.path.join(dist_dir, path, expr))) if not final_cp: logger.warn('Hadoop JARs not found. Data source upload will not ' 'work. Set HADOOP_HOME and/or SPARK_HOME environment ' 'variables to the correct path.') else: final_cp = [os.path.abspath(p) for p in final_cp] port = launch_gateway(classpath=":".join(final_cp), redirect_stdout=sys.stdout, redirect_stderr=sys.stderr, die_on_exit=True) return port
def launch_gateway(self): self.update_log_files() java_sock = socket() java_sock.bind(('', 0)) java_port = java_sock.getsockname()[1] python_sock = socket() python_sock.bind(('', 0)) python_port = python_sock.getsockname()[1] java_sock.close() python_sock.close() port = launch_gateway(classpath=self.classpath, redirect_stderr=self.err, redirect_stdout=self.out, die_on_exit=True, jarpath=self.jarpath, port=java_port) self.gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port), callback_server_parameters=CallbackServerParameters( port=python_port)) python_port = self.gateway.get_callback_server().get_listening_port() self.gateway.java_gateway_server.resetCallbackClient( self.gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) process_name = self.gateway.jvm.java.lang.management.ManagementFactory.getRuntimeMXBean( ).getName() self.java_pid = int(process_name.split('@')[0])
def launch_gateway(cls, javaopts=[], java_path=None): """Launch a `Gateway` in a new Java process. :param javaopts: an array of extra options to pass to Java (the classpath should be specified using the `classpath` parameter, not `javaopts`.) :param java_path: If None, Py4J will use $JAVA_HOME/bin/java if $JAVA_HOME is defined, otherwise it will use "java". :return: An object of `Gateway` """ jars_dir = os.environ["PYPMML_JARS_DIR"] if "PYPMML_JARS_DIR" in os.environ else \ path.join(path.dirname(path.abspath(__file__)), 'jars') launch_classpath = path.join(jars_dir, "*") if not javaopts: java_opts = os.environ.get("JAVA_OPTS") if java_opts: javaopts = java_opts.split() _port = launch_gateway(classpath=launch_classpath, javaopts=javaopts, java_path=java_path, die_on_exit=True) gateway = JavaGateway(gateway_parameters=GatewayParameters( port=_port, auto_convert=True)) return gateway
def run(self, parent_data_objs): # Run the java py4j entry point comp_dir = self._dag_node.comp_root_path() self._logger.info("comp_dir: {}".format(comp_dir)) jar_files = glob.glob(os.path.join(comp_dir, "*.jar")) self._logger.info("Java classpath files: {}".format(jar_files)) component_class = self._dag_node.comp_class() java_jars = [self._mlcomp_jar] + jar_files class_path = ":".join(java_jars) java_gateway = None all_ok = False monitor_proc = None try: total_phys_mem_size_mb = ByteConv.from_bytes( psutil.virtual_memory().total).mbytes jvm_heap_size_option = "-Xmx{}m".format( int(math.ceil(total_phys_mem_size_mb))) java_opts = [jvm_heap_size_option] self._logger.info("JVM options: {}".format(java_opts)) # Note: the jarpath is set to be the path to the mlcomp jar since the launch_gateway code is checking # for the existence of the jar. The py4j jar is packed inside the mlcomp jar. java_port = launch_gateway(port=0, javaopts=java_opts, die_on_exit=True, jarpath=self._mlcomp_jar, classpath=class_path, redirect_stdout=sys.stdout, redirect_stderr=sys.stderr) java_gateway = JavaGateway( gateway_parameters=GatewayParameters(port=java_port), callback_server_parameters=CallbackServerParameters(port=0), python_server_entry_point=MLOpsPY4JWrapper()) python_port = java_gateway.get_callback_server( ).get_listening_port() self._logger.debug("Python port: {}".format(python_port)) java_gateway.java_gateway_server.resetCallbackClient( java_gateway.java_gateway_server.getCallbackClient(). getAddress(), python_port) mlops_wrapper = MLOpsPY4JWrapper() entry_point = java_gateway.jvm.com.parallelm.mlcomp.ComponentEntryPoint( component_class) component_via_py4j = entry_point.getComponent() component_via_py4j.setMLOps(mlops_wrapper) # Configure m = java_gateway.jvm.java.util.HashMap() for key in self._params.keys(): # py4j does not handle nested structures. So the configs which is a dict will not be passed to the java # layer now. if isinstance(self._params[key], dict): continue m[key] = self._params[key] component_via_py4j.configure(m) # Materialized l = java_gateway.jvm.java.util.ArrayList() for obj in parent_data_objs: l.append(obj) self._logger.info("Parent obj: {} type {}".format( obj, type(obj))) self._logger.info("Parent objs: {}".format(l)) if mlops_loaded: monitor_proc = ProcessMonitor(mlops, self._ml_engine) monitor_proc.start() py4j_out_objs = component_via_py4j.materialize(l) self._logger.debug(type(py4j_out_objs)) self._logger.debug(len(py4j_out_objs)) python_out_objs = [] for obj in py4j_out_objs: self._logger.debug("Obj:") self._logger.debug(obj) python_out_objs.append(obj) self._logger.info( "Done running of materialize and getting output objects") all_ok = True except Py4JJavaError as e: self._logger.error("Error in java code: {}".format(e)) raise MLCompException(str(e)) except Exception as e: self._logger.error("General error: {}".format(e)) raise MLCompException(str(e)) finally: self._logger.info("In finally block: all_ok {}".format(all_ok)) if java_gateway: java_gateway.close_callback_server() java_gateway.shutdown() if mlops_loaded and monitor_proc: monitor_proc.stop_gracefully() return python_out_objs
def gateway(self): if self._gateway is None: self._gateway = launch_gateway() return self._gateway
from django.conf import settings from py4j.java_gateway import launch_gateway, JavaGateway,\ GatewayParameters, CallbackServerParameters, is_instance_of logging.basicConfig(level=logging.INFO, format="%(message)s") # Launch a gateway to the all Java JAR dependencies that we have classpath = ':'.join([ os.path.join(settings.DEPS_DIR, jar) for jar in os.listdir('deps') ]) port_number = launch_gateway( classpath=classpath, die_on_exit=True, ) gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port_number), callback_server_parameters=CallbackServerParameters(port=0), ) python_port = gateway.get_callback_server().get_listening_port() gateway.java_gateway_server.resetCallbackClient( gateway.java_gateway_server.getCallbackClient().getAddress(), python_port, ) def java_isinstance(java_object, java_class): ''' Convenience method to abstract away the gateway server to the
def parse_method(text: str) -> Tuple[WarcRecord]: parsed_raw_warc = parse_raw_warc(text) # crasher = 5 / 0 # ToDo: Uncomment # print(crasher) # ToDo: Uncomment return parsed_raw_warc if __name__ == "__main__": session: SparkSession = SparkSession.builder \ .master('local[3, 3]') \ .appName('Failure Exploration') \ .getOrCreate() session.sparkContext.setLogLevel('ERROR') session.sparkContext._gateway.start_callback_server() java_process = launch_gateway() gateway = JavaGateway( gateway_parameters=GatewayParameters(port=java_process), callback_server_parameters=CallbackServerParameters(port=0)) listener = RetryListener() session.sparkContext._jsc.sc().addSparkListener(listener) input_warc = '/Users/a/CC-MAIN-20191013195541-20191013222541-00000.warc' # ToDo: Change path raw_records = extract_raw_records(input_warc, session) warc_records = raw_records.flatMap(parse_method) print(warc_records.count()) gateway.shutdown_callback_server()