Пример #1
0
def start_clientserver_example_app_process(start_java_client=False,
                                           start_short_timeout=False,
                                           start_gc_test=False,
                                           auth_token=None):
    args = ()
    gw_params = GatewayParameters()
    if auth_token:
        args = ("--auth-token", auth_token)
        gw_params = GatewayParameters(auth_token=auth_token)

    # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED
    if start_short_timeout:
        p = Process(target=start_short_timeout_clientserver_example_server,
                    args=args)
    elif start_java_client:
        p = Process(target=start_java_clientserver_example_server, args=args)
    elif start_gc_test:
        p = Process(target=start_java_clientserver_gc_example_server,
                    args=args)
    else:
        p = Process(target=start_clientserver_example_server, args=args)
    p.start()
    sleep()

    check_connection(gateway_parameters=gw_params)
    return p
Пример #2
0
    def __gateway(cls, classpath: str) -> JVMView:
        from py4j.java_gateway import launch_gateway  # ISSUE: ambient

        port = launch_gateway(die_on_exit=True, classpath=classpath)
        gw = JavaGateway(gateway_parameters=GatewayParameters(port=port))
        jvm = gw.jvm
        jvm = JavaGateway(gateway_parameters=GatewayParameters(port=port)).jvm
        return jvm
Пример #3
0
def start_java_multi_client_server_app_process():
    # XXX DO NOT FORGET TO KILL THE PROCESS IF THE TEST DOES NOT SUCCEED
    p = Process(target=start_java_multi_client_server_app)
    p.start()
    sleep()
    # test both gateways...
    check_connection(gateway_parameters=GatewayParameters(port=DEFAULT_PORT))
    check_connection(gateway_parameters=GatewayParameters(port=DEFAULT_PORT +
                                                          2))
    return p
Пример #4
0
 def __init__(
     self, server_starter:
     'nl4py.NetLogoControllerServerStarter.NetLogoControllerServerStarter'):
     self.server_starter = server_starter
     self.java_server = self.server_starter.jg.jvm.nl4py.server.NetLogoControllerServer(
     )
     self.server_gateway = JavaGateway(gateway_parameters=GatewayParameters(
         auto_convert=True, port=server_starter.server_port))
     gs = self.java_server.newGateway()
     self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
         auto_convert=True, port=gs.getPort(), auto_close=True))
     self.app = self.gateway.jvm.nl4py.server.NetLogoAppController(gs)
    def testGatewayAuth(self):
        self.gateway = JavaGateway.launch_gateway(enable_auth=True)

        # Make sure the default client can connect to the server.
        klass = self.gateway.jvm.java.lang.String
        help_page = self.gateway.help(klass, short_name=True, display=False)
        self.assertGreater(len(help_page), 1)

        # Replace the client with one that does not authenticate.
        # Make sure it fails.
        bad_client = GatewayClient(gateway_parameters=GatewayParameters(
            address=self.gateway.gateway_parameters.address,
            port=self.gateway.gateway_parameters.port))
        self.gateway.set_gateway_client(bad_client)
        try:
            self.gateway.help(klass, short_name=True, display=False)
            self.fail("Expected failure to communicate with gateway server.")
        except Exception:
            # Expected
            pass
        finally:
            # Restore a good client. This allows the gateway to be shut down.
            good_client = GatewayClient(
                gateway_parameters=self.gateway.gateway_parameters)
            self.gateway.set_gateway_client(good_client)
    def __init__(self, job_args):

        self.use_api = job_args.get('use_api', False)

        #setup the java gateway binding
        self._gateway = JavaGateway(gateway_parameters=GatewayParameters(
            port=job_args['java_gateway_port']))
        repos = ["opensha", "nshm-nz-opensha"]
        self._repoheads = get_repo_heads(PurePath(job_args['root_folder']),
                                         repos)
        self._output_folder = PurePath(job_args.get('working_path'))

        if self.use_api:
            headers = {"x-api-key": API_KEY}
            self._ruptgen_api = RuptureGenerationTask(
                API_URL,
                S3_URL,
                None,
                with_schema_validation=True,
                headers=headers)
            self._general_api = GeneralTask(API_URL,
                                            S3_URL,
                                            None,
                                            with_schema_validation=True,
                                            headers=headers)
            self._task_relation_api = TaskRelation(API_URL,
                                                   None,
                                                   with_schema_validation=True,
                                                   headers=headers)
            self._toshi_api = ToshiApi(API_URL,
                                       S3_URL,
                                       None,
                                       with_schema_validation=True,
                                       headers=headers)
Пример #7
0
def get_gateway():
    # type: () -> JavaGateway
    global _gateway
    global _lock
    with _lock:
        if _gateway is None:
            # Set the level to WARN to mute the noisy INFO level logs
            logger.level = WARN
            # if Java Gateway is already running
            if 'PYFLINK_GATEWAY_PORT' in os.environ:
                gateway_port = int(os.environ['PYFLINK_GATEWAY_PORT'])
                gateway_param = GatewayParameters(port=gateway_port, auto_convert=True)
                _gateway = JavaGateway(
                    gateway_parameters=gateway_param,
                    callback_server_parameters=CallbackServerParameters(
                        port=0, daemonize=True, daemonize_connections=True))
            else:
                _gateway = launch_gateway()

            callback_server = _gateway.get_callback_server()
            callback_server_listening_address = callback_server.get_listening_address()
            callback_server_listening_port = callback_server.get_listening_port()
            _gateway.jvm.org.apache.flink.client.python.PythonEnvUtils.resetCallbackClient(
                _gateway.java_gateway_server,
                callback_server_listening_address,
                callback_server_listening_port)
            # import the flink view
            import_flink_view(_gateway)
            install_exception_handler()
            install_py4j_hooks()
            _gateway.entry_point.put("PythonFunctionFactory", PythonFunctionFactory())
            _gateway.entry_point.put("Watchdog", Watchdog())
    return _gateway
Пример #8
0
    def testGCCollectNoMemoryManagement(self):
        self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
            enable_memory_management=False))
        gc.collect()
        # Should have nothing in the finalizers
        self.assertEqual(len(ThreadSafeFinalizer.finalizers), 0)

        def internal():
            sb = self.gateway.jvm.java.lang.StringBuffer()
            sb.append("Hello World")
            sb2 = self.gateway.jvm.java.lang.StringBuffer()
            sb2.append("Hello World")
            finalizers_size_middle = len(ThreadSafeFinalizer.finalizers)
            return finalizers_size_middle

        finalizers_size_middle = internal()
        gc.collect()

        # Before collection: two objects created + two returned objects (append
        # returns a stringbuffer reference for easy chaining).
        self.assertEqual(finalizers_size_middle, 0)

        # Assert after collection
        self.assertEqual(len(ThreadSafeFinalizer.finalizers), 0)

        self.gateway.shutdown()
Пример #9
0
 def _validate(self, _):
     # here is where we can get java and python ports and change the
     # defaults for connecting
     try:
         self._bridge = Py4jServiceBridge(
             service_listener=self,
             gateway_parameters=GatewayParameters(port=self._javaport),
             callback_server_parameters=CallbackServerParameters(
                 port=self._pythonport
             ),
         )
         self._bridge.connect()
     except Exception as e:
         self._bridge = None
         raise e
     # Once bridge is connected, instantiate container using bridge id
     container_props = self._prepare_container_props(
         self._supported_intents, None
     )
     if self._default_service_timeout:
         container_props[
             ECF_PY4J_DEFAULT_SERVICE_TIMEOUT
         ] = self._default_service_timeout
     self._container = self._ipopo.instantiate(
         self._config_name, self._bridge.get_id(), container_props
     )
Пример #10
0
    def __init__(self, sourceFile, targetFile):
        self.SWindow = []
        self.TWindow = []
        self.TPredictWindow = []

        self.SDataBuffer = []  #Queue
        self.TDataBuffer = []  #Queue

        self.SInitialDataBuffer = []
        self.TInitialDataBuffer = []

        self.changeDetector = ChangeDetection(Properties.GAMMA,
                                              Properties.SENSITIVITY,
                                              Properties.MAX_WINDOW_SIZE)
        self.ensemble = Ensemble(Properties.ENSEMBLE_SIZE)

        classNameList = []
        self.source = Stream(sourceFile, classNameList,
                             Properties.INITIAL_DATA_SIZE)
        self.target = Stream(targetFile, classNameList,
                             Properties.INITIAL_DATA_SIZE)

        Properties.MAXVAR = self.source.MAXVAR

        self.gateway = JavaGateway(
            start_callback_server=True,
            gateway_parameters=GatewayParameters(port=Properties.PY4JPORT),
            callback_server_parameters=CallbackServerParameters(
                port=Properties.PY4JPORT + 1))
        self.app = self.gateway.entry_point
Пример #11
0
 def _start_gateway() -> JavaGateway:
     # launch Java side with dynamic port and get back the port on which the
     # server was bound to.
     port = launch_gateway(port=0,
                           classpath=Context._find_jar_path('net-1.3.0-all.jar'),
                           javaopts=['-javaagent:' + Context._find_jar_path('quasar-core-0.7.9-jdk8.jar') + '=mb',
                                     '-Xmx2048m'],
                           die_on_exit=True)
     
     # connect python side to Java side with Java dynamic port and start python
     # callback server with a dynamic port
     gateway = JavaGateway(
         gateway_parameters=GatewayParameters(port=port, auto_convert=True),
         callback_server_parameters=CallbackServerParameters(port=0))
     
     # retrieve the port on which the python callback server was bound to.
     python_port = gateway.get_callback_server().get_listening_port()
     
     # tell the Java side to connect to the python callback server with the new
     # python port. Note that we use the java_gateway_server attribute that
     # retrieves the GatewayServer instance.
     gateway.java_gateway_server.resetCallbackClient(
         gateway.java_gateway_server.getCallbackClient().getAddress(),
         python_port)
     return gateway
Пример #12
0
    def _start_gateway(self, p2=Machete):
        # auto select callback server port and reset it in java env
        self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
            port=self.port), callback_server_parameters=CallbackServerParameters(port=0))
        python_port = self.gateway.get_callback_server().get_listening_port()
        self.gateway.java_gateway_server.resetCallbackClient(
            self.gateway.java_gateway_server.getCallbackClient().getAddress(), python_port)
        self.manager = self.gateway.entry_point

        # create pipe between gym_env_api and python_ai for java env
        server, client = Pipe()
        self.pipe = server
        self.p1 = GymAIDisplay(self.gateway, client, False)
        self.manager.registerAI(self.p1.__class__.__name__, self.p1)

        if isinstance(p2, str):
            # p2 is a java class name
            self.p2 = p2
            self.game_to_start = self.manager.createGame(
                "ZEN", "ZEN", self.p1.__class__.__name__, self.p2, self.freq_restart_java)
        else:
            # p2 is a python class
            self.p2 = p2(self.gateway)
            self.manager.registerAI(self.p2.__class__.__name__, self.p2)
            self.game_to_start = self.manager.createGame(
                "ZEN", "ZEN", self.p1.__class__.__name__, self.p2.__class__.__name__, self.freq_restart_java)
        self.game = Thread(target=game_thread,
                           name="game_thread", args=(self, ))
        self.game.start()

        self.game_started = True
        self.round_num = 0
Пример #13
0
    def init(self):
        """
        Sets up the Py4J Gateway.  This is called automatically when other pygw classes are imported.
        """
        if not self.is_initialized:
            try:
                if self.GATEWAY is None:
                    gateway_address = environ.get(PYGW_ADDRESS_ENV,
                                                  DEFAULT_ADDRESS)
                    gateway_port = environ.get(PYGW_PORT_ENV, DEFAULT_PORT)
                    self.GATEWAY = JavaGateway(
                        gateway_parameters=GatewayParameters(
                            auto_field=True,
                            address=gateway_address,
                            port=gateway_port))
                    try:
                        self.geowave_version = self.GATEWAY.jvm.org.locationtech.geowave.core.cli.VersionUtils\
                            .getVersion()
                    except TypeError as e:
                        raise GatewayConfiguration.PyGwGeoWaveNotFoundInGateway(
                            "GeoWave was not found in the configured gateway.  Make sure GeoWave jars are available "
                            "on the classpath of the running gateway.") from e

                    self.is_initialized = True

            except Py4JNetworkError as e:
                raise GatewayConfiguration.PyGwJavaGatewayNotStartedError(
                    "The GeoWave Py4J Java Gateway must be running before you can use pygw."
                ) from e
Пример #14
0
def main():
    p = OptionParser()
    p.add_option('-j',
                 '--jvm_port',
                 help='the port for jvm side',
                 dest='jvm_port',
                 type=int,
                 default=None)
    p.add_option('-p',
                 '--py_port',
                 help='the port for python side',
                 dest='py_port',
                 type=int,
                 default=None)
    (z, args) = p.parse_args()

    app = PyMain()
    global gateway
    gateway = JavaGateway(gateway_parameters=GatewayParameters(
        port=z.jvm_port, auto_field=True, auto_convert=True),
                          callback_server_parameters=CallbackServerParameters(
                              port=z.py_port,
                              daemonize=True,
                              daemonize_connections=False,
                              propagate_java_exceptions=True),
                          python_server_entry_point=app)
    print('Started Listening On {}'.format(
        gateway.get_callback_server().get_listening_port()))
    sys.stdout.flush()
    while app.check():
        time.sleep(1.0)
    print('Exit')
Пример #15
0
def get_java_gateway(port=0,
                     jarpath="",
                     classpath="",
                     javaopts=[],
                     die_on_exit=False,
                     redirect_stdout=None,
                     redirect_stderr=None,
                     daemonize_redirect=True,
                     java_path="java",
                     create_new_process_group=False,
                     enable_auth=False):
    _ret = launch_gateway(port,
                          jarpath,
                          classpath,
                          javaopts,
                          die_on_exit,
                          redirect_stdout=redirect_stdout,
                          redirect_stderr=redirect_stderr,
                          daemonize_redirect=daemonize_redirect,
                          java_path=java_path,
                          create_new_process_group=create_new_process_group,
                          enable_auth=enable_auth,
                          return_proc=True)

    if enable_auth:
        _port, _auth_token, proc = _ret
    else:
        _port, proc, _auth_token = _ret + (None, )
    gateway = JavaGateway(gateway_parameters=GatewayParameters(
        port=_port, auth_token=_auth_token))
    # NOTE 3: the Popen object is now available with gateway._proc
    gateway._proc = proc
    return gateway
Пример #16
0
    def setUp(self):
        key_file = os.path.join(
            os.path.dirname(os.path.realpath(__file__)),
            "selfsigned.pem")

        client_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        client_ssl_context.verify_mode = ssl.CERT_REQUIRED
        client_ssl_context.check_hostname = True
        client_ssl_context.load_verify_locations(cafile=key_file)

        server_ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        server_ssl_context.load_cert_chain(key_file, password='******')

        callback_server_parameters = CallbackServerParameters(
            ssl_context=server_ssl_context)
        # address must match cert, because we're checking hostnames
        gateway_parameters = GatewayParameters(
            address='localhost',
            ssl_context=client_ssl_context)

        self.p = start_example_tls_process()
        self.gateway = JavaGateway(
            gateway_parameters=gateway_parameters,
            callback_server_parameters=callback_server_parameters)
        # It seems SecureServerSocket may need a little more time to
        # initialize on some platforms/slow machines.
        sleep(0.500)
Пример #17
0
    def _setup_py4j_client_connection(self):
        gateway_params = GatewayParameters(port=self._java_port,
                                           auto_field=True,
                                           auto_close=True,
                                           eager_load=True)
        callback_server_params = CallbackServerParameters(
            port=0,
            daemonize=True,
            daemonize_connections=True,
            eager_load=True)
        self._gateway = JavaGateway(
            gateway_parameters=gateway_params,
            callback_server_parameters=callback_server_params,
            python_server_entry_point=self)
        self._component_via_py4j = self._gateway.entry_point.getComponent()
        if not self._component_via_py4j:
            raise MLCompException("None reference of py4j java object!")

        if self._verbose:
            self._logger.debug(
                self._prefix_msg +
                "Py4J component referenced successfully! comp_via_py4j: {}".
                format(self._component_via_py4j))

        self._component_via_py4j.setEnvAttributes(self.get_wid(),
                                                  self._verbose)
Пример #18
0
    def __init__(self, port_num=25333):

        self.process = None
        self.pid = None
        self.port_number = str(port_num)

        this_folder = os.path.dirname(
            os.path.abspath(inspect.getfile(inspect.currentframe())))
        root_folder = os.path.dirname(this_folder)
        jar_file_name = os.path.join(
            root_folder, 'javacnct', 'target',
            'otm-py4j-1.0-SNAPSHOT-jar-with-dependencies.jar')

        #First check if the file exists indeed:
        if os.path.isfile(jar_file_name):

            if platform.system() == "Windows":
                self.openWindows(jar_file_name, self.port_number)
            elif platform.system() in ["Linux", "Darwin"]:
                self.openLinux(jar_file_name, self.port_number)
            else:
                raise Exception('Unknown platform')

            self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
                auto_convert=True, port=int(self.port_number)))
            # self.gateway = JavaGateway(gateway_parameters=GatewayParameters(port=int(self.port_number)))
            # self.gateway = JavaGateway()

        else:
            print("Jar file missing")
Пример #19
0
def init_jvm(jar_path="./libs", max_heap=1024):

    global jvm_gateway

    if jvm_gateway is not None:
        return

    libraries = [
        '{0}{1}klay-python-wrapper-0.3.jar'
    ]
    jar_path = './libs'
    max_heap = 1024

    classpath = os.pathsep.join([lib.format(jar_path, os.sep) for lib in libraries])
    py4j_path = '{0}{1}py4j0.10.8.1.jar'.format(jar_path, os.sep)

    port = launch_gateway(jarpath=py4j_path,
                          classpath=classpath,
                          javaopts=['-Dfile.encoding=UTF8', '-ea', '-Xmx{}m'.format(max_heap)],
                          die_on_exit=True)

    try:
        jvm_gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port, auto_convert=True))
    except Exception as e:
        jvm_gateway = None
        logging.debug('fail')

    logging.debug('success')

    return jvm_gateway.jvm
Пример #20
0
    def _start_gateway(port=0) -> JavaGateway:
        # launch Java side with dynamic port and get back the port on which the
        # server was bound to.
        port = launch_gateway(
            port=port,
            classpath=Context.find_resource_path('net-0.8.22.jar'),
            javaopts=[
                "--add-modules", "java.se", "--add-exports",
                "java.base/jdk.internal.ref=ALL-UNNAMED", "--add-opens",
                "java.base/java.lang=ALL-UNNAMED", "--add-opens",
                "java.base/java.nio=ALL-UNNAMED", "--add-opens",
                "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens",
                "java.management/sun.management=ALL-UNNAMED", "--add-opens",
                "jdk.management/com.sun.management.internal=ALL-UNNAMED"
            ],
            die_on_exit=True)

        # connect python side to Java side with Java dynamic port and start python
        # callback server with a dynamic port
        gateway = JavaGateway(
            gateway_parameters=GatewayParameters(port=port, auto_convert=True),
            callback_server_parameters=CallbackServerParameters(port=0))

        # retrieve the port on which the python callback server was bound to.
        python_port = gateway.get_callback_server().get_listening_port()

        # tell the Java side to connect to the python callback server with the new
        # python port. Note that we use the java_gateway_server attribute that
        # retrieves the GatewayServer instance.
        gateway.java_gateway_server.resetCallbackClient(
            gateway.java_gateway_server.getCallbackClient().getAddress(),
            python_port)
        return gateway
Пример #21
0
    def __init__(self, setting_file_name: str,
                 utility_space: abstractUtilitySpace.AbstractUtilitySpace,
                 negotiation_rule: negotiationRule.NegotiationRule,
                 agent_id: int, agent_num: int, port: int):
        """
        :param str setting_file_name: 効用ドメインファイルのパス
        :param AbstractUtilitySpace utility_space: 効用空間の情報が取得できる
        :param NegotiationRule negotiation_rule: 交渉の時間やタイプ,現在の正規化時間が取得できる
        :param int agent_id: 自分のエージェントに割り振られたid
        :param int agent_num: 交渉参加エージェントの数
        :param int port: 通信を行うポート番号
        """
        self.__utility_space = utility_space
        self.__rule = negotiation_rule
        self.__agent_id = agent_id
        self.__issue_size_list = self.__utility_space.get_issue_size_list()
        self.__opponent_bid = bid.Bid(len(self.__issue_size_list))

        # JVMへ接続
        self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
            port=port))
        self.__setting_file_name = setting_file_name
        # Geniusのagentのinit
        self.__agent = self.gateway.entry_point
        self.__agent_num = agent_num
        self.__random_seed = 0
        self.receive_start_negotiation()
Пример #22
0
    def _start_gateway(port=0) -> JavaGateway:
        # launch Java side with dynamic port and get back the port on which the
        # server was bound to.
        hearthstone_jar_path = Context.find_resource_path(
            'hearthstone-0.8.67.jar')
        net_jar_path = Context.find_resource_path('net-0.8.67-all.jar')
        port = launch_gateway(port=port,
                              classpath=os.pathsep.join(
                                  (hearthstone_jar_path, net_jar_path)),
                              die_on_exit=True)

        # connect python side to Java side with Java dynamic port and start python
        # callback server with a dynamic port
        gateway = JavaGateway(
            gateway_parameters=GatewayParameters(port=port, auto_convert=True),
            callback_server_parameters=CallbackServerParameters(port=0))

        # retrieve the port on which the python callback server was bound to.
        python_port = gateway.get_callback_server().get_listening_port()

        # tell the Java side to connect to the python callback server with the new
        # python port. Note that we use the java_gateway_server attribute that
        # retrieves the GatewayServer instance.
        gateway.java_gateway_server.resetCallbackClient(
            gateway.java_gateway_server.getCallbackClient().getAddress(),
            python_port)
        return gateway
Пример #23
0
 def init_py4j_stuff(self):
     self.logger.info("Connecting to Java Gateway on port " +
                      str(self.port))
     self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
         auto_convert=True, port=self.port))
     self.javaInterface = self.gateway.entry_point
     self.logger.info("Connection to Java successful!")
Пример #24
0
def saveAndOpenSelection(df, dsName:str, objectClassIdx:int, selectionName:str, showObjects:bool=False, showTracks:bool=False, openSelection:bool=False, objectClassIdxDisplay:int=-1, interactiveObjectClassIdx:int=-1, port=25335, python_proxy_port:int=25334, address='127.0.0.1', gateway_parameters={}):
    """Stores a selection to bacmman using python gateway (py4j). Bacmman must be running with an active python gateway server.

    Parameters
    ----------
    df : pandas DataFrame
        each line of the DataFrame is one element of the selection, defined by columns Indices & Position
    dsName : str
        bacmman dataset name to store the selection to.
    objectClassIdx : int
        index of the object class of the elements of the selection in the bacmman dataset
    selectionName : str
        name of the selection
    showObjects : bool
        whether contours of objects should be shown
    showTracks : bool
        whether track links of objects should be shown
    openSelection : bool
        whether the first kymograph of the selection should be open
    objectClassIdxDisplay : int
        if openSelection is true, object class idx of the opened kymograph
    interactiveObjectClassIdx : int
        if openSelection is true, interactive object class idx
    python_proxy_port : int
        python port of the java gateway
    """
    gateway = JavaGateway(python_proxy_port=python_proxy_port, gateway_parameters=GatewayParameters(address=address, port=port, **gateway_parameters))
    try:
        idx = ListConverter().convert(df.Indices.tolist(), gateway._gateway_client)
        pos = ListConverter().convert(df.Position.tolist(), gateway._gateway_client)
        gateway.saveCurrentSelection(dsName, objectClassIdx, selectionName, idx, pos, showObjects, showTracks, openSelection, False, objectClassIdxDisplay, interactiveObjectClassIdx)
    except Py4JNetworkError:
        print("Could not connect, is BACMMAN started?")
Пример #25
0
def clusterJava(dataFrame):
    def get_nodes(node, max_depth, depth):
        leafs = []
        children = node.getChildren()
        if children and depth < max_depth:
            for child in children:
                childleafs = get_nodes(child, max_depth, depth+1)
                leafs = leafs + childleafs
        else:
            leafs.append(node)
        return leafs


    dataFrame['cluster'] = None
    from py4j.java_gateway import JavaGateway, GatewayParameters
    gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True))

    grouped = dataFrame.groupby('location')
    locations = grouped.size().to_frame('size')
    locations['x'] = grouped.first()['x']
    locations['y'] = grouped.first()['y']

    locations_flat = locations.reset_index().as_matrix()

    # print('starting java clusterer')
    # cluster_result_json = gateway.entry_point.run(locations_flat)
    # print('java clusterer done')
    # gateway.close()
    # return cluster_result_json
    resp = requests.post(os.environ.get('CLUSTERER_URL'), None, json.dumps(locations_flat.tolist()))
    return resp.text
Пример #26
0
    def __init__(self):
        """Starts a new instance of SystemDSContext, in which the connection to a JVM systemds instance is handled
        Any new instance of this SystemDS Context, would start a separate new JVM.

        Standard out and standard error form the JVM is also handled in this class, filling up Queues,
        that can be read from to get the printed statements from the JVM.
        """
        command = self.__build_startup_command()
        # TODO add an argument parser here
        port = self.__get_open_port()
        command.append(str(port))

        process = self.__try_startup(command)

        # Handle Std out from the subprocess.
        self.__stdout = Queue()
        self.__stderr = Queue()

        Thread(target=self.__enqueue_output,
               args=(process.stdout, self.__stdout),
               daemon=True).start()
        Thread(target=self.__enqueue_output,
               args=(process.stderr, self.__stderr),
               daemon=True).start()

        # Py4j connect to the started process.
        gwp = GatewayParameters(port=port, eager_load=True)
        self.java_gateway = JavaGateway(gateway_parameters=gwp,
                                        java_process=process)
Пример #27
0
async def on_startup(app):
    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16)
    app['thread_pool'] = thread_pool

    port = launch_gateway(
        jarpath='/spark-2.4.0-bin-hadoop2.7/jars/py4j-0.10.7.jar',
        classpath='/spark-2.4.0-bin-hadoop2.7/jars/*:/hail.jar',
        die_on_exit=True)
    gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port),
                          auto_convert=True)
    app['gateway'] = gateway

    hail_pkg = getattr(gateway.jvm, 'is').hail
    app['hail_pkg'] = hail_pkg

    jbackend = hail_pkg.backend.service.ServiceBackend.apply()
    app['jbackend'] = jbackend

    jhc = hail_pkg.HailContext.apply(jbackend, 'hail.log', False, False, 50,
                                     False, 3)
    app['jhc'] = jhc

    app['users'] = set()

    kube.config.load_incluster_config()
    k8s_client = kube.client.CoreV1Api()
    app['k8s_client'] = k8s_client
Пример #28
0
def search(query, port):
	"""Sends the given query string to the running DocFetcher instance at the
	given port and returns a list of result objects.
	
	The result objects provide the following getter methods for accessing their
	attributes:
	- getAuthors
	- getDateStr - e-mail send date
	- getFilename
	- getLastModifiedStr - last-modified date on files
	- getPathStr - file path
	- getScore - result score as int
	- getSender - e-mail sender
	- getSizeInKB - file size as int
	- getTitle
	- getType
	- isEmail - boolean indicating whether result object is e-mail or file
	
	This method will throw an error if communication with the DocFetcher
	instance fails.
	"""
	from py4j.java_gateway import JavaGateway, GatewayParameters
	from py4j.java_gateway import java_import
	
	gateway = JavaGateway(gateway_parameters=GatewayParameters(port=port))
	java_import(gateway.jvm, "net.sourceforge.docfetcher.gui.Application")
	application = gateway.jvm.net.sourceforge.docfetcher.gui.Application
	
	indexRegistry = application.getIndexRegistry()
	searcher = indexRegistry.getSearcher()
	results = searcher.search(query)
	return results
 def test_all_regular_signals_auto_start(self):
     listener = MockListener(self)
     with gateway_example_app_process(None):
         server_started.connect(listener.started)
         gateway = JavaGateway(
             gateway_parameters=GatewayParameters(),
             callback_server_parameters=CallbackServerParameters())
         server_stopped.connect(listener.stopped,
                                sender=gateway.get_callback_server())
         server_connection_started.connect(
             listener.connection_started,
             sender=gateway.get_callback_server())
         server_connection_stopped.connect(
             listener.connection_stopped,
             sender=gateway.get_callback_server())
         pre_server_shutdown.connect(listener.pre_shutdown,
                                     sender=gateway.get_callback_server())
         post_server_shutdown.connect(listener.post_shutdown,
                                      sender=gateway.get_callback_server())
         example = gateway.entry_point.getNewExample()
         impl = IHelloImpl()
         self.assertEqual("This is Hello!", example.callHello(impl))
         gateway.shutdown()
     self.assertEqual(1, listener.received["started"])
     self.assertEqual(1, listener.received["stopped"])
     self.assertEqual(1, listener.received["pre_shutdown"])
     self.assertEqual(1, listener.received["post_shutdown"])
     self.assertEqual(1, listener.received["connection_started"])
     self.assertEqual(1, listener.received["connection_stopped"])
Пример #30
0
    def __init__(self, n=3):
        "Set up initial board configuration."

        gateway = JavaGateway(
            gateway_parameters=GatewayParameters(port=4242),
            callback_server_parameters=CallbackServerParameters())
        manager = gateway.entry_point