def save_cda_files_in_xds(self, file_list): gateway = JavaGateway() xds_connector = gateway.entry_point oid = "1.2.40.0.10.1.4.3.1" root_temp_upload_path = configParser.get('temp-folders', 'upload') result = [] for file in file_list: if CDAEvaluator.evaluate_file_type(CDAEvaluator, file): cda_file = CDAExtractor(file) patient_id = cda_file.get_patient_id() patient_full_name = cda_file.get_patient_name() if not any(obj['patient_id'] == patient_id for obj in result): result.append(self.create_patient_list(self, patient_id, patient_full_name)) patient_list = Patient.objects.filter(patient_id=patient_id, patient_first_name=patient_full_name['vornamen'][0], patient_last_name=patient_full_name['nachname'][0]) if patient_list is None or len(patient_list) == 0: Patient.objects.create(patient_id=patient_id, patient_first_name=patient_full_name['vornamen'][0], patient_last_name=patient_full_name['nachname'][0]) document_id = cda_file.get_document_id() cda_exist = xds_connector.validateNewDocument(oid, str(patient_id), str(document_id)) # TODO: write patient infos into DB if cda_exist is False: file_name = str(patient_id) + '_' + str(document_id) + '.xml' default_storage.save("Django_Server/recruitmenttool/cda_files/tempUpload/" + file_name, file) xds_connector.uploadDocument(oid, str(patient_id), str(document_id), root_temp_upload_path + "/" + file_name) gateway.close() return result
def _initialize_gateway(gateway_address): (host, port) = gateway_address callback_params = CallbackServerParameters(address=host, port=0) gateway = JavaGateway(GatewayClient(address=host, port=port), start_callback_server=True, auto_convert=True, callback_server_parameters=callback_params) try: java_import(gateway.jvm, "org.apache.spark.SparkEnv") java_import(gateway.jvm, "org.apache.spark.SparkConf") java_import(gateway.jvm, "org.apache.spark.api.java.*") java_import(gateway.jvm, "org.apache.spark.api.python.*") java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*") java_import(gateway.jvm, "org.apache.spark.sql.*") java_import(gateway.jvm, "org.apache.spark.sql.hive.*") java_import(gateway.jvm, "scala.Tuple2") java_import(gateway.jvm, "scala.collection.immutable.List") except Py4JError as e: log_error('Error while initializing java gateway: {}'.format(e)) gateway.close() return None log_debug('Java Gateway initialized {}'.format(gateway)) return gateway
def _initialize_gateway(gateway_address): (host, port) = gateway_address callback_params = CallbackServerParameters(address=host, port=0) gateway = JavaGateway(GatewayClient(address=host, port=port), start_callback_server=True, auto_convert=True, callback_server_parameters=callback_params) try: java_import(gateway.jvm, "org.apache.spark.SparkEnv") java_import(gateway.jvm, "org.apache.spark.SparkConf") java_import(gateway.jvm, "org.apache.spark.api.java.*") java_import(gateway.jvm, "org.apache.spark.api.python.*") java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*") java_import(gateway.jvm, "org.apache.spark.sql.*") java_import(gateway.jvm, "org.apache.spark.sql.hive.*") java_import(gateway.jvm, "scala.Tuple2") java_import(gateway.jvm, "scala.collection.immutable.List") except Py4JError as e: log_error('Error while initializing java gateway: {}'.format(e)) gateway.close() return None return gateway
def testCallbackServer(self): # A close is required to stop the thread. gateway = JavaGateway( callback_server_parameters=CallbackServerParameters()) gateway.close() self.assertTrue(True) sleep(2)
def shutdown_gateway(event, gateway: JavaGateway, resource_name: str, shutdown_jvm: bool): if shutdown_jvm: gateway.shutdown() else: gateway.close() logger.info('Py4J gateway (%s) shut down', resource_name)
def get_visualized_cda(request, patient_id, document_id): # cda_file = model.CDAFile.objects.all().filter(cda_id=1234567.1).first() print(BASE_DIR) root_tempDownload_path = configParser.get('temp-folders', 'download') root_tempCache_path = configParser.get('temp-folders', 'cache') cda_file_path = root_tempCache_path + "/" + patient_id + "/" + patient_id + "_" + document_id + ".xml" # try to get CDA File from tempDownload Folder if not path.exists(cda_file_path): cda_file_path = root_tempDownload_path + "/" + patient_id + "/" + patient_id + "_" + document_id + ".xml" # try to download CDA Files from Repository if not path.exists(cda_file_path): gateway = JavaGateway() xds_connector = gateway.entry_point oid = "1.2.40.0.10.1.4.3.1" cda_file_path = xds_connector.queryDocumentWithId( oid, patient_id, document_id) gateway.close() if cda_file_path == "NO_DOCUMENT_FOUND": return Response("NO DOCUMENT FOUND", status=status.HTTP_400_BAD_REQUEST) stylesheet_path = configParser.get('xml-to-html-files', 'xsl-file') cda_html = CDATransformer.transform_xml_to_xsl(CDATransformer, cda_file_path, stylesheet_path) return HttpResponse(cda_html)
class MySm(object): def __init__(self): self.gateway = JavaGateway( gateway_parameters=GatewayParameters(port=25666)) # 连接Java网关 self.sm = self.gateway.entry_point.getSm() def __del__(self): self.gateway.close()
def setup(self, config): self.risk_threshold = self.config["tuning_config"]["risk_threshold"] self.max_concurrent_trials = self.config["tuning_config"][ "max_concurrent_trials"] self.payoff_empirical_min = self.config["tuning_config"][ "obj_fnc_params"]["payoff_empirical_min"] self.payoff_empirical_max = self.config["tuning_config"][ "obj_fnc_params"]["payoff_empirical_max"] self.min_eval_eps = self.config["tuning_config"]["eval_eps_min"] self.max_eval_eps = self.config["tuning_config"]["eval_eps_max"] self.lock = threading.Lock() self.train_results = queue.Queue() self.eval_results = queue.Queue() self.java_gateway = None self.my_id = self.trial_id self.start_time = time.time() self.am_stopped = False self.cur_max_score = self.payoff_empirical_min self.cur_max_train_score = self.payoff_empirical_min default_gateway = JavaGateway() while self.java_gateway is None: for i in range(self.max_concurrent_trials): try: self.my_port = 25334 + 2 * i java_port = self.my_port - 1 # if addJavaPort fails, it means it is being used by some other trial running # in a different process if not default_gateway.addJavaPort(java_port): if i == self.max_concurrent_trials - 1: time.sleep(0.5) continue gateway_params = GatewayParameters( port=java_port, enable_memory_management=False) callback_params = CallbackServerParameters( port=self.my_port) self.java_gateway = JavaGateway( gateway_parameters=gateway_params, callback_server_parameters=callback_params) break except: default_gateway.removeJavaPort(self.my_port - 1) continue default_gateway.close(keep_callback_server=True, close_callback_server_connections=False) encoded_trial_params = TuningParser.encode_trial_params(self.config) self.results_receiver = _TrialResultsReceiver(self) self.java_gateway.entry_point.runTrial(self.trial_id, encoded_trial_params, self.results_receiver) self.additional_setup()
def check_connection(gateway_parameters=None): test_gateway = JavaGateway(gateway_parameters=gateway_parameters) try: # Call a dummy method just to make sure we can connect to the JVM test_gateway.jvm.System.currentTimeMillis() except Py4JNetworkError: # We could not connect. Let"s wait a long time. # If it fails after that, there is a bug with our code! sleep(2) finally: test_gateway.close()
def test_gateway_connection(): test_gateway = JavaGateway() try: # Call a dummy method just to make sure we can connect to the JVM test_gateway.jvm.System.lineSeparator() except Py4JNetworkError: # We could not connect. Let"s wait a long time. # If it fails after that, there is a bug with our code! sleep(2) finally: test_gateway.close()
def tearDownClass(cls): gateway = JavaGateway() workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\ getWorkspace() root = workspace.getRoot() pm = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor() project1 = root.getProject('project1core3.0') project1.delete(True, True, pm) time.sleep(1) gateway.close() stop_eclipse() clean_test_dir()
def stop_eclipse(): gateway = JavaGateway() try: gateway.entry_point.closeEclipse() time.sleep(1) gateway.shutdown() except Exception: pass try: gateway.close() except Exception: pass
def download_all_files_from_patient(patient_id): # download all file from XDS # TODO: check if cda is already in temp = cache gateway = JavaGateway() xds_connector = gateway.entry_point oid = "1.2.40.0.10.1.4.3.1" print("Patient Files get processed from: " + str(patient_id)) xds_connector.downloadPatientFiles(oid, str(patient_id)) gateway.close() return glob.glob( configParser.get('temp-folders', 'download') + "/" + str( patient_id) + "/*.xml")
def collect_trajectories(actor, critic, port, game_num, p2, rnn, n_frame): logger.info(f'start fight with {p2}') logger.info(f'game_num value {game_num}') error = True while error: gateway = JavaGateway( gateway_parameters=GatewayParameters(port=port), callback_server_parameters=CallbackServerParameters()) try: manager = gateway.entry_point current_time = int(time.time() * 1000) # register AIs collect_data_helper = CollectDataHelper(logger) agent = SoundAgent(gateway, actor=actor, critic=critic, collect_data_helper=collect_data_helper, logger=logger, n_frame=n_frame, rnn=rnn) sandbox_agent = SandboxAgent(gateway) manager.registerAI(f'SoundAgent', agent) manager.registerAI('Sandbox', sandbox_agent) game = manager.createGame('ZEN', 'ZEN', 'SoundAgent', p2, game_num) # start game manager.runGame(game) # finish game logger.info('Finish game') sys.stdout.flush() # close gateway gateway.close_callback_server() gateway.close() error = False except Exception as ex: print(ex) logger.info('There is an error with the gateway, restarting') gateway.close_callback_server() gateway.close() error = True # return agent.get_trajectories_data() agent_data = process_game_agent_data(actor, critic, agent.collect_data_helper, rnn) # try: # kill_proc_tree(java_env.pid, False) # except: # print('kill process') # agent.reset() return agent_data
def load_from_envs() -> HDFSWrapperBase: isReady = False while isReady == False: print("Testando conexão com a JVM") try: java_gateway = JavaGateway(eager_load=True) isReady = True print("Java pronto para atender novas requisições") java_gateway.close() except Exception as e: print( "Não foi possível conectar ao Java, tentando novamente em 2 segundos" ) time.sleep(2) return HDFSWrapperJava()
def load_into_hive(table_name, csv_path, hdfs_path, hdfs_path_dir, fild): gateway = None try: gateway = JavaGateway() ff = gateway.jvm.java.util.ArrayList() for k in fild: ff.append(k) hive_util = gateway.entry_point.getHiveUtil() func = hive_util.uploadToHive func(table_name, csv_path, hdfs_path, hdfs_path_dir, ff) except Exception as e: raise e finally: if gateway is not None: gateway.close()
def check_eclipse(): '''Check that Eclipse is started and that recodoc can communicate with it.''' gateway = JavaGateway() try: success = gateway.entry_point.getServer().getListeningPort() > 0 except Exception: success = False if success: print('Connection to Eclipse: OK') else: print('Connection to Eclipse: ERROR') gateway.close() return success
def download_report(f, t): gateway = None try: gateway = JavaGateway() ff = gateway.jvm.java.util.ArrayList() tt = gateway.jvm.java.util.ArrayList() for k in f: ff.append(k) for k in t: tt.append(k) hive_util = gateway.entry_point.getHiveUtil() func = hive_util.downloadFiles func(ff, tt) except Exception as e: raise e finally: if gateway is not None: gateway.close()
def testLinkEclipseProject(self): create_code_local('project1', 'core', '3.0') to_path = get_codebase_path('project1', 'core', '3.0') to_path = os.path.join(to_path, 'src') os.rmdir(to_path) from_path = os.path.join(settings.TESTDATA, 'testproject1', 'src') shutil.copytree(from_path, to_path) link_eclipse('project1', 'core', '3.0') gateway = JavaGateway() workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\ getWorkspace() root = workspace.getRoot() pm = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor() project1 = root.getProject('project1core3.0') self.assertIsNotNone(project1) project1.delete(True, True, pm) time.sleep(1) gateway.close()
def py4j_common_hive_util(func, *param): gateway = None try: gateway = JavaGateway() hive_util = gateway.entry_point.getHiveUtil() func = getattr(hive_util, func) result = func(*param) except Py4JNetworkError as e: response = Response.fail(ERRORS.PY4J_CONNECTION_ERROR, None) return HttpResponse(response.to_json()) except Py4JJavaError as e: # 查询出错,返回错误信息 response = Response.fail(ERRORS.HIVE_QUERY_ERROR, e.java_exception.getMessage()) return HttpResponse(response.to_json()) finally: if gateway is not None: gateway.close() return result
def link_eclipse(pname, bname, release): '''Add the Java Project created with create_code_local to the Eclipse workspace.''' project_key = pname + bname + release codebase_path = get_codebase_path(pname, bname, release) gateway = JavaGateway() workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\ getWorkspace() root = workspace.getRoot() path = gateway.jvm.org.eclipse.core.runtime.Path(os.path.join( codebase_path, PROJECT_FILE)) project_desc = workspace.loadProjectDescription(path) new_project = root.getProject(project_key) nmonitor = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor() # gateway.jvm.py4j.GatewayServer.turnLoggingOn() # To avoid workbench problem (don't know why it needs some time). time.sleep(1) new_project.create(project_desc, nmonitor) new_project.open(nmonitor) gateway.close()
def link_eclipse(pname, bname, release): '''Add the Java Project created with create_code_local to the Eclipse workspace.''' project_key = pname + bname + release codebase_path = get_codebase_path(pname, bname, release) gateway = JavaGateway() workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\ getWorkspace() root = workspace.getRoot() path = gateway.jvm.org.eclipse.core.runtime.Path( os.path.join(codebase_path, PROJECT_FILE)) project_desc = workspace.loadProjectDescription(path) new_project = root.getProject(project_key) nmonitor = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor() # gateway.jvm.py4j.GatewayServer.turnLoggingOn() # To avoid workbench problem (don't know why it needs some time). time.sleep(1) new_project.create(project_desc, nmonitor) new_project.open(nmonitor) gateway.close()
def prepare_test_data(request): gateway = JavaGateway() xds_connector = gateway.entry_point oid = "1.2.40.0.10.1.4.3.1" cda_test_files = glob.glob(configParser.get('demo-files', 'cda-files')) for file in cda_test_files: cda_file = CDAExtractor(file) patient_id = cda_file.get_patient_id() patient_full_name = cda_file.get_patient_name() patient_list = Patient.objects.filter(patient_id=patient_id) if patient_list is None or len(patient_list) == 0: Patient.objects.create( patient_id=patient_id, patient_first_name=patient_full_name['vornamen'][0], patient_last_name=patient_full_name['nachname'][0]) document_id = cda_file.get_document_id() cda_exist = xds_connector.validateNewDocument(oid, str(patient_id), str(document_id)) # TODO: write patient infos into DB if cda_exist is False: xds_connector.uploadDocument(oid, str(patient_id), str(document_id), BASE_DIR + file) gateway.close() return Response("TEST DATA UPLAODED", status=status.HTTP_201_CREATED)
def imageProcessor(image, filterRequested): decidedImage = image #(Step 1) Given an image name, if it fails to find the image, error must be raised. if not os.path.isfile(image): raise IOError #Don't filter out jpg. They are already compressed enough! ext = os.path.splitext(image)[-1].lower() if ext == ".jpg": filterRequested = False; #(Step 2) Using OCR, verify whether the image is for the advertisement purpose or not. score = 0 advertisement = False #If there is a brand name, it is possible that it's going to be the advertisement if Detect_Logo(image): score += 1 score += contentVerify(image) #print(score) #If the score reaches the threshold, then consider it as an advertisement if score >= 10: advertisement = True #(Step 3) Using Java, Convert image from original to Grayscale without modifying the original image if filterRequested and advertisement: gateway = JavaGateway() convertImage = gateway.entry_point convertImage.runImageProcessing(image) decidedImage = 'Filtered.png' gateway.close() #based on the "filterRequest" Boolean, send the name of original/updated image return decidedImage
class LocalJvmBridge(object): _bridge = None def __init__(self): self.process = None self.gateway = None self.app = None self.port = 0 @classmethod def inst(cls): if cls._bridge is None: cls._bridge = LocalJvmBridge() cls._bridge.init() return cls._bridge def init(self): # noinspection PyProtectedMember from .config import g_config debug_mode = g_config["debug_mode"] # redirect stdout if in debug_mode # Note: if enableLazyXXX is used in Pipeline stages, Java side will print stuff. # This case is very difficult to handle, so do not set redirect_stdout to True. redirect_stdout = sys.stdout if debug_mode else None # redirect stderr if in debug_mode and not in IPython (will fail) redirect_stderr = sys.stderr if debug_mode and not in_ipython( ) else None try: # noinspection PyUnresolvedReferences from pyflink.pyflink_gateway_server import construct_log_settings if os.name != 'nt': try: from pyflink.pyflink_gateway_server import prepare_environment_variables env = dict(os.environ) prepare_environment_variables(env) log_settings = construct_log_settings(env) except TypeError: # noinspection PyArgumentList log_settings = construct_log_settings() else: log_settings = [] except ImportError: log_settings = [] javaopts = g_config['java_opts'].copy() javaopts += log_settings if os.name == 'nt': # Limit Java side to only detect and use Java implementation of BLAS/LAPACK/ARPACK. # Java process started through py4j will stuck when trying to load native libraries. # See https://github.com/bartdag/py4j/issues/444 javaopts += [ "-Dcom.github.fommil.netlib.BLAS=com.github.fommil.netlib.F2jBLAS", "-Dcom.github.fommil.netlib.LAPACK=com.github.fommil.netlib.F2jLAPACK", "-Dcom.github.fommil.netlib.ARPACK=com.github.fommil.netlib.F2jARPACK" ] if g_config['gateway_port'] is not None: self.port = int(g_config['gateway_port']) else: self.port = launch_gateway(port=0, javaopts=javaopts, die_on_exit=True, daemonize_redirect=True, redirect_stdout=redirect_stdout, redirect_stderr=redirect_stderr, classpath=os.pathsep.join( list_all_jars())) print('JVM listening on 127.0.0.1:{}'.format(self.port)) self.gateway = JavaGateway( gateway_parameters=GatewayParameters(port=self.port, auto_field=True, auto_convert=True), callback_server_parameters=CallbackServerParameters( port=0, daemonize=True, daemonize_connections=True), start_callback_server=False) callback_server_port = self.gateway.get_callback_server( ).get_listening_port() self.gateway.java_gateway_server.resetCallbackClient( self.gateway.java_gateway_server.getCallbackClient().getAddress(), callback_server_port) def close(self): self.gateway.close(keep_callback_server=True) self._bridge = None
class FightingiceEnv_Display_NoFrameskip(gym.Env): metadata = {'render.modes': ['human']} def __init__(self, **kwargs): self.freq_restart_java = 3 self.java_env_path = os.getcwd() if "java_env_path" in kwargs.keys(): self.java_env_path = kwargs["java_env_path"] if "freq_restart_java" in kwargs.keys(): self.freq_restart_java = kwargs["freq_restart_java"] if "port" in kwargs.keys(): self.port = kwargs["port"] else: try: import port_for self.port = port_for.select_random( ) # select one random port for java env except: raise ImportError( "Pass port=[your_port] when make env, or install port_for to set startup port automatically, maybe pip install port_for can help" ) _actions = "AIR AIR_A AIR_B AIR_D_DB_BA AIR_D_DB_BB AIR_D_DF_FA AIR_D_DF_FB AIR_DA AIR_DB AIR_F_D_DFA AIR_F_D_DFB AIR_FA AIR_FB AIR_GUARD AIR_GUARD_RECOV AIR_RECOV AIR_UA AIR_UB BACK_JUMP BACK_STEP CHANGE_DOWN CROUCH CROUCH_A CROUCH_B CROUCH_FA CROUCH_FB CROUCH_GUARD CROUCH_GUARD_RECOV CROUCH_RECOV DASH DOWN FOR_JUMP FORWARD_WALK JUMP LANDING NEUTRAL RISE STAND STAND_A STAND_B STAND_D_DB_BA STAND_D_DB_BB STAND_D_DF_FA STAND_D_DF_FB STAND_D_DF_FC STAND_F_D_DFA STAND_F_D_DFB STAND_FA STAND_FB STAND_GUARD STAND_GUARD_RECOV STAND_RECOV THROW_A THROW_B THROW_HIT THROW_SUFFER" action_strs = _actions.split(" ") self.observation_space = spaces.Box(low=0, high=1, shape=(96, 64, 1)) self.action_space = spaces.Discrete(len(action_strs)) os_name = platform.system() if os_name.startswith("Linux"): self.system_name = "linux" elif os_name.startswith("Darwin"): self.system_name = "macos" else: self.system_name = "windows" if self.system_name == "linux": # first check java can be run, can only be used on Linux java_version = subprocess.check_output( 'java -version 2>&1 | awk -F[\\\"_] \'NR==1{print $2}\'', shell=True) if java_version == b"\n": raise ModuleNotFoundError("Java is not installed") else: print("Please make sure you can run java if you see some error") # second check if FightingIce is installed correct start_jar_path = os.path.join(self.java_env_path, "FightingICE.jar") start_data_path = os.path.join(self.java_env_path, "data") start_lib_path = os.path.join(self.java_env_path, "lib") lwjgl_path = os.path.join(start_lib_path, "lwjgl", "*") lib_path = os.path.join(start_lib_path, "*") start_system_lib_path = os.path.join(self.java_env_path, "lib", "natives", self.system_name) natives_path = os.path.join(start_system_lib_path, "*") if os.path.exists(start_jar_path) and os.path.exists( start_data_path) and os.path.exists( start_lib_path) and os.path.exists(start_system_lib_path): pass else: error_message = "FightingICE is not installed in your script launched path {}, set path when make() or start script in FightingICE path".format( self.java_env_path) raise FileExistsError(error_message) self.java_ai_path = os.path.join(self.java_env_path, "data", "ai") ai_path = os.path.join(self.java_ai_path, "*") if self.system_name == "windows": self.start_up_str = "{};{};{};{};{}".format( start_jar_path, lwjgl_path, natives_path, lib_path, ai_path) self.need_set_memory_when_start = True else: self.start_up_str = "{}:{}:{}:{}:{}".format( start_jar_path, lwjgl_path, natives_path, lib_path, ai_path) self.need_set_memory_when_start = False self.game_started = False self.round_num = 0 def _start_java_game(self): # start game print("Start java env in {} and port {}".format( self.java_env_path, self.port)) devnull = open(os.devnull, 'w') if self.system_name == "windows": # -Xms1024m -Xmx1024m we need set this in windows self.java_env = subprocess.Popen([ "java", "-Xms1024m", "-Xmx1024m", "-cp", self.start_up_str, "Main", "--port", str(self.port), "--py4j", "--fastmode", "--grey-bg", "--inverted-player", "1", "--mute", "--limithp", "400", "400" ], stdout=devnull, stderr=devnull) elif self.system_name == "linux": self.java_env = subprocess.Popen([ "java", "-cp", self.start_up_str, "Main", "--port", str(self.port), "--py4j", "--fastmode", "--grey-bg", "--inverted-player", "1", "--mute", "--limithp", "400", "400" ], stdout=devnull, stderr=devnull) elif self.system_name == "macos": self.java_env = subprocess.Popen([ "java", "-XstartOnFirstThread", "-cp", self.start_up_str, "Main", "--port", str(self.port), "--py4j", "--fastmode", "--grey-bg", "--inverted-player", "1", "--mute", "--limithp", "400", "400" ], stdout=devnull, stderr=devnull) # self.java_env = subprocess.Popen(["java", "-cp", "/home/myt/gym-fightingice/gym_fightingice/FightingICE.jar:/home/myt/gym-fightingice/gym_fightingice/lib/lwjgl/*:/home/myt/gym-fightingice/gym_fightingice/lib/natives/linux/*:/home/myt/gym-fightingice/gym_fightingice/lib/*", "Main", "--port", str(self.free_port), "--py4j", "--c1", "ZEN", "--c2", "ZEN","--fastmode", "--grey-bg", "--inverted-player", "1", "--mute"]) # sleep 3s for java starting, if your machine is slow, make it longer time.sleep(3) def _start_gateway(self, p2=Machete): # auto select callback server port and reset it in java env self.gateway = JavaGateway( gateway_parameters=GatewayParameters(port=self.port), callback_server_parameters=CallbackServerParameters(port=0)) python_port = self.gateway.get_callback_server().get_listening_port() self.gateway.java_gateway_server.resetCallbackClient( self.gateway.java_gateway_server.getCallbackClient().getAddress(), python_port) self.manager = self.gateway.entry_point # create pipe between gym_env_api and python_ai for java env server, client = Pipe() self.pipe = server self.p1 = GymAIDisplay(self.gateway, client, False) self.manager.registerAI(self.p1.__class__.__name__, self.p1) if isinstance(p2, str): # p2 is a java class name self.p2 = p2 self.game_to_start = self.manager.createGame( "ZEN", "ZEN", self.p1.__class__.__name__, self.p2, self.freq_restart_java) else: # p2 is a python class self.p2 = p2(self.gateway) self.manager.registerAI(self.p2.__class__.__name__, self.p2) self.game_to_start = self.manager.createGame( "ZEN", "ZEN", self.p1.__class__.__name__, self.p2.__class__.__name__, self.freq_restart_java) self.game = Thread(target=game_thread, name="game_thread", args=(self, )) self.game.start() self.game_started = True self.round_num = 0 def _close_gateway(self): self.gateway.close_callback_server() self.gateway.close() del self.gateway def _close_java_game(self): self.java_env.kill() del self.java_env self.pipe.close() del self.pipe self.game_started = False def reset(self, p2=Machete): # start java game if game is not started if self.game_started is False: try: self._close_gateway() self._close_java_game() except: pass self._start_java_game() self._start_gateway(p2) # to provide crash, restart java game in some freq if self.round_num == self.freq_restart_java * 3: # 3 is for round in one game try: self._close_gateway() self._close_java_game() self._start_java_game() except: raise SystemExit("Can not restart game") self._start_gateway(p2) # just reset is anything ok self.pipe.send("reset") self.round_num += 1 obs = self.pipe.recv() return obs def step(self, action): # check if game is running, if not try restart # when restart, dict will contain crash info, agent should do something, it is a BUG in this version if self.game_started is False: dict = {} dict["pre_game_crashed"] = True return self.reset(), 0, None, dict self.pipe.send(["step", action]) if self.pipe.poll(5): message = self.pipe.recv() new_obs, reward, done, dict = message else: new_obs, reward = self.p1.get_obs(), self.p1.get_reward() dict = {} dict["no_data_receive"] = True logging.warning( "server can not receive, request to reset the game") return new_obs, reward, True, dict return new_obs, reward, done, dict def render(self, mode='human'): # no need pass def close(self): if self.game_started: self._close_java_game()
class JavaSnippetParser(object): HANDLE_SEPARATOR = ":" TYPE_KIND = "T" METHOD_KIND = "M" FIELD_KIND = "F" ANNOTATION_KIND = "A" ANNOTATION_PARAMETER_KIND = "P" ENUM_KIND = "E" ENUM_VALUE_KIND = "V" REQUEST_NAME = 'recodoc_request' def __init__(self, project, source): self.project = project self.source = source self._load() def _load(self): self.gateway = JavaGateway(start_callback_server=False) self.PPACoreUtil = \ self.gateway.jvm.ca.mcgill.cs.swevo.ppa.util.PPACoreUtil self.class_kind = CodeElementKind.objects.get(kind='class') self.unknown_kind = CodeElementKind.objects.get(kind='unknown') self.method_kind = CodeElementKind.objects.get(kind='method') self.field_kind = CodeElementKind.objects.get(kind='field') self.enumeration_kind = CodeElementKind.objects.get(kind='enumeration') self.annotation_kind = CodeElementKind.objects.get(kind='annotation') def parse(self, progress_monitor=NullProgressMonitor()): options = self.gateway.jvm.ca.mcgill.cs.swevo.ppa.PPAOptions() mcodes = CodeSnippet.objects.filter(language='j').\ filter(project=self.project).\ filter(source=self.source) #mcodes = CodeSnippet.objects.filter(pk=122170) count = mcodes.count() progress_monitor.start('Parsing Code Snippets ({0})' .format(count), count) for mcode in mcodes: # Don't parse snippets that are already parsed. if mcode.single_code_references.count() > 0: progress_monitor.word('Skipped a parsed code snippet', 1) continue text = mcode.snippet_text if text != None and text != '': try: cu = self._get_cu(text, options) if cu != None: visitor = self.gateway.jvm.ca.mcgill.cs.swevo.ppa.\ util.NameMapVisitor(True, True, True) cu.accept(visitor) self._process_name( visitor.getBindings(), visitor.getDeclarations(), visitor.getNodes(), mcode) except Exception: logger.exception( 'Error with snippet: PK: {0}. Text:\n\n: {1}' .format(mcode.pk, text)) finally: try: self.PPACoreUtil.cleanUpAll(self.REQUEST_NAME) except Exception: print_exc() else: print('issue!') gc.collect() progress_monitor.work('Parsed a Code Snippet: pk={0}' .format(mcode.pk), 1) progress_monitor.done() self.gateway.close() def _get_cu(self, text, options): try: if is_cu_body(text): cu = self.PPACoreUtil.getCU(text, options, self.REQUEST_NAME) elif is_class_body(text): cu = self.PPACoreUtil.getSnippet(text, options, True, self.REQUEST_NAME) else: cu = self.PPACoreUtil.getSnippet(text, options, False, self.REQUEST_NAME) except Exception: cu = None logger.exception('Error while processing cu for {0}'.format(text)) return cu def _process_name(self, bindings, declarations, nodes, mcode): for i, binding in enumerate(bindings): declaration = declarations[i] parts = binding.split(self.HANDLE_SEPARATOR) kind = parts[0][2] kind_hint = None content = binding if kind == self.TYPE_KIND: kind_hint = self.class_kind elif kind == self.METHOD_KIND: kind_hint = self.method_kind elif kind == self.FIELD_KIND: kind_hint = self.field_kind elif kind == self.ANNOTATION_KIND: kind_hint = self.annotation_kind elif kind == self.ENUM_KIND: kind_hint = self.enumeration_kind else: logger.warning('Not a high level code reference {0}'.\ format(kind)) if kind_hint is not None: code = SingleCodeReference( content=content, xpath=mcode.xpath, file_path=mcode.file_path, kind_hint=kind_hint, declaration=declaration, snippet=mcode, source=mcode.source, index=(mcode.index * -1000) - i) code.project = self.project if mcode.project_release is not None: code.project_release = mcode.project_release code.local_context = mcode.local_context code.mid_context = mcode.mid_context code.global_context = mcode.global_context code.save()
class PowerDynSimEnv(gym.Env): metadata = {} _case_files = "" _dyn_sim_config_file = "" _rl_config_file = "" default_time_step = 0.1 action_type = 'discrete' a_gateway = None # define InterPSS dynamic simulation service ipss_app = None def __init__(self, case_files, dyn_sim_config_file, rl_config_file, server_port_num=25333): # change from global to class-level variable to support parallel process #global gateway self.a_gateway = JavaGateway(gateway_parameters=GatewayParameters( port=server_port_num, auto_convert=True)) #global ipss_app self.ipss_app = self.a_gateway.entry_point from gym import spaces _case_files = transfer2JavaStringAry(self.a_gateway, case_files) #initialize the power system simulation service # {observation_history_length,observation_space_dim, action_location_num, action_level_num}; dim_ary = self.ipss_app.initStudyCase(_case_files, dyn_sim_config_file, rl_config_file) observation_history_length = dim_ary[0] observation_space_dim = dim_ary[1] # set agent-environment interaction time step, self.time_step = self.ipss_app.getEnvTimeStep( ) if self.ipss_app.getEnvTimeStep() > 0 else self.default_time_step self.action_type = self.ipss_app.getActionSpaceType() print('action type = ', self.action_space) # for discrete action space #define action and observation spaces """ if(action_location_num == 1): self.action_space = spaces.Discrete(action_level_num) # Discrete, 1-D dimension else: #print('N-D dimension Discrete Action space is not supported it yet...TODO') # the following is based on the latest gym dev version # action_def_vector = np.ones(action_location_num, dtype=np.int32)*action_level_num # for gym version 0.10.4, it is parametrized by passing an array of arrays containing [min, max] for each discrete action space # for exmaple, MultiDiscrete([ [0,4], [0,1], [0,1] ]) action_def_vector = np.ones((action_location_num,2),dtype=np.int32) action_def_vector[:,1] = action_level_num -1 aa = np.asarray(action_def_vector, dtype=np.int32) self.action_space = spaces.MultiDiscrete(action_def_vector) # Discrete, N-D dimension """ if self.action_type.lower() == 'discrete': print( 'observation_history_length,observation_space_dim, action_location_num, action_level_num = ' ) print(dim_ary[0], dim_ary[1], dim_ary[2], dim_ary[3]) action_location_num = dim_ary[2] action_level_num = dim_ary[3] action_num = action_level_num**action_location_num self.action_space = spaces.Discrete(action_num) self.cnts = np.ones(action_location_num) * action_level_num elif self.action_type.lower() == 'continuous': print( 'observation_history_length,observation_space_dim, action_location_num = ' ) print(dim_ary[0], dim_ary[1], dim_ary[2]) action_ranges = transfer2DJavaArray2NumpyArray( self.ipss_app.getActionValueRanges()) print('action value ranges = ', action_ranges) low = action_ranges[:, 0] high = action_ranges[:, 1] print('action range low =', low, 'action range high =', high) self.action_space = spaces.Box(low, high, dtype=action_ranges.dtype) print(self.action_space) self.observation_space = spaces.Box( -999, 999, shape=(observation_history_length * observation_space_dim, )) # Continuous self.seed() #TOOD get the initial states self.state = None self.steps_beyond_done = None self.restart_simulation = True def seed(self, seed=None): self.np_random, seed = seeding.np_random(seed) return [seed] def step(self, action): assert self.action_space.contains( action), "%r (%s) invalid" % (action, type(action)) # step-1 convert from Gym discrete actions into actions applied to power system simulator actionMapped = None if self.action_type == 'discrete': actionMapped = refer_new(action, self.cnts) elif self.action_type == 'continuous': actionMapped = np.asarray(action) #print(actionMapped) actionPyAry = np.asarray(actionMapped, dtype=np.float64) # print(actionPyAry, 'len = ', actionPyAry.size) # np array size = number of elements in the array actionJavaAry = self.a_gateway.new_array(self.a_gateway.jvm.double, actionPyAry.size) if (actionPyAry.size == 1): actionJavaAry[0] = float(action) else: i = 0 for x in actionPyAry: actionJavaAry[i] = x i = i + 1 # step-2 apply the actions to the simulator and run the simulation for one interaction step forward self.ipss_app.nextStepDynSim(self.time_step, actionJavaAry, self.action_type) # step-3 retrieve the state from InterPSS simulation service # observations is a Java_Collections array observations = self.ipss_app.getEnvObversations() # convert it from Java_collections array to native Python array self.state = transfer2DJavaArray2NumpyArray(observations) # print('observation shape: ', np.shape(self.state)) # step-4 check the states to see whether it go beyond the limits done = self.ipss_app.isSimulationDone() if not done: reward = self.ipss_app.getReward() elif self.steps_beyond_done is None: self.steps_beyond_done = 0 reward = self.ipss_app.getReward( ) # even it is done, ipss_app would calculate and return a corresponding reward else: if self.steps_beyond_done == 0: logger.warning( "You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior." ) self.steps_beyond_done += 1 reward = 0.0 return np.array(self.state).ravel(), reward, done, {} def reset(self): # reset need to randomize the operation state and fault location, and fault time study_cases = self.ipss_app.getStudyCases() total_case_num = len(study_cases) if total_case_num == 0: total_case_num = 1 case_Idx = np.random.randint(0, total_case_num) # an integer total_fault_buses = len(self.ipss_app.getFaultBusCandidates()) fault_bus_idx = np.random.randint( 0, total_fault_buses ) # an integer, in the range of [0, total_bus_num-1] #fault_bus_idx = 3 # an integer, in the range of [0, total_bus_num-1] #fault_start_time =random.uniform(0.99, 1.01) # a double number, in the range of [0.2, 1] fault_start_time_ary = transfer1DJavaArray2NumpyArray( self.ipss_app.getFaultStartTimeCandidates()) fault_start_time = fault_start_time_ary[np.random.randint( 0, len(fault_start_time_ary))] ftd_candidates = transfer1DJavaArray2NumpyArray( self.ipss_app.getFaultDurationCandidates()) fault_duation_time = ftd_candidates[np.random.randint( 0, len(ftd_candidates) )] # a double number, in the range of [0.08, 0.4] # reset initial state to states of time = 0, non-fault self.ipss_app.reset(case_Idx, fault_bus_idx, fault_start_time, fault_duation_time) #self.state = None # observations is a Java_Collections array observations = self.ipss_app.getEnvObversations() # convert it from Java_collections array to native Python array self.state = transfer2DJavaArray2NumpyArray(observations) #print(self.state) self.steps_beyond_done = None self.restart_simulation = True return np.array(self.state).ravel() # init the system with a specific state and fault def validate(self, case_Idx, fault_bus_idx, fault_start_time, fault_duation_time): total_bus_num = self.ipss_app.getTotalBusNum() self.ipss_app.reset(case_Idx, fault_bus_idx, fault_start_time, fault_duation_time) # observations is a Java_Collections array observations = self.ipss_app.getEnvObversations() # convert it from Java_collections array to native Python array self.state = transfer2DJavaArray2NumpyArray(observations) self.steps_beyond_done = None self.restart_simulation = True return np.array(self.state).ravel() def close_connection(self): self.a_gateway.close(keep_callback_server=True, close_callback_server_connections=False)
class JavaParser(object): '''Parses a Java codebase and creates the appropriate CodeElement. This parser uses multiple threads to speed up the parsing. This parser requires access to Eclipse/Py4J''' JAVA_SRC_FOLDER = 'src' def __init__(self, codebase, project_key, opt_input): ''' :param project_key: The name of the project in the Eclipse workspace. :param codebase: The codebase instance to which the CodeElement will be associated with. :param opt_input: Optional input. Not used by this parser. ''' self.project_name = project_key self.gateway = JavaGateway() self.hierarchies = deque() # list of tuples. [(parent, child, child)] self.codebase = codebase self.queue = Queue() self.package_kind = CodeElementKind.objects.get(kind='package') self.ASTParser = self.gateway.jvm.org.eclipse.jdt.core.dom.ASTParser self.JLS3 = self.gateway.jvm.org.eclipse.jdt.core.dom.AST.JLS3 def _get_package_root(self): ResourcePlugin = self.gateway.jvm.org.eclipse.core.resources.\ ResourcesPlugin workspaceRoot = ResourcePlugin.getWorkspace().getRoot() project = workspaceRoot.getProject(self.project_name) java_project = self.gateway.jvm.org.eclipse.jdt.core.JavaCore.\ create(project) src_folder = project.getFolder(JavaParser.JAVA_SRC_FOLDER) proot = java_project.getPackageFragmentRoot(src_folder) return proot def _parse_packages(self, proot): packages = [] for package in proot.getChildren(): if package.hasChildren(): package_name = package.getElementName() package_code_element = CodeElement(codebase=self.codebase, simple_name=package_name, fqn=package_name, eclipse_handle=package.getHandleIdentifier(), kind=self.package_kind, parser=JAVA_PARSER) package_code_element.save() packages.append((package, package_code_element)) return packages def parse(self, progress_monitor=NullProgressMonitor()): '''Parses the codebase and creates CodeElement instances. :progress_monitor: A progress monitor to track the parsing progress. ''' proot = self._get_package_root() packages = self._parse_packages(proot) progress_monitor.start('Parsing Java Project', len(packages)) # Start workers: for _ in xrange(0, PARSER_WORKER): worker = CUWorker(self.queue, self.codebase, self.hierarchies, self.gateway, progress_monitor) worker.start() start = time.time() for (package, package_code_element) in packages: gc.collect() # for Py4J cunits = package.getCompilationUnits() unit_length = float(len(cunits)) for cunit in cunits: ast_parser = self.ASTParser.newParser(self.JLS3) ast_parser.setResolveBindings(True) ast_parser.setSource(cunit) cu = ast_parser.createAST(None) winput = (cu, package_code_element, cunit.getElementName(), 1.0 / unit_length) self.queue.put(winput) for _ in xrange(0, PARSER_WORKER): self.queue.put(None) progress_monitor.info('Done parsing packages. Waiting for CUs.') self.queue.join() progress_monitor.done() self.gateway.close() print('Time: ' + str(time.time() - start)) self.parse_hierarchy(progress_monitor) def parse_hierarchy(self, progress_monitor=NullProgressMonitor()): '''Builds the hierarchy of the parsed CodeElement instances. Must be called *after* parse. :param progress_monitor: ''' queue = Queue() for hierarchy in self.hierarchies: queue.put(hierarchy) progress_monitor.start('Parsing Java Hierarchy', len(self.hierarchies)) start = time.time() for _ in xrange(0, HIERARCHY_WORKER): # Sentinel value queue.put(None) worker = HierarchyWorker(queue, self.codebase, progress_monitor) worker.start() queue.join() progress_monitor.done() self.gateway.close() print('Time: ' + str(time.time() - start))
def testCallbackServer(self): # A close is required to stop the thread. gateway = JavaGateway(start_callback_server=True) gateway.close() self.assertTrue(True) time.sleep(1)
class Py4jServiceBridge(object): '''Py4jServiceBridge class This class provides and API for consumers to use the Py4jServiceBridge. This allows a bridge between Python and the OSGi service registry. ''' def __init__(self,service_listener=None,connection_listener=None,gateway_parameters=None,callback_server_parameters=None): self._gateway = None self._lock = RLock() self._consumer = None self._imported_endpoints = {} self._imported_endpoints_lock = RLock() self._exported_endpoints_lock = RLock() self._exported_endpoints = {} self._map_converter = MapConverter() self._list_converter = ListConverter() self._service_listener = service_listener self._connection_listener = connection_listener self._connection = None self._gateway_parameters = gateway_parameters self._callback_server_parameters = callback_server_parameters def get_id(self): ''' Get the py4j id for this service bridge. :return: The py4j id...e.g. py4j://localhost:23334/python for this service bridge ''' cb = self._gateway.get_callback_server() return createLocalPy4jId(hostname=cb.get_listening_address(),port=cb.get_listening_port()) def export(self,svc,export_props=None): ''' Export the given svc via the given export_props. Note that the export_props must have and ENDPOINT_ID and contain the other standard Endpoint Description service properties as described by the OSGI R5+ Chapter 122 (Remote Service Admin) in the enterprise specification. :param svc: The Python service to export to java service registry :param export_props: An optional dictionary of Python properties. Note these properties must contain all the properties describing the service as required by the OSGI Endpoint Description. :return: The ENDPOINT_ID value from the export_props. This value may be used to get subsequent access to the svc and/or the export_properties via get_export_endpoint ''' with self._lock: self._raise_not_connected() '''The Java class attribute must be present''' java = getattr(svc,PY4J_JAVA_ATTRIBUTE) '''The Java.implements must be present''' objClass = getattr(java,PY4J_JAVA_IMPLEMENTS_ATTRIBUTE) if isinstance(objClass,str): objClass = [objClass] props = {} '''The export_properties attribute does not need to be present''' j_props = getattr(java,EXPORT_PROPERTIES_NAME,None) if j_props: props = osgiservicebridge.merge_dicts(props,j_props) if export_props: props = osgiservicebridge.merge_dicts(props,export_props) sec = props.get(osgiservicebridge.REMOTE_CONFIGS_SUPPORTED,None) if not sec: sec = [PY4J_EXPORTED_CONFIG] # check that OBJECTCLASS is set, if not props1 = get_edef_props(object_class=objClass,exported_cfgs=sec,ep_namespace=None,ecf_ep_id=self.get_id(),ep_rsvc_id=None,ep_ts=None) export_props = merge_dicts(props1, props) try: endpointid = export_props[ENDPOINT_ID] except KeyError: raise ArgumentError('Cannot export service since no ENDPOINT_ID present in export_props') with self._exported_endpoints_lock: self._exported_endpoints[endpointid] = (svc,export_props) # without holding lock, call __export try: self.__export(svc,export_props) except Exception as e: # if it fails, remove from exported endpoints self.remove_export_endpoint(endpointid) raise e return endpointid def update(self,update_props): ''' Update the previously exported svc with the given update_props. Note that the update_props must have and ENDPOINT_ID and contain the other standard Endpoint Description service properties as described by the OSGI R5+ Chapter 122 (Remote Service Admin) in the enterprise specification. :param update_props: A dictionary of Python properties. Note that these properties must contain all the properties describing the service as required by the OSGI Endpoint Description and must contain an ENDPOINT_ID that matches the endpoint ID previously returned from export :return: old endpoint props (dict) or None :raise: ArgumentError if there is not an ENDPOINT_ID value in the update_props ''' with self._lock: self._raise_not_connected() try: endpointid = update_props[ENDPOINT_ID] except KeyError: raise ArgumentError('Cannot update service since no ENDPOINT_ID present in update_props') # get lock and make sure oldendpoint is in exported_endpoints with self._exported_endpoints_lock: oldendpoint = self.get_export_endpoint(endpointid) if oldendpoint: self._exported_endpoints[endpointid] = (oldendpoint[0],update_props) else: return None try: self.__update(update_props) except Exception as e: # if exception, restore old endpoint with self._exported_endpoints_lock: self._exported_endpoints[endpointid] = oldendpoint raise e return oldendpoint[1] def unexport(self,endpointid): ''' Unexport the svc via the given export_props. Note that the export_props must have an ENDPOINT_ID and contain the other standard Endpoint Description service properties as described by the OSGI R5+ Chapter 122 (Remote Service Admin) in the enterprise specification. :param export_props: A dictionary of Python properties. Note that these properties must contain all the properties describing the service as required by the OSGI Endpoint Description. :return: The endpoint service if successfully unexported, None if unsuccessfully exported ''' with self._lock: self._raise_not_connected() if endpointid: endpoint = self.remove_export_endpoint(endpointid) if endpoint: try: self.__unexport(endpoint[1]) except Exception as e: with self._exported_endpoints_lock: self._exported_endpoints = endpoint raise e return endpoint def get_jvm(self): ''' Get the Py4JService Gateway's JVM :return: The Gateway's jvm ''' return self._gateway.jvm def get_gateway(self): ''' Get the Py4JService JavaGateway :return: The Gateway ''' return self._gateway def get_callback_server_parameters(self): ''' Get the callback server parameters used for the gateway :return: callback server parameters used for the gateway ''' return self._gateway.callback_server_parameters def get_import_endpoint(self,endpointid): ''' Get import endpoint for given endpointid :param endpointid: the endpoint id to find :return: the tuple of the endpoint or None if no endpoint exists for given endpointid. Element 0 is the service, element 1 is the dictionary of properties. ''' with self._imported_endpoints_lock: return self._imported_endpoints.get(endpointid,None) def get_export_endpoint(self,endpointid): ''' Get export endpoint for given endpointid :param endpointid: the endpoint id to find :return: the tuple of the endpoint or None if no endpoint exists for given endpointid. Element 0 is the service, element 1 is the dictionary of properties. ''' with self._exported_endpoints_lock: return self._exported_endpoints.get(endpointid,None) def get_export_endpoint_for_rsid(self,rsId): with self._exported_endpoints_lock: for eptuple in self._exported_endpoints.values(): val = eptuple[1][osgiservicebridge.ECF_RSVC_ID] if not val is None and val == rsId: return eptuple[0] return None def isconnected(self): ''' Returns True if the gateway callback server is connected, False if not ''' with self._lock: return True if self._gateway is not None else False def connect(self,gateway_parameters=None,callback_server_parameters=None,path_hook=None): ''' Connect gateway to Java side :param gateway_parameters an overriding GatewayParameters instance. If None then self._gateway_parameters is used. If that is None, then new GatewayParameters (defaults) is used :param callback_server_parameters an overriding CallbackServerParameters instance. If None then self._callback_server_parameters is used. If that is None, then new CallbackServerParameters (defaults) is used ''' if callback_server_parameters: self._callback_server_parameters = callback_server_parameters else: self._callback_server_parameters = self._callback_server_parameters if self._callback_server_parameters else CallbackServerParameters() if gateway_parameters: self._gateway_parameters = gateway_parameters else: self._gateway_parameters = self._gateway_parameters if self._gateway_parameters else GatewayParameters() with self._lock: if not self._gateway is None: raise OSError('already connected to java gateway') server_started.connect(self._started) self._gateway = JavaGateway(gateway_parameters=self._gateway_parameters,callback_server_parameters=self._callback_server_parameters) cbserver = self._gateway.get_callback_server() server_stopped.connect( self._stopped, sender=cbserver) server_connection_started.connect( self._connection_started, sender=cbserver) server_connection_stopped.connect( self._connection_stopped, sender=cbserver) pre_server_shutdown.connect( self._pre_shutdown, sender=cbserver) post_server_shutdown.connect( self._post_shutdown, sender=cbserver) class JavaRemoteServiceDiscoverer(object): def __init__(self, bridge): self._bridge = bridge def _external_discoverService(self,proxy,props): self._bridge._import_service_from_java(proxy,props) def _external_updateDiscoveredService(self,props): self._bridge._modify_service_from_java(props) def _external_undiscoverService(self,props): self._bridge._unimport_service_from_java(props) def _call_endpoint(self,rsId,methodName,serializedArgs): endpoint = self._bridge.get_export_endpoint_for_rsid(rsId) if endpoint: return endpoint._raw_bytes_from_java(methodName,serializedArgs) else: msg = 'No endpoint for rsId=%s methodName=%s serializedArgs=%s' % (rsId,methodName,serializedArgs) _logger.error(msg) raise Exception(msg) class Java: implements = [JAVA_DIRECT_ENDPOINT_CLASS, PY4J_CALL_BY_VALUE_CLASS] if not path_hook: path_hook = JavaPathHook(self) self._bridge = JavaRemoteServiceDiscoverer(self) '''Call _getExternalDirectDiscovery first, so that we are ready to export''' self._consumer = self._gateway.entry_point._getExternalDirectDiscovery() '''Then call _setDirectBridge so that java side can now call us to notify about exported services''' path_list = self._gateway.entry_point._setDirectBridge(path_hook,self._bridge, self._bridge, self.get_id()) path_hook.initialize_path(path_list) def get_module_type(self,modname): with self._lock: if not self.isconnected(): raise ImportError() return self._gateway.entry_point.getModuleType(modname) def get_module_code(self,modname,ispackage): with self._lock: if not self.isconnected(): raise ImportError() return self._gateway.entry_point.getModuleCode(modname,ispackage) def disconnect(self): with self._lock: if self.isconnected(): self._gateway.close() self._gateway = None self._consumer = None self._bridge = None with self._imported_endpoints_lock: self._imported_endpoints.clear() with self._exported_endpoints_lock: self._exported_endpoints.clear() def _started(self, sender, **kwargs): if self._connection_listener: self._connection_listener.started(kwargs["server"]) def _stopped(self, sender, **kwargs): if self._connection_listener: self._connection_listener.stopped(kwargs["server"]) def _connection_started(self, sender, **kwargs): with self._lock: self._connection = kwargs["connection"] if self._connection_listener: self._connection_listener.connection_started(self._connection) def _connection_stopped(self, sender, **kwargs): with self._lock: self._connection = None if self._connection_listener: self._connection_listener.connection_stopped(kwargs['connection'],kwargs.pop('exception',None)) def _pre_shutdown(self, sender, **kwargs): if self._connection_listener: self._connection_listener.pre_shutdown(kwargs["server"]) def _post_shutdown(self, sender, **kwargs): with self._imported_endpoints_lock: for endpointid in self._imported_endpoints.keys(): endpoint = self._imported_endpoints.get(endpointid,None) if self._service_listener and endpoint: try: self._service_listener.service_unimported(self, endpointid, endpoint[0], endpoint[1]) except: _logger.exception('_unimport_service_from_java listener threw exception endpointid={0}'.format(endpointid)) self.disconnect(); if self._connection_listener: self._connection_listener.post_shutdown(kwargs["server"]) # Methods called by java def _import_service_from_java(self,proxy,props): local_props = prepare_java_props(props) endpointid = local_props.get(ENDPOINT_ID,None) if endpointid: endpoint = (proxy,local_props) with self._imported_endpoints_lock: self._imported_endpoints[endpointid] = endpoint if self._service_listener: try: self._service_listener.service_imported(self, endpointid, endpoint[0], endpoint[1]) except: _logger.exception('__import_service_from_java listener threw exception endpointid={0}'.format(endpointid)) def _modify_service_from_java(self,newprops): newendpoint = None python_newprops = self._prepare_props(newprops) endpointid = python_newprops.get(ENDPOINT_ID,None) if endpointid: with self._imported_endpoints_lock: oldendpoint = self._imported_endpoints[endpointid] newendpoint = (oldendpoint[0],python_newprops) self._imported_endpoints[endpointid] = newendpoint if self._service_listener: try: self._service_listener.service_modified(self, endpointid, newendpoint[0], newendpoint[1]) except: _logger.exception('__modify_service_from_java listener threw exception endpointid={0}'.format(endpointid)) def _unimport_service_from_java(self,props): endpoint = None local_props = prepare_java_props(props) endpointid = local_props.get(ENDPOINT_ID,None) if endpointid: endpoint = self.remove_import_endpoint(endpointid) if self._service_listener and endpoint: try: self._service_listener.service_unimported(self, endpointid, endpoint[0], endpoint[1]) except: _logger.exception('__unimport_service_from_java listener threw exception endpointid={0}'.format(endpointid)) def remove_export_endpoint(self,endpointid): with self._exported_endpoints_lock: return self._exported_endpoints.pop(endpointid, None) def remove_import_endpoint(self,endpointid): with self._imported_endpoints_lock: return self._imported_endpoints.pop(endpointid, None) def _convert_string_list(self,l): llength = len(l) r = self._gateway.new_array(self._gateway.jvm.java.lang.String,llength) for i in range(0,llength): r[i] = l[i] return r def _convert_props_for_java(self,props): with self._lock: self._raise_not_connected() result = {} for item in props.items(): val = item[1] if isinstance(val, type([])): val = self._convert_string_list(val) result[item[0]] = val return self._map_converter.convert(result,self._gateway._gateway_client) def _raise_not_connected(self): if not self.isconnected(): raise OSError('Not connected to java gateway') def __export(self,svc,props): try: self._consumer._java_discoverService(svc,self._convert_props_for_java(props)) except Exception as e: _logger.error(e) raise e def __update(self,props): try: self._consumer._java_updateDiscoveredService(self._convert_props_for_java(props)) except Exception as e: _logger.error(e) raise e def __unexport(self,props): try: self._consumer._java_undiscoverService(self._convert_props_for_java(props)) except Exception as e: _logger.error(e) raise e
class Java(object): def __init__(self): java_cmd = subprocess.check_output("which java", shell=True).rstrip() try: self.gateway = JavaGateway() self.gateway.entry_point.getNumParseErrors("") except py4j.protocol.Py4JNetworkError: pass else: raise RuntimeError("Old java server still running") if os.path.isfile(JAR_PATH): pass else: py4j_jar = os.path.join(sys.prefix, 'share/py4j/py4j0.10.6.jar') subprocess.check_call( "mvn install:install-file -Dfile=" + py4j_jar + " -DgroupId=py4j -DartifactId=py4j -Dversion=0.10.6 -Dpackaging=jar -DgeneratePom=true", shell=True) subprocess.check_call("mvn package", shell=True) assert os.path.isfile(JAR_PATH) self.java_server = subprocess.Popen([java_cmd, "-jar", JAR_PATH], preexec_fn=os.setsid) tries = 100 while True: try: time.sleep(0.1) self.gateway = JavaGateway() self.gateway.entry_point.getNumParseErrors("") except py4j.protocol.Py4JNetworkError: tries -= 1 if tries > 0: continue else: raise else: break def __del__(self): #error("killing %i" % self.java_server.pid) if hasattr(self, 'gateway'): self.gateway.shutdown() self.gateway.close() del self.gateway if hasattr(self, 'java_server'): os.killpg(os.getpgid(self.java_server.pid), signal.SIGTERM) self.java_server.wait() def get_num_parse_errors(self, java_source): return self.gateway.entry_point.getNumParseErrors(java_source) @staticmethod def fix_extra_quotes(lexeme): l = tuple(lexeme) def lex(self, java_source): # WARNING: assumes lines not in mac format lpos = 0 lines = [0] while True: lpos = java_source.find('\n', lpos) + 1 if lpos <= 0: break lines.append(lpos) def convert_position(i): line = bisect_right(lines, i) col = i - lines[line - 1] return (line, col) def convert(lexeme): t, v, s, e, string = tuple(lexeme) assert not (' \n\r\t' in string) return (t, v, convert_position(s), convert_position(e), string) return [convert(l) for l in self.gateway.entry_point.lex(java_source)]
class NetLogoControllerServerStarter: __gw = None # New gateway connection ##server is in netlogo app folder def __init__(self): self.__gw = JavaGateway() print('Shutting down old server instance...') self.shutdownServer() #atexit.register(self.shutdownServer) '''Internal method to start JavaGateway server. Will be called by starServer on seperate thread''' def __runServer(self): __server_name = "nl4py.server.NetLogoControllerServer" try: nl_path = os.environ['NETLOGO_APP'] except KeyError: #looks like the NETLOGO_APP variable isn't set... #Trying to use os dependent defaults print("NETLOGO_APP was not set, trying to find NetLogo .jar files") import platform if (platform.system() == "Windows"): nl_path = "C:/Program Files/NetLogo 6.0.2/app" if (platform.system() == "Darwin"): nl_path = "/Applications/NetLogo 6.0.2/Java" pass nl_path = os.path.join(os.path.abspath(nl_path), "*") os.pathsep server_path = "./server/*" classpath = nl_path + os.pathsep + server_path xmx = psutil.virtual_memory().available / 1024 / 1024 / 1024 xms = "-Xms" + str(int(math.floor(xmx - 2))) + "G" xmx = "-Xmx" + str(int(math.floor(xmx))) + "G" subprocess.call([ "java", xmx, "-XX:-UseGCOverheadLimit", "-cp", classpath, __server_name ]) '''Starts JavaGateway server''' def startServer(self): #Fire up the NetLogo Controller server through python thread = threading.Thread(target=self.__runServer, args=()) thread.start() time.sleep(3) '''Send shutdown signal to the JavaGateway server. No further communication is possible unless server is restarted''' def shutdownServer(self): if (self.__gw != None): try: logging.disable(logging.CRITICAL) __bridge = self.__gw.entry_point __bridge.shutdownServer() self.__gw.close(keep_callback_server=False, close_callback_server_connections=True) self.__gw = None except Exception as e: pass
def run(): # ---------------------------- # # PREPARE INPUT # ---------------------------- # # parse input args parser = ArgumentParser(description='Create dashboard files') parser.add_argument('ontology', type=str, help='Input ontology file') parser.add_argument('registry', type=FileType('r'), help='Registry YAML file') parser.add_argument('schema', type=FileType('r'), help='OBO JSON schema') parser.add_argument('relations', type=FileType('r'), help='Table containing RO IRIs and labels') parser.add_argument('profile', type=str, help='Optional location of profile.txt file.') parser.add_argument('configfile', type=str, help='Location of the dashboard config file', default='build/robot.jar') parser.add_argument('outdir', type=str, help='Output directory') parser.add_argument('robot_jar',type=str,help='Location of your local ROBOT jar', default='build/robot.jar') args = parser.parse_args() config = DashboardConfig(args.configfile) owl = os.path.basename(args.ontology) namespace = os.path.splitext(owl)[0] ontology_file = args.ontology registry = args.registry schema = json.load(args.schema) contact_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "http://obofoundry.org/config/registry_schema/contact", "title": "registry_schema", "properties": { "contact": schema['properties']['contact'], }, "required": ["contact"], "level": "error" } license_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "$id": "http://obofoundry.org/config/registry_schema/license", "title": "registry_schema", "properties": { "license": schema['properties']['license'], }, "required": ["license"], "level": "error" } robot_jar = args.robot_jar ro_file = args.relations profile = args.profile # Create the build directory for this ontology ontology_dir = args.outdir os.makedirs(ontology_dir, exist_ok=True) # Launch the JVM using the robot JAR py4j.java_gateway.launch_gateway( jarpath=robot_jar, classpath='org.obolibrary.robot.PythonOperation', die_on_exit=True, port=25333) # Activate gateway to JVM gateway = JavaGateway() try: robot_gateway = gateway.jvm.org.obolibrary.robot # IOHelper for working with ontologies io_helper = robot_gateway.IOHelper() # Handle ontology file big = namespace in BIG_ONTS # Load dashboard data if exists dashboard_yml = os.path.join(ontology_dir, "dashboard.yml") data_yml = dict() if os.path.isfile(dashboard_yml): with open(dashboard_yml, 'r') as f: data_yml = yaml.load(f, Loader=yaml.SafeLoader) if 'changed' not in data_yml or 'results' not in data_yml or data_yml['changed'] == True: print("Analysis has to be updated, running.") else: sys.exit(0) if not big: # Load ontology as OWLOntology object if not ontology_file or not os.path.exists(ontology_file) or dash_utils.whitespace_only(ontology_file): # If ontology_file is None, the file does not exist, or the file is empty # Then the ontology is None ont_or_file = None else: try: ont_or_file = io_helper.loadOntology(ontology_file) except Exception: print('ERROR: Unable to load \'{0}\''.format(ontology_file), flush=True) ont_or_file = None # Get the Verison IRI version_iri = dash_utils.get_version_iri(ont_or_file) else: # Just provide path to file ont_or_file = ontology_file # Get the version IRI by text parsing version_iri = dash_utils.get_big_version_iri(ont_or_file) # Get the registry data yaml_data_raw = yaml.load(registry, Loader=yaml.SafeLoader) yaml_data = [] for o in yaml_data_raw['ontologies']: yaml_data.append(yaml_data_raw['ontologies'][o]) data = dash_utils.get_data(namespace, yaml_data) # Map of all ontologies to their domains domain_map = dash_utils.get_domains(yaml_data) # Map of RO labels to RO IRIs ro_props = fp_007.get_ro_properties(ro_file) if 'is_obsolete' in data and data['is_obsolete'] == 'true': # do not run on obsolete ontologies print('{0} is obsolete and will not be checked...'.format(namespace), flush=True) sys.exit(0) # ---------------------------- # # RUN CHECKS # ---------------------------- # print('-----------------\nChecking ' + namespace, flush=True) # Get the report based on if it's big or not report = None good_format = None if big: if namespace != 'gaz': # Report currently takes TOO LONG for GAZ print('Running ROBOT report on {0}...'.format(namespace), flush=True) report_obj = report_utils.BigReport(robot_gateway, namespace, ont_or_file, profile) report = report_obj.get_report() good_format = report_obj.get_good_format() else: if ont_or_file: # Ontology is not None print('Running ROBOT report on {0}...'.format(namespace), flush=True) report = report_utils.run_report(robot_gateway, io_helper, ont_or_file, profile) # Execute the numbered checks check_map = {} try: if big: check_map[1] = fp_001.big_is_open(ont_or_file, data, license_schema) else: check_map[1] = fp_001.is_open(ont_or_file, data, license_schema) except Exception as e: check_map[1] = 'INFO|unable to run check 1' print('ERROR: unable to run check 1 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: if big: check_map[2] = fp_002.big_is_common_format(good_format) else: check_map[2] = fp_002.is_common_format(ont_or_file) except Exception as e: check_map[2] = 'INFO|unable to run check 2' print('ERROR: unable to run check 2 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: if big: check_map[3] = fp_003.big_has_valid_uris(namespace, ont_or_file, ontology_dir) else: check_map[3] = fp_003.has_valid_uris(robot_gateway, namespace, ont_or_file, ontology_dir) except Exception as e: check_map[3] = 'INFO|unable to run check 3' print('ERROR: unable to run check 3 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: if big: check_map[4] = fp_004.big_has_versioning(ont_or_file) else: check_map[4] = fp_004.has_versioning(ont_or_file) except Exception as e: check_map[4] = 'INFO|unable to run check 4' print('ERROR: unable to run check 4 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[5] = fp_005.has_scope(data, domain_map) except Exception as e: check_map[5] = 'INFO|unable to run check 5' print('ERROR: unable to run check 5 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[6] = fp_006.has_valid_definitions(report) except Exception as e: check_map[6] = 'INFO|unable to run check 6' print('ERROR: unable to run check 6 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: if big: check_map[7] = fp_007.big_has_valid_relations(namespace, ont_or_file, ro_props, ontology_dir) else: check_map[7] = fp_007.has_valid_relations(namespace, ont_or_file, ro_props, ontology_dir) except Exception as e: check_map[7] = 'INFO|unable to run check 7' print('ERROR: unable to run check 7 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[8] = fp_008.has_documentation(data) except Exception as e: check_map[8] = 'INFO|unable to run check 8' print('ERROR: unable to run check 8 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[9] = fp_009.has_users(data) except Exception as e: check_map[9] = 'INFO|unable to run check 9' print('ERROR: unable to run check 9 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[11] = fp_011.has_contact(data, contact_schema) except Exception as e: check_map[11] = 'INFO|unable to run check 11' print('ERROR: unable to run check 11 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[12] = fp_012.has_valid_labels(report) except Exception as e: check_map[12] = 'INFO|unable to run check 12' print('ERROR: unable to run check 12 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: if big: check_map[16] = fp_016.big_is_maintained(ont_or_file) else: check_map[16] = fp_016.is_maintained(ont_or_file) except Exception as e: check_map[16] = 'INFO|unable to run check 16' print('ERROR: unable to run check 16 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) try: check_map[20] = fp_020.is_responsive(data) except Exception as e: check_map[20] = 'INFO|unable to run check 20' print('ERROR: unable to run check 20 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) # finally, add the report results to the dashboard and save the report try: check_map['report'] = report_utils.process_report(robot_gateway, report, ontology_dir) except Exception as e: check_map['report'] = 'INFO|unable to save report' print('ERROR: unable to save ROBOT report for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True) # ---------------------------- # # SAVE RESULTS # ---------------------------- # # Parse results err = 0 warn = 0 info = 0 all_checks = {} for check, result in check_map.items(): if result is None or 'status' not in result: print('Missing result for check {0}'.format(check), flush=True) continue status = result['status'] if status == 'ERROR': err += 1 elif status == 'WARN': warn += 1 elif status == 'INFO': info += 1 elif status != 'PASS': print('Unknown status "{0}" for check {1}'.format(status, check), flush=True) continue key = check if check in PRINCIPLE_MAP: key = PRINCIPLE_MAP[check] elif check == 'report': key = 'ROBOT Report' all_checks[key] = result # Summary status if err > 0: summary = 'ERROR' summary_comment = '{0} errors'.format(err) elif warn > 0: summary = 'WARN' summary_comment = '{0} warnings'.format(warn) elif info > 0: summary = 'INFO' summary_comment = '{0} info messages'.format(info) else: summary = 'PASS' summary_comment = '' summary_count = dict() summary_count['ERROR'] = err summary_count['WARN'] = warn summary_count['INFO'] = info date = datetime.datetime.today() save_data = {'namespace': namespace, 'version': version_iri, 'summary': {'status': summary, 'comment': summary_comment, 'summary_count': summary_count}, 'results': all_checks} oboscore_weights = config.get_oboscore_weights() oboscore_maximpacts = config.get_oboscore_max_impact() for key in save_data: data_yml[key] = save_data[key] data_yml['metrics']['Info: Experimental OBO score']['_dashboard'] = round_float( float(compute_dashboard_score(data_yml, oboscore_weights, oboscore_maximpacts)) / 100) oboscore = compute_obo_score(data_yml['metrics']['Info: Experimental OBO score']['_impact'], data_yml['metrics']['Info: Experimental OBO score']['_reuse'], data_yml['metrics']['Info: Experimental OBO score']['_dashboard'], data_yml['metrics']['Info: Experimental OBO score']['_impact_external'], oboscore_weights) data_yml['metrics']['Info: Experimental OBO score']['oboscore'] = round_float(oboscore['score']) data_yml['metrics']['Info: Experimental OBO score']['_formula'] = oboscore['formula'] # Save to YAML file print('Saving results to {0}'.format(dashboard_yml)) with open(dashboard_yml, 'w+') as f: yaml.dump(data_yml, f) except Exception: logging.exception(f"Creating dashboard for {ontology_file} failed") try: gateway.close() except Exception: pass sys.exit(0)
# This script tests the star brightness commands. # Created by Toni Sagrista from py4j.java_gateway import JavaGateway, GatewayParameters gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True)) gs = gateway.entry_point gs.setStarBrightness(100.0) gs.sleep(2) gs.setStarBrightness(70.0) gs.sleep(2) gs.setStarBrightness(50.0) gs.sleep(2) gs.setStarBrightness(30.0) gs.sleep(2) gs.setStarBrightness(12.0) gs.sleep(2) gateway.close()