def save_cda_files_in_xds(self, file_list):
gateway = JavaGateway()
xds_connector = gateway.entry_point
oid = "1.2.40.0.10.1.4.3.1"
root_temp_upload_path = configParser.get('temp-folders', 'upload')
result = []
for file in file_list:
if CDAEvaluator.evaluate_file_type(CDAEvaluator, file):
cda_file = CDAExtractor(file)
patient_id = cda_file.get_patient_id()
patient_full_name = cda_file.get_patient_name()
if not any(obj['patient_id'] == patient_id for obj in result):
result.append(self.create_patient_list(self, patient_id, patient_full_name))
patient_list = Patient.objects.filter(patient_id=patient_id,
patient_first_name=patient_full_name['vornamen'][0],
patient_last_name=patient_full_name['nachname'][0])
if patient_list is None or len(patient_list) == 0:
Patient.objects.create(patient_id=patient_id, patient_first_name=patient_full_name['vornamen'][0],
patient_last_name=patient_full_name['nachname'][0])
document_id = cda_file.get_document_id()
cda_exist = xds_connector.validateNewDocument(oid, str(patient_id), str(document_id))
# TODO: write patient infos into DB
if cda_exist is False:
file_name = str(patient_id) + '_' + str(document_id) + '.xml'
default_storage.save("Django_Server/recruitmenttool/cda_files/tempUpload/" + file_name, file)
xds_connector.uploadDocument(oid, str(patient_id), str(document_id),
root_temp_upload_path + "/" + file_name)
gateway.close()
return result
def _initialize_gateway(gateway_address):
(host, port) = gateway_address
callback_params = CallbackServerParameters(address=host, port=0)
gateway = JavaGateway(GatewayClient(address=host, port=port),
start_callback_server=True,
auto_convert=True,
callback_server_parameters=callback_params)
try:
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
java_import(gateway.jvm, "scala.collection.immutable.List")
except Py4JError as e:
log_error('Error while initializing java gateway: {}'.format(e))
gateway.close()
return None
log_debug('Java Gateway initialized {}'.format(gateway))
return gateway
def _initialize_gateway(gateway_address):
(host, port) = gateway_address
callback_params = CallbackServerParameters(address=host, port=0)
gateway = JavaGateway(GatewayClient(address=host, port=port),
start_callback_server=True,
auto_convert=True,
callback_server_parameters=callback_params)
try:
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
java_import(gateway.jvm, "org.apache.spark.api.java.*")
java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
java_import(gateway.jvm, "org.apache.spark.sql.*")
java_import(gateway.jvm, "org.apache.spark.sql.hive.*")
java_import(gateway.jvm, "scala.Tuple2")
java_import(gateway.jvm, "scala.collection.immutable.List")
except Py4JError as e:
log_error('Error while initializing java gateway: {}'.format(e))
gateway.close()
return None
return gateway
def testCallbackServer(self):
# A close is required to stop the thread.
gateway = JavaGateway(
callback_server_parameters=CallbackServerParameters())
gateway.close()
self.assertTrue(True)
sleep(2)
def shutdown_gateway(event, gateway: JavaGateway, resource_name: str, shutdown_jvm: bool):
if shutdown_jvm:
gateway.shutdown()
else:
gateway.close()
logger.info('Py4J gateway (%s) shut down', resource_name)
def get_visualized_cda(request, patient_id, document_id):
# cda_file = model.CDAFile.objects.all().filter(cda_id=1234567.1).first()
print(BASE_DIR)
root_tempDownload_path = configParser.get('temp-folders', 'download')
root_tempCache_path = configParser.get('temp-folders', 'cache')
cda_file_path = root_tempCache_path + "/" + patient_id + "/" + patient_id + "_" + document_id + ".xml"
# try to get CDA File from tempDownload Folder
if not path.exists(cda_file_path):
cda_file_path = root_tempDownload_path + "/" + patient_id + "/" + patient_id + "_" + document_id + ".xml"
# try to download CDA Files from Repository
if not path.exists(cda_file_path):
gateway = JavaGateway()
xds_connector = gateway.entry_point
oid = "1.2.40.0.10.1.4.3.1"
cda_file_path = xds_connector.queryDocumentWithId(
oid, patient_id, document_id)
gateway.close()
if cda_file_path == "NO_DOCUMENT_FOUND":
return Response("NO DOCUMENT FOUND",
status=status.HTTP_400_BAD_REQUEST)
stylesheet_path = configParser.get('xml-to-html-files', 'xsl-file')
cda_html = CDATransformer.transform_xml_to_xsl(CDATransformer,
cda_file_path,
stylesheet_path)
return HttpResponse(cda_html)
def testCallbackServer(self):
# A close is required to stop the thread.
gateway = JavaGateway(
callback_server_parameters=CallbackServerParameters())
gateway.close()
self.assertTrue(True)
sleep(2)
class MySm(object):
def __init__(self):
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=25666)) # 连接Java网关
self.sm = self.gateway.entry_point.getSm()
def __del__(self):
self.gateway.close()
def setup(self, config):
self.risk_threshold = self.config["tuning_config"]["risk_threshold"]
self.max_concurrent_trials = self.config["tuning_config"][
"max_concurrent_trials"]
self.payoff_empirical_min = self.config["tuning_config"][
"obj_fnc_params"]["payoff_empirical_min"]
self.payoff_empirical_max = self.config["tuning_config"][
"obj_fnc_params"]["payoff_empirical_max"]
self.min_eval_eps = self.config["tuning_config"]["eval_eps_min"]
self.max_eval_eps = self.config["tuning_config"]["eval_eps_max"]
self.lock = threading.Lock()
self.train_results = queue.Queue()
self.eval_results = queue.Queue()
self.java_gateway = None
self.my_id = self.trial_id
self.start_time = time.time()
self.am_stopped = False
self.cur_max_score = self.payoff_empirical_min
self.cur_max_train_score = self.payoff_empirical_min
default_gateway = JavaGateway()
while self.java_gateway is None:
for i in range(self.max_concurrent_trials):
try:
self.my_port = 25334 + 2 * i
java_port = self.my_port - 1
# if addJavaPort fails, it means it is being used by some other trial running
# in a different process
if not default_gateway.addJavaPort(java_port):
if i == self.max_concurrent_trials - 1:
time.sleep(0.5)
continue
gateway_params = GatewayParameters(
port=java_port, enable_memory_management=False)
callback_params = CallbackServerParameters(
port=self.my_port)
self.java_gateway = JavaGateway(
gateway_parameters=gateway_params,
callback_server_parameters=callback_params)
break
except:
default_gateway.removeJavaPort(self.my_port - 1)
continue
default_gateway.close(keep_callback_server=True,
close_callback_server_connections=False)
encoded_trial_params = TuningParser.encode_trial_params(self.config)
self.results_receiver = _TrialResultsReceiver(self)
self.java_gateway.entry_point.runTrial(self.trial_id,
encoded_trial_params,
self.results_receiver)
self.additional_setup()
def check_connection(gateway_parameters=None):
test_gateway = JavaGateway(gateway_parameters=gateway_parameters)
try:
# Call a dummy method just to make sure we can connect to the JVM
test_gateway.jvm.System.currentTimeMillis()
except Py4JNetworkError:
# We could not connect. Let"s wait a long time.
# If it fails after that, there is a bug with our code!
sleep(2)
finally:
test_gateway.close()
def test_gateway_connection():
test_gateway = JavaGateway()
try:
# Call a dummy method just to make sure we can connect to the JVM
test_gateway.jvm.System.lineSeparator()
except Py4JNetworkError:
# We could not connect. Let"s wait a long time.
# If it fails after that, there is a bug with our code!
sleep(2)
finally:
test_gateway.close()
def check_connection(gateway_parameters=None):
test_gateway = JavaGateway(gateway_parameters=gateway_parameters)
try:
# Call a dummy method just to make sure we can connect to the JVM
test_gateway.jvm.System.currentTimeMillis()
except Py4JNetworkError:
# We could not connect. Let"s wait a long time.
# If it fails after that, there is a bug with our code!
sleep(2)
finally:
test_gateway.close()
def test_gateway_connection():
test_gateway = JavaGateway()
try:
# Call a dummy method just to make sure we can connect to the JVM
test_gateway.jvm.System.lineSeparator()
except Py4JNetworkError:
# We could not connect. Let"s wait a long time.
# If it fails after that, there is a bug with our code!
sleep(2)
finally:
test_gateway.close()
def tearDownClass(cls):
gateway = JavaGateway()
workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\
getWorkspace()
root = workspace.getRoot()
pm = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor()
project1 = root.getProject('project1core3.0')
project1.delete(True, True, pm)
time.sleep(1)
gateway.close()
stop_eclipse()
clean_test_dir()
def stop_eclipse():
gateway = JavaGateway()
try:
gateway.entry_point.closeEclipse()
time.sleep(1)
gateway.shutdown()
except Exception:
pass
try:
gateway.close()
except Exception:
pass
def stop_eclipse():
gateway = JavaGateway()
try:
gateway.entry_point.closeEclipse()
time.sleep(1)
gateway.shutdown()
except Exception:
pass
try:
gateway.close()
except Exception:
pass
def download_all_files_from_patient(patient_id):
# download all file from XDS
# TODO: check if cda is already in temp = cache
gateway = JavaGateway()
xds_connector = gateway.entry_point
oid = "1.2.40.0.10.1.4.3.1"
print("Patient Files get processed from: " + str(patient_id))
xds_connector.downloadPatientFiles(oid, str(patient_id))
gateway.close()
return glob.glob(
configParser.get('temp-folders', 'download') + "/" + str(
patient_id) + "/*.xml")
def collect_trajectories(actor, critic, port, game_num, p2, rnn, n_frame):
logger.info(f'start fight with {p2}')
logger.info(f'game_num value {game_num}')
error = True
while error:
gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=port),
callback_server_parameters=CallbackServerParameters())
try:
manager = gateway.entry_point
current_time = int(time.time() * 1000)
# register AIs
collect_data_helper = CollectDataHelper(logger)
agent = SoundAgent(gateway,
actor=actor,
critic=critic,
collect_data_helper=collect_data_helper,
logger=logger,
n_frame=n_frame,
rnn=rnn)
sandbox_agent = SandboxAgent(gateway)
manager.registerAI(f'SoundAgent', agent)
manager.registerAI('Sandbox', sandbox_agent)
game = manager.createGame('ZEN', 'ZEN', 'SoundAgent', p2, game_num)
# start game
manager.runGame(game)
# finish game
logger.info('Finish game')
sys.stdout.flush()
# close gateway
gateway.close_callback_server()
gateway.close()
error = False
except Exception as ex:
print(ex)
logger.info('There is an error with the gateway, restarting')
gateway.close_callback_server()
gateway.close()
error = True
# return agent.get_trajectories_data()
agent_data = process_game_agent_data(actor, critic,
agent.collect_data_helper, rnn)
# try:
# kill_proc_tree(java_env.pid, False)
# except:
# print('kill process')
# agent.reset()
return agent_data
def load_from_envs() -> HDFSWrapperBase:
isReady = False
while isReady == False:
print("Testando conexão com a JVM")
try:
java_gateway = JavaGateway(eager_load=True)
isReady = True
print("Java pronto para atender novas requisições")
java_gateway.close()
except Exception as e:
print(
"Não foi possível conectar ao Java, tentando novamente em 2 segundos"
)
time.sleep(2)
return HDFSWrapperJava()
def load_into_hive(table_name, csv_path, hdfs_path, hdfs_path_dir, fild):
gateway = None
try:
gateway = JavaGateway()
ff = gateway.jvm.java.util.ArrayList()
for k in fild:
ff.append(k)
hive_util = gateway.entry_point.getHiveUtil()
func = hive_util.uploadToHive
func(table_name, csv_path, hdfs_path, hdfs_path_dir, ff)
except Exception as e:
raise e
finally:
if gateway is not None:
gateway.close()
def check_eclipse():
'''Check that Eclipse is started and that recodoc can communicate with
it.'''
gateway = JavaGateway()
try:
success = gateway.entry_point.getServer().getListeningPort() > 0
except Exception:
success = False
if success:
print('Connection to Eclipse: OK')
else:
print('Connection to Eclipse: ERROR')
gateway.close()
return success
def check_eclipse():
'''Check that Eclipse is started and that recodoc can communicate with
it.'''
gateway = JavaGateway()
try:
success = gateway.entry_point.getServer().getListeningPort() > 0
except Exception:
success = False
if success:
print('Connection to Eclipse: OK')
else:
print('Connection to Eclipse: ERROR')
gateway.close()
return success
def download_report(f, t):
gateway = None
try:
gateway = JavaGateway()
ff = gateway.jvm.java.util.ArrayList()
tt = gateway.jvm.java.util.ArrayList()
for k in f:
ff.append(k)
for k in t:
tt.append(k)
hive_util = gateway.entry_point.getHiveUtil()
func = hive_util.downloadFiles
func(ff, tt)
except Exception as e:
raise e
finally:
if gateway is not None:
gateway.close()
def testLinkEclipseProject(self):
create_code_local('project1', 'core', '3.0')
to_path = get_codebase_path('project1', 'core', '3.0')
to_path = os.path.join(to_path, 'src')
os.rmdir(to_path)
from_path = os.path.join(settings.TESTDATA, 'testproject1', 'src')
shutil.copytree(from_path, to_path)
link_eclipse('project1', 'core', '3.0')
gateway = JavaGateway()
workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\
getWorkspace()
root = workspace.getRoot()
pm = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor()
project1 = root.getProject('project1core3.0')
self.assertIsNotNone(project1)
project1.delete(True, True, pm)
time.sleep(1)
gateway.close()
def testLinkEclipseProject(self):
create_code_local('project1', 'core', '3.0')
to_path = get_codebase_path('project1', 'core', '3.0')
to_path = os.path.join(to_path, 'src')
os.rmdir(to_path)
from_path = os.path.join(settings.TESTDATA, 'testproject1', 'src')
shutil.copytree(from_path, to_path)
link_eclipse('project1', 'core', '3.0')
gateway = JavaGateway()
workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\
getWorkspace()
root = workspace.getRoot()
pm = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor()
project1 = root.getProject('project1core3.0')
self.assertIsNotNone(project1)
project1.delete(True, True, pm)
time.sleep(1)
gateway.close()
def py4j_common_hive_util(func, *param):
gateway = None
try:
gateway = JavaGateway()
hive_util = gateway.entry_point.getHiveUtil()
func = getattr(hive_util, func)
result = func(*param)
except Py4JNetworkError as e:
response = Response.fail(ERRORS.PY4J_CONNECTION_ERROR, None)
return HttpResponse(response.to_json())
except Py4JJavaError as e:
# 查询出错，返回错误信息
response = Response.fail(ERRORS.HIVE_QUERY_ERROR,
e.java_exception.getMessage())
return HttpResponse(response.to_json())
finally:
if gateway is not None:
gateway.close()
return result
def link_eclipse(pname, bname, release):
'''Add the Java Project created with create_code_local to the Eclipse
workspace.'''
project_key = pname + bname + release
codebase_path = get_codebase_path(pname, bname, release)
gateway = JavaGateway()
workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\
getWorkspace()
root = workspace.getRoot()
path = gateway.jvm.org.eclipse.core.runtime.Path(os.path.join(
codebase_path, PROJECT_FILE))
project_desc = workspace.loadProjectDescription(path)
new_project = root.getProject(project_key)
nmonitor = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor()
# gateway.jvm.py4j.GatewayServer.turnLoggingOn()
# To avoid workbench problem (don't know why it needs some time).
time.sleep(1)
new_project.create(project_desc, nmonitor)
new_project.open(nmonitor)
gateway.close()
def link_eclipse(pname, bname, release):
'''Add the Java Project created with create_code_local to the Eclipse
workspace.'''
project_key = pname + bname + release
codebase_path = get_codebase_path(pname, bname, release)
gateway = JavaGateway()
workspace = gateway.jvm.org.eclipse.core.resources.ResourcesPlugin.\
getWorkspace()
root = workspace.getRoot()
path = gateway.jvm.org.eclipse.core.runtime.Path(
os.path.join(codebase_path, PROJECT_FILE))
project_desc = workspace.loadProjectDescription(path)
new_project = root.getProject(project_key)
nmonitor = gateway.jvm.org.eclipse.core.runtime.NullProgressMonitor()
# gateway.jvm.py4j.GatewayServer.turnLoggingOn()
# To avoid workbench problem (don't know why it needs some time).
time.sleep(1)
new_project.create(project_desc, nmonitor)
new_project.open(nmonitor)
gateway.close()
def prepare_test_data(request):
gateway = JavaGateway()
xds_connector = gateway.entry_point
oid = "1.2.40.0.10.1.4.3.1"
cda_test_files = glob.glob(configParser.get('demo-files', 'cda-files'))
for file in cda_test_files:
cda_file = CDAExtractor(file)
patient_id = cda_file.get_patient_id()
patient_full_name = cda_file.get_patient_name()
patient_list = Patient.objects.filter(patient_id=patient_id)
if patient_list is None or len(patient_list) == 0:
Patient.objects.create(
patient_id=patient_id,
patient_first_name=patient_full_name['vornamen'][0],
patient_last_name=patient_full_name['nachname'][0])
document_id = cda_file.get_document_id()
cda_exist = xds_connector.validateNewDocument(oid, str(patient_id),
str(document_id))
# TODO: write patient infos into DB
if cda_exist is False:
xds_connector.uploadDocument(oid, str(patient_id),
str(document_id), BASE_DIR + file)
gateway.close()
return Response("TEST DATA UPLAODED", status=status.HTTP_201_CREATED)
def imageProcessor(image, filterRequested):
decidedImage = image
#(Step 1) Given an image name, if it fails to find the image, error must be raised.
if not os.path.isfile(image):
raise IOError
#Don't filter out jpg. They are already compressed enough!
ext = os.path.splitext(image)[-1].lower()
if ext == ".jpg":
filterRequested = False;
#(Step 2) Using OCR, verify whether the image is for the advertisement purpose or not.
score = 0
advertisement = False
#If there is a brand name, it is possible that it's going to be the advertisement
if Detect_Logo(image):
score += 1
score += contentVerify(image)
#print(score)
#If the score reaches the threshold, then consider it as an advertisement
if score >= 10:
advertisement = True
#(Step 3) Using Java, Convert image from original to Grayscale without modifying the original image
if filterRequested and advertisement:
gateway = JavaGateway()
convertImage = gateway.entry_point
convertImage.runImageProcessing(image)
decidedImage = 'Filtered.png'
gateway.close()
#based on the "filterRequest" Boolean, send the name of original/updated image
return decidedImage
class LocalJvmBridge(object):
_bridge = None
def __init__(self):
self.process = None
self.gateway = None
self.app = None
self.port = 0
@classmethod
def inst(cls):
if cls._bridge is None:
cls._bridge = LocalJvmBridge()
cls._bridge.init()
return cls._bridge
def init(self):
# noinspection PyProtectedMember
from .config import g_config
debug_mode = g_config["debug_mode"]
# redirect stdout if in debug_mode
# Note: if enableLazyXXX is used in Pipeline stages, Java side will print stuff.
# This case is very difficult to handle, so do not set redirect_stdout to True.
redirect_stdout = sys.stdout if debug_mode else None
# redirect stderr if in debug_mode and not in IPython (will fail)
redirect_stderr = sys.stderr if debug_mode and not in_ipython(
) else None
try:
# noinspection PyUnresolvedReferences
from pyflink.pyflink_gateway_server import construct_log_settings
if os.name != 'nt':
try:
from pyflink.pyflink_gateway_server import prepare_environment_variables
env = dict(os.environ)
prepare_environment_variables(env)
log_settings = construct_log_settings(env)
except TypeError:
# noinspection PyArgumentList
log_settings = construct_log_settings()
else:
log_settings = []
except ImportError:
log_settings = []
javaopts = g_config['java_opts'].copy()
javaopts += log_settings
if os.name == 'nt':
# Limit Java side to only detect and use Java implementation of BLAS/LAPACK/ARPACK.
# Java process started through py4j will stuck when trying to load native libraries.
# See https://github.com/bartdag/py4j/issues/444
javaopts += [
"-Dcom.github.fommil.netlib.BLAS=com.github.fommil.netlib.F2jBLAS",
"-Dcom.github.fommil.netlib.LAPACK=com.github.fommil.netlib.F2jLAPACK",
"-Dcom.github.fommil.netlib.ARPACK=com.github.fommil.netlib.F2jARPACK"
]
if g_config['gateway_port'] is not None:
self.port = int(g_config['gateway_port'])
else:
self.port = launch_gateway(port=0,
javaopts=javaopts,
die_on_exit=True,
daemonize_redirect=True,
redirect_stdout=redirect_stdout,
redirect_stderr=redirect_stderr,
classpath=os.pathsep.join(
list_all_jars()))
print('JVM listening on 127.0.0.1:{}'.format(self.port))
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=self.port,
auto_field=True,
auto_convert=True),
callback_server_parameters=CallbackServerParameters(
port=0, daemonize=True, daemonize_connections=True),
start_callback_server=False)
callback_server_port = self.gateway.get_callback_server(
).get_listening_port()
self.gateway.java_gateway_server.resetCallbackClient(
self.gateway.java_gateway_server.getCallbackClient().getAddress(),
callback_server_port)
def close(self):
self.gateway.close(keep_callback_server=True)
self._bridge = None
class FightingiceEnv_Display_NoFrameskip(gym.Env):
metadata = {'render.modes': ['human']}
def __init__(self, **kwargs):
self.freq_restart_java = 3
self.java_env_path = os.getcwd()
if "java_env_path" in kwargs.keys():
self.java_env_path = kwargs["java_env_path"]
if "freq_restart_java" in kwargs.keys():
self.freq_restart_java = kwargs["freq_restart_java"]
if "port" in kwargs.keys():
self.port = kwargs["port"]
else:
try:
import port_for
self.port = port_for.select_random(
) # select one random port for java env
except:
raise ImportError(
"Pass port=[your_port] when make env, or install port_for to set startup port automatically, maybe pip install port_for can help"
)
_actions = "AIR AIR_A AIR_B AIR_D_DB_BA AIR_D_DB_BB AIR_D_DF_FA AIR_D_DF_FB AIR_DA AIR_DB AIR_F_D_DFA AIR_F_D_DFB AIR_FA AIR_FB AIR_GUARD AIR_GUARD_RECOV AIR_RECOV AIR_UA AIR_UB BACK_JUMP BACK_STEP CHANGE_DOWN CROUCH CROUCH_A CROUCH_B CROUCH_FA CROUCH_FB CROUCH_GUARD CROUCH_GUARD_RECOV CROUCH_RECOV DASH DOWN FOR_JUMP FORWARD_WALK JUMP LANDING NEUTRAL RISE STAND STAND_A STAND_B STAND_D_DB_BA STAND_D_DB_BB STAND_D_DF_FA STAND_D_DF_FB STAND_D_DF_FC STAND_F_D_DFA STAND_F_D_DFB STAND_FA STAND_FB STAND_GUARD STAND_GUARD_RECOV STAND_RECOV THROW_A THROW_B THROW_HIT THROW_SUFFER"
action_strs = _actions.split(" ")
self.observation_space = spaces.Box(low=0, high=1, shape=(96, 64, 1))
self.action_space = spaces.Discrete(len(action_strs))
os_name = platform.system()
if os_name.startswith("Linux"):
self.system_name = "linux"
elif os_name.startswith("Darwin"):
self.system_name = "macos"
else:
self.system_name = "windows"
if self.system_name == "linux":
# first check java can be run, can only be used on Linux
java_version = subprocess.check_output(
'java -version 2>&1 | awk -F[\\\"_] \'NR==1{print $2}\'',
shell=True)
if java_version == b"\n":
raise ModuleNotFoundError("Java is not installed")
else:
print("Please make sure you can run java if you see some error")
# second check if FightingIce is installed correct
start_jar_path = os.path.join(self.java_env_path, "FightingICE.jar")
start_data_path = os.path.join(self.java_env_path, "data")
start_lib_path = os.path.join(self.java_env_path, "lib")
lwjgl_path = os.path.join(start_lib_path, "lwjgl", "*")
lib_path = os.path.join(start_lib_path, "*")
start_system_lib_path = os.path.join(self.java_env_path, "lib",
"natives", self.system_name)
natives_path = os.path.join(start_system_lib_path, "*")
if os.path.exists(start_jar_path) and os.path.exists(
start_data_path) and os.path.exists(
start_lib_path) and os.path.exists(start_system_lib_path):
pass
else:
error_message = "FightingICE is not installed in your script launched path {}, set path when make() or start script in FightingICE path".format(
self.java_env_path)
raise FileExistsError(error_message)
self.java_ai_path = os.path.join(self.java_env_path, "data", "ai")
ai_path = os.path.join(self.java_ai_path, "*")
if self.system_name == "windows":
self.start_up_str = "{};{};{};{};{}".format(
start_jar_path, lwjgl_path, natives_path, lib_path, ai_path)
self.need_set_memory_when_start = True
else:
self.start_up_str = "{}:{}:{}:{}:{}".format(
start_jar_path, lwjgl_path, natives_path, lib_path, ai_path)
self.need_set_memory_when_start = False
self.game_started = False
self.round_num = 0
def _start_java_game(self):
# start game
print("Start java env in {} and port {}".format(
self.java_env_path, self.port))
devnull = open(os.devnull, 'w')
if self.system_name == "windows":
# -Xms1024m -Xmx1024m we need set this in windows
self.java_env = subprocess.Popen([
"java", "-Xms1024m", "-Xmx1024m", "-cp", self.start_up_str,
"Main", "--port",
str(self.port), "--py4j", "--fastmode", "--grey-bg",
"--inverted-player", "1", "--mute", "--limithp", "400", "400"
],
stdout=devnull,
stderr=devnull)
elif self.system_name == "linux":
self.java_env = subprocess.Popen([
"java", "-cp", self.start_up_str, "Main", "--port",
str(self.port), "--py4j", "--fastmode", "--grey-bg",
"--inverted-player", "1", "--mute", "--limithp", "400", "400"
],
stdout=devnull,
stderr=devnull)
elif self.system_name == "macos":
self.java_env = subprocess.Popen([
"java", "-XstartOnFirstThread", "-cp", self.start_up_str,
"Main", "--port",
str(self.port), "--py4j", "--fastmode", "--grey-bg",
"--inverted-player", "1", "--mute", "--limithp", "400", "400"
],
stdout=devnull,
stderr=devnull)
# self.java_env = subprocess.Popen(["java", "-cp", "/home/myt/gym-fightingice/gym_fightingice/FightingICE.jar:/home/myt/gym-fightingice/gym_fightingice/lib/lwjgl/*:/home/myt/gym-fightingice/gym_fightingice/lib/natives/linux/*:/home/myt/gym-fightingice/gym_fightingice/lib/*", "Main", "--port", str(self.free_port), "--py4j", "--c1", "ZEN", "--c2", "ZEN","--fastmode", "--grey-bg", "--inverted-player", "1", "--mute"])
# sleep 3s for java starting, if your machine is slow, make it longer
time.sleep(3)
def _start_gateway(self, p2=Machete):
# auto select callback server port and reset it in java env
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(port=self.port),
callback_server_parameters=CallbackServerParameters(port=0))
python_port = self.gateway.get_callback_server().get_listening_port()
self.gateway.java_gateway_server.resetCallbackClient(
self.gateway.java_gateway_server.getCallbackClient().getAddress(),
python_port)
self.manager = self.gateway.entry_point
# create pipe between gym_env_api and python_ai for java env
server, client = Pipe()
self.pipe = server
self.p1 = GymAIDisplay(self.gateway, client, False)
self.manager.registerAI(self.p1.__class__.__name__, self.p1)
if isinstance(p2, str):
# p2 is a java class name
self.p2 = p2
self.game_to_start = self.manager.createGame(
"ZEN", "ZEN", self.p1.__class__.__name__, self.p2,
self.freq_restart_java)
else:
# p2 is a python class
self.p2 = p2(self.gateway)
self.manager.registerAI(self.p2.__class__.__name__, self.p2)
self.game_to_start = self.manager.createGame(
"ZEN", "ZEN", self.p1.__class__.__name__,
self.p2.__class__.__name__, self.freq_restart_java)
self.game = Thread(target=game_thread,
name="game_thread",
args=(self, ))
self.game.start()
self.game_started = True
self.round_num = 0
def _close_gateway(self):
self.gateway.close_callback_server()
self.gateway.close()
del self.gateway
def _close_java_game(self):
self.java_env.kill()
del self.java_env
self.pipe.close()
del self.pipe
self.game_started = False
def reset(self, p2=Machete):
# start java game if game is not started
if self.game_started is False:
try:
self._close_gateway()
self._close_java_game()
except:
pass
self._start_java_game()
self._start_gateway(p2)
# to provide crash, restart java game in some freq
if self.round_num == self.freq_restart_java * 3: # 3 is for round in one game
try:
self._close_gateway()
self._close_java_game()
self._start_java_game()
except:
raise SystemExit("Can not restart game")
self._start_gateway(p2)
# just reset is anything ok
self.pipe.send("reset")
self.round_num += 1
obs = self.pipe.recv()
return obs
def step(self, action):
# check if game is running, if not try restart
# when restart, dict will contain crash info, agent should do something, it is a BUG in this version
if self.game_started is False:
dict = {}
dict["pre_game_crashed"] = True
return self.reset(), 0, None, dict
self.pipe.send(["step", action])
if self.pipe.poll(5):
message = self.pipe.recv()
new_obs, reward, done, dict = message
else:
new_obs, reward = self.p1.get_obs(), self.p1.get_reward()
dict = {}
dict["no_data_receive"] = True
logging.warning(
"server can not receive, request to reset the game")
return new_obs, reward, True, dict
return new_obs, reward, done, dict
def render(self, mode='human'):
# no need
pass
def close(self):
if self.game_started:
self._close_java_game()
class JavaSnippetParser(object):
HANDLE_SEPARATOR = ":"
TYPE_KIND = "T"
METHOD_KIND = "M"
FIELD_KIND = "F"
ANNOTATION_KIND = "A"
ANNOTATION_PARAMETER_KIND = "P"
ENUM_KIND = "E"
ENUM_VALUE_KIND = "V"
REQUEST_NAME = 'recodoc_request'
def __init__(self, project, source):
self.project = project
self.source = source
self._load()
def _load(self):
self.gateway = JavaGateway(start_callback_server=False)
self.PPACoreUtil = \
self.gateway.jvm.ca.mcgill.cs.swevo.ppa.util.PPACoreUtil
self.class_kind = CodeElementKind.objects.get(kind='class')
self.unknown_kind = CodeElementKind.objects.get(kind='unknown')
self.method_kind = CodeElementKind.objects.get(kind='method')
self.field_kind = CodeElementKind.objects.get(kind='field')
self.enumeration_kind = CodeElementKind.objects.get(kind='enumeration')
self.annotation_kind = CodeElementKind.objects.get(kind='annotation')
def parse(self, progress_monitor=NullProgressMonitor()):
options = self.gateway.jvm.ca.mcgill.cs.swevo.ppa.PPAOptions()
mcodes = CodeSnippet.objects.filter(language='j').\
filter(project=self.project).\
filter(source=self.source)
#mcodes = CodeSnippet.objects.filter(pk=122170)
count = mcodes.count()
progress_monitor.start('Parsing Code Snippets ({0})'
.format(count), count)
for mcode in mcodes:
# Don't parse snippets that are already parsed.
if mcode.single_code_references.count() > 0:
progress_monitor.word('Skipped a parsed code snippet', 1)
continue
text = mcode.snippet_text
if text != None and text != '':
try:
cu = self._get_cu(text, options)
if cu != None:
visitor = self.gateway.jvm.ca.mcgill.cs.swevo.ppa.\
util.NameMapVisitor(True, True, True)
cu.accept(visitor)
self._process_name(
visitor.getBindings(),
visitor.getDeclarations(),
visitor.getNodes(),
mcode)
except Exception:
logger.exception(
'Error with snippet: PK: {0}. Text:\n\n: {1}'
.format(mcode.pk, text))
finally:
try:
self.PPACoreUtil.cleanUpAll(self.REQUEST_NAME)
except Exception:
print_exc()
else:
print('issue!')
gc.collect()
progress_monitor.work('Parsed a Code Snippet: pk={0}'
.format(mcode.pk), 1)
progress_monitor.done()
self.gateway.close()
def _get_cu(self, text, options):
try:
if is_cu_body(text):
cu = self.PPACoreUtil.getCU(text, options, self.REQUEST_NAME)
elif is_class_body(text):
cu = self.PPACoreUtil.getSnippet(text, options, True,
self.REQUEST_NAME)
else:
cu = self.PPACoreUtil.getSnippet(text, options, False,
self.REQUEST_NAME)
except Exception:
cu = None
logger.exception('Error while processing cu for {0}'.format(text))
return cu
def _process_name(self, bindings, declarations, nodes, mcode):
for i, binding in enumerate(bindings):
declaration = declarations[i]
parts = binding.split(self.HANDLE_SEPARATOR)
kind = parts[0][2]
kind_hint = None
content = binding
if kind == self.TYPE_KIND:
kind_hint = self.class_kind
elif kind == self.METHOD_KIND:
kind_hint = self.method_kind
elif kind == self.FIELD_KIND:
kind_hint = self.field_kind
elif kind == self.ANNOTATION_KIND:
kind_hint = self.annotation_kind
elif kind == self.ENUM_KIND:
kind_hint = self.enumeration_kind
else:
logger.warning('Not a high level code reference {0}'.\
format(kind))
if kind_hint is not None:
code = SingleCodeReference(
content=content,
xpath=mcode.xpath,
file_path=mcode.file_path,
kind_hint=kind_hint,
declaration=declaration,
snippet=mcode,
source=mcode.source,
index=(mcode.index * -1000) - i)
code.project = self.project
if mcode.project_release is not None:
code.project_release = mcode.project_release
code.local_context = mcode.local_context
code.mid_context = mcode.mid_context
code.global_context = mcode.global_context
code.save()
class PowerDynSimEnv(gym.Env):
metadata = {}
_case_files = ""
_dyn_sim_config_file = ""
_rl_config_file = ""
default_time_step = 0.1
action_type = 'discrete'
a_gateway = None
# define InterPSS dynamic simulation service
ipss_app = None
def __init__(self,
case_files,
dyn_sim_config_file,
rl_config_file,
server_port_num=25333):
# change from global to class-level variable to support parallel process
#global gateway
self.a_gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=server_port_num, auto_convert=True))
#global ipss_app
self.ipss_app = self.a_gateway.entry_point
from gym import spaces
_case_files = transfer2JavaStringAry(self.a_gateway, case_files)
#initialize the power system simulation service
# {observation_history_length,observation_space_dim, action_location_num, action_level_num};
dim_ary = self.ipss_app.initStudyCase(_case_files, dyn_sim_config_file,
rl_config_file)
observation_history_length = dim_ary[0]
observation_space_dim = dim_ary[1]
# set agent-environment interaction time step,
self.time_step = self.ipss_app.getEnvTimeStep(
) if self.ipss_app.getEnvTimeStep() > 0 else self.default_time_step
self.action_type = self.ipss_app.getActionSpaceType()
print('action type = ', self.action_space)
# for discrete action space
#define action and observation spaces
"""
if(action_location_num == 1):
self.action_space = spaces.Discrete(action_level_num) # Discrete, 1-D dimension
else:
#print('N-D dimension Discrete Action space is not supported it yet...TODO')
# the following is based on the latest gym dev version
# action_def_vector = np.ones(action_location_num, dtype=np.int32)*action_level_num
# for gym version 0.10.4, it is parametrized by passing an array of arrays containing [min, max] for each discrete action space
# for exmaple, MultiDiscrete([ [0,4], [0,1], [0,1] ])
action_def_vector = np.ones((action_location_num,2),dtype=np.int32)
action_def_vector[:,1] = action_level_num -1
aa = np.asarray(action_def_vector, dtype=np.int32)
self.action_space = spaces.MultiDiscrete(action_def_vector) # Discrete, N-D dimension
"""
if self.action_type.lower() == 'discrete':
print(
'observation_history_length,observation_space_dim, action_location_num, action_level_num = '
)
print(dim_ary[0], dim_ary[1], dim_ary[2], dim_ary[3])
action_location_num = dim_ary[2]
action_level_num = dim_ary[3]
action_num = action_level_num**action_location_num
self.action_space = spaces.Discrete(action_num)
self.cnts = np.ones(action_location_num) * action_level_num
elif self.action_type.lower() == 'continuous':
print(
'observation_history_length,observation_space_dim, action_location_num = '
)
print(dim_ary[0], dim_ary[1], dim_ary[2])
action_ranges = transfer2DJavaArray2NumpyArray(
self.ipss_app.getActionValueRanges())
print('action value ranges = ', action_ranges)
low = action_ranges[:, 0]
high = action_ranges[:, 1]
print('action range low =', low, 'action range high =', high)
self.action_space = spaces.Box(low,
high,
dtype=action_ranges.dtype)
print(self.action_space)
self.observation_space = spaces.Box(
-999,
999,
shape=(observation_history_length *
observation_space_dim, )) # Continuous
self.seed()
#TOOD get the initial states
self.state = None
self.steps_beyond_done = None
self.restart_simulation = True
def seed(self, seed=None):
self.np_random, seed = seeding.np_random(seed)
return [seed]
def step(self, action):
assert self.action_space.contains(
action), "%r (%s) invalid" % (action, type(action))
# step-1 convert from Gym discrete actions into actions applied to power system simulator
actionMapped = None
if self.action_type == 'discrete':
actionMapped = refer_new(action, self.cnts)
elif self.action_type == 'continuous':
actionMapped = np.asarray(action)
#print(actionMapped)
actionPyAry = np.asarray(actionMapped, dtype=np.float64)
# print(actionPyAry, 'len = ', actionPyAry.size)
# np array size = number of elements in the array
actionJavaAry = self.a_gateway.new_array(self.a_gateway.jvm.double,
actionPyAry.size)
if (actionPyAry.size == 1):
actionJavaAry[0] = float(action)
else:
i = 0
for x in actionPyAry:
actionJavaAry[i] = x
i = i + 1
# step-2 apply the actions to the simulator and run the simulation for one interaction step forward
self.ipss_app.nextStepDynSim(self.time_step, actionJavaAry,
self.action_type)
# step-3 retrieve the state from InterPSS simulation service
# observations is a Java_Collections array
observations = self.ipss_app.getEnvObversations()
# convert it from Java_collections array to native Python array
self.state = transfer2DJavaArray2NumpyArray(observations)
# print('observation shape: ', np.shape(self.state))
# step-4 check the states to see whether it go beyond the limits
done = self.ipss_app.isSimulationDone()
if not done:
reward = self.ipss_app.getReward()
elif self.steps_beyond_done is None:
self.steps_beyond_done = 0
reward = self.ipss_app.getReward(
) # even it is done, ipss_app would calculate and return a corresponding reward
else:
if self.steps_beyond_done == 0:
logger.warning(
"You are calling 'step()' even though this environment has already returned done = True. You should always call 'reset()' once you receive 'done = True' -- any further steps are undefined behavior."
)
self.steps_beyond_done += 1
reward = 0.0
return np.array(self.state).ravel(), reward, done, {}
def reset(self):
# reset need to randomize the operation state and fault location, and fault time
study_cases = self.ipss_app.getStudyCases()
total_case_num = len(study_cases)
if total_case_num == 0:
total_case_num = 1
case_Idx = np.random.randint(0, total_case_num) # an integer
total_fault_buses = len(self.ipss_app.getFaultBusCandidates())
fault_bus_idx = np.random.randint(
0, total_fault_buses
) # an integer, in the range of [0, total_bus_num-1]
#fault_bus_idx = 3 # an integer, in the range of [0, total_bus_num-1]
#fault_start_time =random.uniform(0.99, 1.01) # a double number, in the range of [0.2, 1]
fault_start_time_ary = transfer1DJavaArray2NumpyArray(
self.ipss_app.getFaultStartTimeCandidates())
fault_start_time = fault_start_time_ary[np.random.randint(
0, len(fault_start_time_ary))]
ftd_candidates = transfer1DJavaArray2NumpyArray(
self.ipss_app.getFaultDurationCandidates())
fault_duation_time = ftd_candidates[np.random.randint(
0, len(ftd_candidates)
)] # a double number, in the range of [0.08, 0.4]
# reset initial state to states of time = 0, non-fault
self.ipss_app.reset(case_Idx, fault_bus_idx, fault_start_time,
fault_duation_time)
#self.state = None
# observations is a Java_Collections array
observations = self.ipss_app.getEnvObversations()
# convert it from Java_collections array to native Python array
self.state = transfer2DJavaArray2NumpyArray(observations)
#print(self.state)
self.steps_beyond_done = None
self.restart_simulation = True
return np.array(self.state).ravel()
# init the system with a specific state and fault
def validate(self, case_Idx, fault_bus_idx, fault_start_time,
fault_duation_time):
total_bus_num = self.ipss_app.getTotalBusNum()
self.ipss_app.reset(case_Idx, fault_bus_idx, fault_start_time,
fault_duation_time)
# observations is a Java_Collections array
observations = self.ipss_app.getEnvObversations()
# convert it from Java_collections array to native Python array
self.state = transfer2DJavaArray2NumpyArray(observations)
self.steps_beyond_done = None
self.restart_simulation = True
return np.array(self.state).ravel()
def close_connection(self):
self.a_gateway.close(keep_callback_server=True,
close_callback_server_connections=False)
class JavaParser(object):
'''Parses a Java codebase and creates the appropriate CodeElement.
This parser uses multiple threads to speed up the parsing. This parser
requires access to Eclipse/Py4J'''
JAVA_SRC_FOLDER = 'src'
def __init__(self, codebase, project_key, opt_input):
'''
:param project_key: The name of the project in the Eclipse workspace.
:param codebase: The codebase instance to which the CodeElement will
be associated with.
:param opt_input: Optional input. Not used by this parser.
'''
self.project_name = project_key
self.gateway = JavaGateway()
self.hierarchies = deque() # list of tuples. [(parent, child, child)]
self.codebase = codebase
self.queue = Queue()
self.package_kind = CodeElementKind.objects.get(kind='package')
self.ASTParser = self.gateway.jvm.org.eclipse.jdt.core.dom.ASTParser
self.JLS3 = self.gateway.jvm.org.eclipse.jdt.core.dom.AST.JLS3
def _get_package_root(self):
ResourcePlugin = self.gateway.jvm.org.eclipse.core.resources.\
ResourcesPlugin
workspaceRoot = ResourcePlugin.getWorkspace().getRoot()
project = workspaceRoot.getProject(self.project_name)
java_project = self.gateway.jvm.org.eclipse.jdt.core.JavaCore.\
create(project)
src_folder = project.getFolder(JavaParser.JAVA_SRC_FOLDER)
proot = java_project.getPackageFragmentRoot(src_folder)
return proot
def _parse_packages(self, proot):
packages = []
for package in proot.getChildren():
if package.hasChildren():
package_name = package.getElementName()
package_code_element = CodeElement(codebase=self.codebase,
simple_name=package_name, fqn=package_name,
eclipse_handle=package.getHandleIdentifier(),
kind=self.package_kind, parser=JAVA_PARSER)
package_code_element.save()
packages.append((package, package_code_element))
return packages
def parse(self, progress_monitor=NullProgressMonitor()):
'''Parses the codebase and creates CodeElement instances.
:progress_monitor: A progress monitor to track the parsing progress.
'''
proot = self._get_package_root()
packages = self._parse_packages(proot)
progress_monitor.start('Parsing Java Project', len(packages))
# Start workers:
for _ in xrange(0, PARSER_WORKER):
worker = CUWorker(self.queue, self.codebase, self.hierarchies,
self.gateway, progress_monitor)
worker.start()
start = time.time()
for (package, package_code_element) in packages:
gc.collect() # for Py4J
cunits = package.getCompilationUnits()
unit_length = float(len(cunits))
for cunit in cunits:
ast_parser = self.ASTParser.newParser(self.JLS3)
ast_parser.setResolveBindings(True)
ast_parser.setSource(cunit)
cu = ast_parser.createAST(None)
winput = (cu, package_code_element, cunit.getElementName(),
1.0 / unit_length)
self.queue.put(winput)
for _ in xrange(0, PARSER_WORKER):
self.queue.put(None)
progress_monitor.info('Done parsing packages. Waiting for CUs.')
self.queue.join()
progress_monitor.done()
self.gateway.close()
print('Time: ' + str(time.time() - start))
self.parse_hierarchy(progress_monitor)
def parse_hierarchy(self, progress_monitor=NullProgressMonitor()):
'''Builds the hierarchy of the parsed CodeElement instances.
Must be called *after* parse.
:param progress_monitor:
'''
queue = Queue()
for hierarchy in self.hierarchies:
queue.put(hierarchy)
progress_monitor.start('Parsing Java Hierarchy', len(self.hierarchies))
start = time.time()
for _ in xrange(0, HIERARCHY_WORKER):
# Sentinel value
queue.put(None)
worker = HierarchyWorker(queue, self.codebase, progress_monitor)
worker.start()
queue.join()
progress_monitor.done()
self.gateway.close()
print('Time: ' + str(time.time() - start))
def testCallbackServer(self):
# A close is required to stop the thread.
gateway = JavaGateway(start_callback_server=True)
gateway.close()
self.assertTrue(True)
time.sleep(1)
class Py4jServiceBridge(object):
'''Py4jServiceBridge class
This class provides and API for consumers to use the Py4jServiceBridge. This
allows a bridge between Python and the OSGi service registry.
'''
def __init__(self,service_listener=None,connection_listener=None,gateway_parameters=None,callback_server_parameters=None):
self._gateway = None
self._lock = RLock()
self._consumer = None
self._imported_endpoints = {}
self._imported_endpoints_lock = RLock()
self._exported_endpoints_lock = RLock()
self._exported_endpoints = {}
self._map_converter = MapConverter()
self._list_converter = ListConverter()
self._service_listener = service_listener
self._connection_listener = connection_listener
self._connection = None
self._gateway_parameters = gateway_parameters
self._callback_server_parameters = callback_server_parameters
def get_id(self):
'''
Get the py4j id for this service bridge.
:return: The py4j id...e.g. py4j://localhost:23334/python for this service bridge
'''
cb = self._gateway.get_callback_server()
return createLocalPy4jId(hostname=cb.get_listening_address(),port=cb.get_listening_port())
def export(self,svc,export_props=None):
'''
Export the given svc via the given export_props. Note that the export_props must have and ENDPOINT_ID
and contain the other standard Endpoint Description service properties as described by the OSGI
R5+ Chapter 122 (Remote Service Admin) in the enterprise specification.
:param svc: The Python service to export to java service registry
:param export_props: An optional dictionary of Python properties. Note these properties
must contain all the properties describing the service as required by the OSGI Endpoint Description.
:return: The ENDPOINT_ID value from the export_props. This value may be used to get subsequent
access to the svc and/or the export_properties via get_export_endpoint
'''
with self._lock:
self._raise_not_connected()
'''The Java class attribute must be present'''
java = getattr(svc,PY4J_JAVA_ATTRIBUTE)
'''The Java.implements must be present'''
objClass = getattr(java,PY4J_JAVA_IMPLEMENTS_ATTRIBUTE)
if isinstance(objClass,str):
objClass = [objClass]
props = {}
'''The export_properties attribute does not need to be present'''
j_props = getattr(java,EXPORT_PROPERTIES_NAME,None)
if j_props:
props = osgiservicebridge.merge_dicts(props,j_props)
if export_props:
props = osgiservicebridge.merge_dicts(props,export_props)
sec = props.get(osgiservicebridge.REMOTE_CONFIGS_SUPPORTED,None)
if not sec:
sec = [PY4J_EXPORTED_CONFIG]
# check that OBJECTCLASS is set, if not
props1 = get_edef_props(object_class=objClass,exported_cfgs=sec,ep_namespace=None,ecf_ep_id=self.get_id(),ep_rsvc_id=None,ep_ts=None)
export_props = merge_dicts(props1, props)
try:
endpointid = export_props[ENDPOINT_ID]
except KeyError:
raise ArgumentError('Cannot export service since no ENDPOINT_ID present in export_props')
with self._exported_endpoints_lock:
self._exported_endpoints[endpointid] = (svc,export_props)
# without holding lock, call __export
try:
self.__export(svc,export_props)
except Exception as e:
# if it fails, remove from exported endpoints
self.remove_export_endpoint(endpointid)
raise e
return endpointid
def update(self,update_props):
'''
Update the previously exported svc with the given update_props. Note that the update_props must have and ENDPOINT_ID
and contain the other standard Endpoint Description service properties as described by the OSGI
R5+ Chapter 122 (Remote Service Admin) in the enterprise specification.
:param update_props: A dictionary of Python properties. Note that these properties must contain all
the properties describing the service as required by the OSGI Endpoint Description and must contain
an ENDPOINT_ID that matches the endpoint ID previously returned from export
:return: old endpoint props (dict) or None
:raise: ArgumentError if there is not an ENDPOINT_ID value in the update_props
'''
with self._lock:
self._raise_not_connected()
try:
endpointid = update_props[ENDPOINT_ID]
except KeyError:
raise ArgumentError('Cannot update service since no ENDPOINT_ID present in update_props')
# get lock and make sure oldendpoint is in exported_endpoints
with self._exported_endpoints_lock:
oldendpoint = self.get_export_endpoint(endpointid)
if oldendpoint:
self._exported_endpoints[endpointid] = (oldendpoint[0],update_props)
else:
return None
try:
self.__update(update_props)
except Exception as e:
# if exception, restore old endpoint
with self._exported_endpoints_lock:
self._exported_endpoints[endpointid] = oldendpoint
raise e
return oldendpoint[1]
def unexport(self,endpointid):
'''
Unexport the svc via the given export_props. Note that the export_props must have an ENDPOINT_ID
and contain the other standard Endpoint Description service properties as described by the OSGI
R5+ Chapter 122 (Remote Service Admin) in the enterprise specification.
:param export_props: A dictionary of Python properties. Note that these properties must contain all
the properties describing the service as required by the OSGI Endpoint Description.
:return: The endpoint service if successfully unexported, None if unsuccessfully
exported
'''
with self._lock:
self._raise_not_connected()
if endpointid:
endpoint = self.remove_export_endpoint(endpointid)
if endpoint:
try:
self.__unexport(endpoint[1])
except Exception as e:
with self._exported_endpoints_lock:
self._exported_endpoints = endpoint
raise e
return endpoint
def get_jvm(self):
'''
Get the Py4JService Gateway's JVM
:return: The Gateway's jvm
'''
return self._gateway.jvm
def get_gateway(self):
'''
Get the Py4JService JavaGateway
:return: The Gateway
'''
return self._gateway
def get_callback_server_parameters(self):
'''
Get the callback server parameters used for the gateway
:return: callback server parameters used for the gateway
'''
return self._gateway.callback_server_parameters
def get_import_endpoint(self,endpointid):
'''
Get import endpoint for given endpointid
:param endpointid: the endpoint id to find
:return: the tuple of the endpoint or None if no
endpoint exists for given endpointid. Element 0 is the
service, element 1 is the dictionary of properties.
'''
with self._imported_endpoints_lock:
return self._imported_endpoints.get(endpointid,None)
def get_export_endpoint(self,endpointid):
'''
Get export endpoint for given endpointid
:param endpointid: the endpoint id to find
:return: the tuple of the endpoint or None if no
endpoint exists for given endpointid. Element 0 is the
service, element 1 is the dictionary of properties.
'''
with self._exported_endpoints_lock:
return self._exported_endpoints.get(endpointid,None)
def get_export_endpoint_for_rsid(self,rsId):
with self._exported_endpoints_lock:
for eptuple in self._exported_endpoints.values():
val = eptuple[1][osgiservicebridge.ECF_RSVC_ID]
if not val is None and val == rsId:
return eptuple[0]
return None
def isconnected(self):
'''
Returns True if the gateway callback server is connected,
False if not
'''
with self._lock:
return True if self._gateway is not None else False
def connect(self,gateway_parameters=None,callback_server_parameters=None,path_hook=None):
'''
Connect gateway to Java side
:param gateway_parameters an overriding GatewayParameters instance.
If None then self._gateway_parameters is used. If that is None, then new GatewayParameters (defaults) is used
:param callback_server_parameters an overriding CallbackServerParameters instance.
If None then self._callback_server_parameters is used. If that is None, then new CallbackServerParameters (defaults) is used
'''
if callback_server_parameters:
self._callback_server_parameters = callback_server_parameters
else:
self._callback_server_parameters = self._callback_server_parameters if self._callback_server_parameters else CallbackServerParameters()
if gateway_parameters:
self._gateway_parameters = gateway_parameters
else:
self._gateway_parameters = self._gateway_parameters if self._gateway_parameters else GatewayParameters()
with self._lock:
if not self._gateway is None:
raise OSError('already connected to java gateway')
server_started.connect(self._started)
self._gateway = JavaGateway(gateway_parameters=self._gateway_parameters,callback_server_parameters=self._callback_server_parameters)
cbserver = self._gateway.get_callback_server()
server_stopped.connect(
self._stopped, sender=cbserver)
server_connection_started.connect(
self._connection_started,
sender=cbserver)
server_connection_stopped.connect(
self._connection_stopped,
sender=cbserver)
pre_server_shutdown.connect(
self._pre_shutdown, sender=cbserver)
post_server_shutdown.connect(
self._post_shutdown, sender=cbserver)
class JavaRemoteServiceDiscoverer(object):
def __init__(self, bridge):
self._bridge = bridge
def _external_discoverService(self,proxy,props):
self._bridge._import_service_from_java(proxy,props)
def _external_updateDiscoveredService(self,props):
self._bridge._modify_service_from_java(props)
def _external_undiscoverService(self,props):
self._bridge._unimport_service_from_java(props)
def _call_endpoint(self,rsId,methodName,serializedArgs):
endpoint = self._bridge.get_export_endpoint_for_rsid(rsId)
if endpoint:
return endpoint._raw_bytes_from_java(methodName,serializedArgs)
else:
msg = 'No endpoint for rsId=%s methodName=%s serializedArgs=%s' % (rsId,methodName,serializedArgs)
_logger.error(msg)
raise Exception(msg)
class Java:
implements = [JAVA_DIRECT_ENDPOINT_CLASS, PY4J_CALL_BY_VALUE_CLASS]
if not path_hook:
path_hook = JavaPathHook(self)
self._bridge = JavaRemoteServiceDiscoverer(self)
'''Call _getExternalDirectDiscovery first, so that we are ready to export'''
self._consumer = self._gateway.entry_point._getExternalDirectDiscovery()
'''Then call _setDirectBridge so that java side can now call us
to notify about exported services'''
path_list = self._gateway.entry_point._setDirectBridge(path_hook,self._bridge, self._bridge, self.get_id())
path_hook.initialize_path(path_list)
def get_module_type(self,modname):
with self._lock:
if not self.isconnected():
raise ImportError()
return self._gateway.entry_point.getModuleType(modname)
def get_module_code(self,modname,ispackage):
with self._lock:
if not self.isconnected():
raise ImportError()
return self._gateway.entry_point.getModuleCode(modname,ispackage)
def disconnect(self):
with self._lock:
if self.isconnected():
self._gateway.close()
self._gateway = None
self._consumer = None
self._bridge = None
with self._imported_endpoints_lock:
self._imported_endpoints.clear()
with self._exported_endpoints_lock:
self._exported_endpoints.clear()
def _started(self, sender, **kwargs):
if self._connection_listener:
self._connection_listener.started(kwargs["server"])
def _stopped(self, sender, **kwargs):
if self._connection_listener:
self._connection_listener.stopped(kwargs["server"])
def _connection_started(self, sender, **kwargs):
with self._lock:
self._connection = kwargs["connection"]
if self._connection_listener:
self._connection_listener.connection_started(self._connection)
def _connection_stopped(self, sender, **kwargs):
with self._lock:
self._connection = None
if self._connection_listener:
self._connection_listener.connection_stopped(kwargs['connection'],kwargs.pop('exception',None))
def _pre_shutdown(self, sender, **kwargs):
if self._connection_listener:
self._connection_listener.pre_shutdown(kwargs["server"])
def _post_shutdown(self, sender, **kwargs):
with self._imported_endpoints_lock:
for endpointid in self._imported_endpoints.keys():
endpoint = self._imported_endpoints.get(endpointid,None)
if self._service_listener and endpoint:
try:
self._service_listener.service_unimported(self, endpointid, endpoint[0], endpoint[1])
except:
_logger.exception('_unimport_service_from_java listener threw exception endpointid={0}'.format(endpointid))
self.disconnect();
if self._connection_listener:
self._connection_listener.post_shutdown(kwargs["server"])
# Methods called by java
def _import_service_from_java(self,proxy,props):
local_props = prepare_java_props(props)
endpointid = local_props.get(ENDPOINT_ID,None)
if endpointid:
endpoint = (proxy,local_props)
with self._imported_endpoints_lock:
self._imported_endpoints[endpointid] = endpoint
if self._service_listener:
try:
self._service_listener.service_imported(self, endpointid, endpoint[0], endpoint[1])
except:
_logger.exception('__import_service_from_java listener threw exception endpointid={0}'.format(endpointid))
def _modify_service_from_java(self,newprops):
newendpoint = None
python_newprops = self._prepare_props(newprops)
endpointid = python_newprops.get(ENDPOINT_ID,None)
if endpointid:
with self._imported_endpoints_lock:
oldendpoint = self._imported_endpoints[endpointid]
newendpoint = (oldendpoint[0],python_newprops)
self._imported_endpoints[endpointid] = newendpoint
if self._service_listener:
try:
self._service_listener.service_modified(self, endpointid, newendpoint[0], newendpoint[1])
except:
_logger.exception('__modify_service_from_java listener threw exception endpointid={0}'.format(endpointid))
def _unimport_service_from_java(self,props):
endpoint = None
local_props = prepare_java_props(props)
endpointid = local_props.get(ENDPOINT_ID,None)
if endpointid:
endpoint = self.remove_import_endpoint(endpointid)
if self._service_listener and endpoint:
try:
self._service_listener.service_unimported(self, endpointid, endpoint[0], endpoint[1])
except:
_logger.exception('__unimport_service_from_java listener threw exception endpointid={0}'.format(endpointid))
def remove_export_endpoint(self,endpointid):
with self._exported_endpoints_lock:
return self._exported_endpoints.pop(endpointid, None)
def remove_import_endpoint(self,endpointid):
with self._imported_endpoints_lock:
return self._imported_endpoints.pop(endpointid, None)
def _convert_string_list(self,l):
llength = len(l)
r = self._gateway.new_array(self._gateway.jvm.java.lang.String,llength)
for i in range(0,llength):
r[i] = l[i]
return r
def _convert_props_for_java(self,props):
with self._lock:
self._raise_not_connected()
result = {}
for item in props.items():
val = item[1]
if isinstance(val, type([])):
val = self._convert_string_list(val)
result[item[0]] = val
return self._map_converter.convert(result,self._gateway._gateway_client)
def _raise_not_connected(self):
if not self.isconnected():
raise OSError('Not connected to java gateway')
def __export(self,svc,props):
try:
self._consumer._java_discoverService(svc,self._convert_props_for_java(props))
except Exception as e:
_logger.error(e)
raise e
def __update(self,props):
try:
self._consumer._java_updateDiscoveredService(self._convert_props_for_java(props))
except Exception as e:
_logger.error(e)
raise e
def __unexport(self,props):
try:
self._consumer._java_undiscoverService(self._convert_props_for_java(props))
except Exception as e:
_logger.error(e)
raise e
class Java(object):
def __init__(self):
java_cmd = subprocess.check_output("which java", shell=True).rstrip()
try:
self.gateway = JavaGateway()
self.gateway.entry_point.getNumParseErrors("")
except py4j.protocol.Py4JNetworkError:
pass
else:
raise RuntimeError("Old java server still running")
if os.path.isfile(JAR_PATH):
pass
else:
py4j_jar = os.path.join(sys.prefix, 'share/py4j/py4j0.10.6.jar')
subprocess.check_call(
"mvn install:install-file -Dfile=" + py4j_jar +
" -DgroupId=py4j -DartifactId=py4j -Dversion=0.10.6 -Dpackaging=jar -DgeneratePom=true",
shell=True)
subprocess.check_call("mvn package", shell=True)
assert os.path.isfile(JAR_PATH)
self.java_server = subprocess.Popen([java_cmd, "-jar", JAR_PATH],
preexec_fn=os.setsid)
tries = 100
while True:
try:
time.sleep(0.1)
self.gateway = JavaGateway()
self.gateway.entry_point.getNumParseErrors("")
except py4j.protocol.Py4JNetworkError:
tries -= 1
if tries > 0:
continue
else:
raise
else:
break
def __del__(self):
#error("killing %i" % self.java_server.pid)
if hasattr(self, 'gateway'):
self.gateway.shutdown()
self.gateway.close()
del self.gateway
if hasattr(self, 'java_server'):
os.killpg(os.getpgid(self.java_server.pid), signal.SIGTERM)
self.java_server.wait()
def get_num_parse_errors(self, java_source):
return self.gateway.entry_point.getNumParseErrors(java_source)
@staticmethod
def fix_extra_quotes(lexeme):
l = tuple(lexeme)
def lex(self, java_source):
# WARNING: assumes lines not in mac format
lpos = 0
lines = [0]
while True:
lpos = java_source.find('\n', lpos) + 1
if lpos <= 0:
break
lines.append(lpos)
def convert_position(i):
line = bisect_right(lines, i)
col = i - lines[line - 1]
return (line, col)
def convert(lexeme):
t, v, s, e, string = tuple(lexeme)
assert not (' \n\r\t' in string)
return (t, v, convert_position(s), convert_position(e), string)
return [convert(l) for l in self.gateway.entry_point.lex(java_source)]
class NetLogoControllerServerStarter:
__gw = None # New gateway connection
##server is in netlogo app folder
def __init__(self):
self.__gw = JavaGateway()
print('Shutting down old server instance...')
self.shutdownServer()
#atexit.register(self.shutdownServer)
'''Internal method to start JavaGateway server. Will be called by starServer on seperate thread'''
def __runServer(self):
__server_name = "nl4py.server.NetLogoControllerServer"
try:
nl_path = os.environ['NETLOGO_APP']
except KeyError:
#looks like the NETLOGO_APP variable isn't set...
#Trying to use os dependent defaults
print("NETLOGO_APP was not set, trying to find NetLogo .jar files")
import platform
if (platform.system() == "Windows"):
nl_path = "C:/Program Files/NetLogo 6.0.2/app"
if (platform.system() == "Darwin"):
nl_path = "/Applications/NetLogo 6.0.2/Java"
pass
nl_path = os.path.join(os.path.abspath(nl_path), "*")
os.pathsep
server_path = "./server/*"
classpath = nl_path + os.pathsep + server_path
xmx = psutil.virtual_memory().available / 1024 / 1024 / 1024
xms = "-Xms" + str(int(math.floor(xmx - 2))) + "G"
xmx = "-Xmx" + str(int(math.floor(xmx))) + "G"
subprocess.call([
"java", xmx, "-XX:-UseGCOverheadLimit", "-cp", classpath,
__server_name
])
'''Starts JavaGateway server'''
def startServer(self):
#Fire up the NetLogo Controller server through python
thread = threading.Thread(target=self.__runServer, args=())
thread.start()
time.sleep(3)
'''Send shutdown signal to the JavaGateway server. No further communication is possible unless server is restarted'''
def shutdownServer(self):
if (self.__gw != None):
try:
logging.disable(logging.CRITICAL)
__bridge = self.__gw.entry_point
__bridge.shutdownServer()
self.__gw.close(keep_callback_server=False,
close_callback_server_connections=True)
self.__gw = None
except Exception as e:
pass
def run():
# ---------------------------- #
# PREPARE INPUT
# ---------------------------- #
# parse input args
parser = ArgumentParser(description='Create dashboard files')
parser.add_argument('ontology', type=str, help='Input ontology file')
parser.add_argument('registry', type=FileType('r'), help='Registry YAML file')
parser.add_argument('schema', type=FileType('r'), help='OBO JSON schema')
parser.add_argument('relations', type=FileType('r'), help='Table containing RO IRIs and labels')
parser.add_argument('profile', type=str, help='Optional location of profile.txt file.')
parser.add_argument('configfile', type=str, help='Location of the dashboard config file', default='build/robot.jar')
parser.add_argument('outdir', type=str, help='Output directory')
parser.add_argument('robot_jar',type=str,help='Location of your local ROBOT jar', default='build/robot.jar')
args = parser.parse_args()
config = DashboardConfig(args.configfile)
owl = os.path.basename(args.ontology)
namespace = os.path.splitext(owl)[0]
ontology_file = args.ontology
registry = args.registry
schema = json.load(args.schema)
contact_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://obofoundry.org/config/registry_schema/contact",
"title": "registry_schema",
"properties": {
"contact": schema['properties']['contact'],
},
"required": ["contact"],
"level": "error"
}
license_schema = {
"$schema": "http://json-schema.org/draft-07/schema#",
"$id": "http://obofoundry.org/config/registry_schema/license",
"title": "registry_schema",
"properties": {
"license": schema['properties']['license'],
},
"required": ["license"],
"level": "error"
}
robot_jar = args.robot_jar
ro_file = args.relations
profile = args.profile
# Create the build directory for this ontology
ontology_dir = args.outdir
os.makedirs(ontology_dir, exist_ok=True)
# Launch the JVM using the robot JAR
py4j.java_gateway.launch_gateway(
jarpath=robot_jar, classpath='org.obolibrary.robot.PythonOperation', die_on_exit=True, port=25333)
# Activate gateway to JVM
gateway = JavaGateway()
try:
robot_gateway = gateway.jvm.org.obolibrary.robot
# IOHelper for working with ontologies
io_helper = robot_gateway.IOHelper()
# Handle ontology file
big = namespace in BIG_ONTS
# Load dashboard data if exists
dashboard_yml = os.path.join(ontology_dir, "dashboard.yml")
data_yml = dict()
if os.path.isfile(dashboard_yml):
with open(dashboard_yml, 'r') as f:
data_yml = yaml.load(f, Loader=yaml.SafeLoader)
if 'changed' not in data_yml or 'results' not in data_yml or data_yml['changed'] == True:
print("Analysis has to be updated, running.")
else:
sys.exit(0)
if not big:
# Load ontology as OWLOntology object
if not ontology_file or not os.path.exists(ontology_file) or dash_utils.whitespace_only(ontology_file):
# If ontology_file is None, the file does not exist, or the file is empty
# Then the ontology is None
ont_or_file = None
else:
try:
ont_or_file = io_helper.loadOntology(ontology_file)
except Exception:
print('ERROR: Unable to load \'{0}\''.format(ontology_file), flush=True)
ont_or_file = None
# Get the Verison IRI
version_iri = dash_utils.get_version_iri(ont_or_file)
else:
# Just provide path to file
ont_or_file = ontology_file
# Get the version IRI by text parsing
version_iri = dash_utils.get_big_version_iri(ont_or_file)
# Get the registry data
yaml_data_raw = yaml.load(registry, Loader=yaml.SafeLoader)
yaml_data = []
for o in yaml_data_raw['ontologies']:
yaml_data.append(yaml_data_raw['ontologies'][o])
data = dash_utils.get_data(namespace, yaml_data)
# Map of all ontologies to their domains
domain_map = dash_utils.get_domains(yaml_data)
# Map of RO labels to RO IRIs
ro_props = fp_007.get_ro_properties(ro_file)
if 'is_obsolete' in data and data['is_obsolete'] == 'true':
# do not run on obsolete ontologies
print('{0} is obsolete and will not be checked...'.format(namespace), flush=True)
sys.exit(0)
# ---------------------------- #
# RUN CHECKS
# ---------------------------- #
print('-----------------\nChecking ' + namespace, flush=True)
# Get the report based on if it's big or not
report = None
good_format = None
if big:
if namespace != 'gaz':
# Report currently takes TOO LONG for GAZ
print('Running ROBOT report on {0}...'.format(namespace), flush=True)
report_obj = report_utils.BigReport(robot_gateway, namespace, ont_or_file, profile)
report = report_obj.get_report()
good_format = report_obj.get_good_format()
else:
if ont_or_file:
# Ontology is not None
print('Running ROBOT report on {0}...'.format(namespace), flush=True)
report = report_utils.run_report(robot_gateway, io_helper, ont_or_file, profile)
# Execute the numbered checks
check_map = {}
try:
if big:
check_map[1] = fp_001.big_is_open(ont_or_file, data, license_schema)
else:
check_map[1] = fp_001.is_open(ont_or_file, data, license_schema)
except Exception as e:
check_map[1] = 'INFO|unable to run check 1'
print('ERROR: unable to run check 1 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
if big:
check_map[2] = fp_002.big_is_common_format(good_format)
else:
check_map[2] = fp_002.is_common_format(ont_or_file)
except Exception as e:
check_map[2] = 'INFO|unable to run check 2'
print('ERROR: unable to run check 2 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
if big:
check_map[3] = fp_003.big_has_valid_uris(namespace, ont_or_file, ontology_dir)
else:
check_map[3] = fp_003.has_valid_uris(robot_gateway, namespace, ont_or_file, ontology_dir)
except Exception as e:
check_map[3] = 'INFO|unable to run check 3'
print('ERROR: unable to run check 3 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
if big:
check_map[4] = fp_004.big_has_versioning(ont_or_file)
else:
check_map[4] = fp_004.has_versioning(ont_or_file)
except Exception as e:
check_map[4] = 'INFO|unable to run check 4'
print('ERROR: unable to run check 4 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[5] = fp_005.has_scope(data, domain_map)
except Exception as e:
check_map[5] = 'INFO|unable to run check 5'
print('ERROR: unable to run check 5 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[6] = fp_006.has_valid_definitions(report)
except Exception as e:
check_map[6] = 'INFO|unable to run check 6'
print('ERROR: unable to run check 6 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
if big:
check_map[7] = fp_007.big_has_valid_relations(namespace, ont_or_file, ro_props, ontology_dir)
else:
check_map[7] = fp_007.has_valid_relations(namespace, ont_or_file, ro_props, ontology_dir)
except Exception as e:
check_map[7] = 'INFO|unable to run check 7'
print('ERROR: unable to run check 7 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[8] = fp_008.has_documentation(data)
except Exception as e:
check_map[8] = 'INFO|unable to run check 8'
print('ERROR: unable to run check 8 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[9] = fp_009.has_users(data)
except Exception as e:
check_map[9] = 'INFO|unable to run check 9'
print('ERROR: unable to run check 9 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[11] = fp_011.has_contact(data, contact_schema)
except Exception as e:
check_map[11] = 'INFO|unable to run check 11'
print('ERROR: unable to run check 11 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[12] = fp_012.has_valid_labels(report)
except Exception as e:
check_map[12] = 'INFO|unable to run check 12'
print('ERROR: unable to run check 12 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
if big:
check_map[16] = fp_016.big_is_maintained(ont_or_file)
else:
check_map[16] = fp_016.is_maintained(ont_or_file)
except Exception as e:
check_map[16] = 'INFO|unable to run check 16'
print('ERROR: unable to run check 16 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
try:
check_map[20] = fp_020.is_responsive(data)
except Exception as e:
check_map[20] = 'INFO|unable to run check 20'
print('ERROR: unable to run check 20 for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
# finally, add the report results to the dashboard and save the report
try:
check_map['report'] = report_utils.process_report(robot_gateway, report, ontology_dir)
except Exception as e:
check_map['report'] = 'INFO|unable to save report'
print('ERROR: unable to save ROBOT report for {0}\nCAUSE:\n{1}'.format(namespace, str(e)), flush=True)
# ---------------------------- #
# SAVE RESULTS
# ---------------------------- #
# Parse results
err = 0
warn = 0
info = 0
all_checks = {}
for check, result in check_map.items():
if result is None or 'status' not in result:
print('Missing result for check {0}'.format(check), flush=True)
continue
status = result['status']
if status == 'ERROR':
err += 1
elif status == 'WARN':
warn += 1
elif status == 'INFO':
info += 1
elif status != 'PASS':
print('Unknown status "{0}" for check {1}'.format(status, check), flush=True)
continue
key = check
if check in PRINCIPLE_MAP:
key = PRINCIPLE_MAP[check]
elif check == 'report':
key = 'ROBOT Report'
all_checks[key] = result
# Summary status
if err > 0:
summary = 'ERROR'
summary_comment = '{0} errors'.format(err)
elif warn > 0:
summary = 'WARN'
summary_comment = '{0} warnings'.format(warn)
elif info > 0:
summary = 'INFO'
summary_comment = '{0} info messages'.format(info)
else:
summary = 'PASS'
summary_comment = ''
summary_count = dict()
summary_count['ERROR'] = err
summary_count['WARN'] = warn
summary_count['INFO'] = info
date = datetime.datetime.today()
save_data = {'namespace': namespace, 'version': version_iri,
'summary': {'status': summary, 'comment': summary_comment, 'summary_count': summary_count}, 'results': all_checks}
oboscore_weights = config.get_oboscore_weights()
oboscore_maximpacts = config.get_oboscore_max_impact()
for key in save_data:
data_yml[key] = save_data[key]
data_yml['metrics']['Info: Experimental OBO score']['_dashboard'] = round_float(
float(compute_dashboard_score(data_yml, oboscore_weights, oboscore_maximpacts)) / 100)
oboscore = compute_obo_score(data_yml['metrics']['Info: Experimental OBO score']['_impact'],
data_yml['metrics']['Info: Experimental OBO score']['_reuse'],
data_yml['metrics']['Info: Experimental OBO score']['_dashboard'],
data_yml['metrics']['Info: Experimental OBO score']['_impact_external'],
oboscore_weights)
data_yml['metrics']['Info: Experimental OBO score']['oboscore'] = round_float(oboscore['score'])
data_yml['metrics']['Info: Experimental OBO score']['_formula'] = oboscore['formula']
# Save to YAML file
print('Saving results to {0}'.format(dashboard_yml))
with open(dashboard_yml, 'w+') as f:
yaml.dump(data_yml, f)
except Exception:
logging.exception(f"Creating dashboard for {ontology_file} failed")
try:
gateway.close()
except Exception:
pass
sys.exit(0)
# This script tests the star brightness commands. # Created by Toni Sagrista from py4j.java_gateway import JavaGateway, GatewayParameters gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True)) gs = gateway.entry_point gs.setStarBrightness(100.0) gs.sleep(2) gs.setStarBrightness(70.0) gs.sleep(2) gs.setStarBrightness(50.0) gs.sleep(2) gs.setStarBrightness(30.0) gs.sleep(2) gs.setStarBrightness(12.0) gs.sleep(2) gateway.close()
def testCallbackServer(self):
# A close is required to stop the thread.
gateway = JavaGateway(start_callback_server=True)
gateway.close()
self.assertTrue(True)
time.sleep(1)