def __build_startup_command(self):
command = ["java", "-cp"]
root = os.environ.get("SYSTEMDS_ROOT")
if root == None:
# If there is no systemds install default to use the PIP packaged java files.
root = os.path.join(get_module_dir(), "systemds-java")
# nt means its Windows
cp_separator = ";" if os.name == "nt" else ":"
if os.environ.get("SYSTEMDS_ROOT") != None:
lib_cp = os.path.join(root, "target", "lib", "*")
systemds_cp = os.path.join(root, "target", "SystemDS.jar")
classpath = cp_separator.join([lib_cp, systemds_cp])
command.append(classpath)
files = glob(os.path.join(root, "conf", "log4j*.properties"))
if len(files) > 1:
print("WARNING: Multiple logging files found selecting: " +
files[0])
if len(files) == 0:
print("WARNING: No log4j file found at: " +
os.path.join(root, "conf") +
" therefore using default settings")
else:
command.append("-Dlog4j.configuration=file:" + files[0])
else:
lib_cp = os.path.join(root, "lib", "*")
command.append(lib_cp)
command.append("org.apache.sysds.api.PythonDMLScript")
return command
def __init__(self):
global PROCESS_LOCK
global PROCESS
global ACTIVE_PROCESS_CONNECTIONS
# make sure that a process is only started if necessary and no other thread
# is killing the process while the connection is established
PROCESS_LOCK.acquire()
try:
# attempt connection to manually started java instance
self._java_gateway = JavaGateway(eager_load=True)
except Py4JNetworkError:
# if no java instance is running start it
systemds_java_path = os.path.join(get_module_dir(),
'systemds-java')
cp_separator = ':'
if os.name == 'nt': # nt means its Windows
cp_separator = ';'
lib_cp = os.path.join(systemds_java_path, 'lib', '*')
systemds_cp = os.path.join(systemds_java_path, '*')
classpath = cp_separator.join([lib_cp, systemds_cp])
process = subprocess.Popen([
'java', '-cp', classpath,
'org.apache.sysds.pythonapi.PythonDMLScript'
],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE)
process.stdout.readline(
) # wait for 'Gateway Server Started\n' written by server
assert process.poll() is None, "Could not start JMLC server"
self._java_gateway = JavaGateway()
PROCESS = process
if PROCESS is not None:
ACTIVE_PROCESS_CONNECTIONS += 1
PROCESS_LOCK.release()
def __init__(self):
"""Starts a new instance of SystemDSContext, in which the connection to a JVM systemds instance is handled
Any new instance of this SystemDS Context, would start a separate new JVM.
Standard out and standard error form the JVM is also handled in this class, filling up Queues,
that can be read from to get the printed statements from the JVM.
"""
systemds_java_path = os.path.join(get_module_dir(), "systemds-java")
# nt means its Windows
cp_separator = ";" if os.name == "nt" else ":"
lib_cp = os.path.join(systemds_java_path, "lib", "*")
systemds_cp = os.path.join(systemds_java_path, "*")
classpath = cp_separator.join([lib_cp, systemds_cp])
# TODO make use of JavaHome env-variable if set to find java, instead of just using any java available.
command = ["java", "-cp", classpath]
sys_root = os.environ.get("SYSTEMDS_ROOT")
if sys_root != None:
files = glob(os.path.join(sys_root, "conf", "log4j*.properties"))
if len(files) > 1:
print("WARNING: Multiple logging files")
if len(files) == 0:
print("WARNING: No log4j file found at: "
+ os.path.join(sys_root, "conf")
+ " therefore using default settings")
else:
# print("Using Log4J file at " + files[0])
command.append("-Dlog4j.configuration=file:" + files[0])
else:
print("Default Log4J used, since environment $SYSTEMDS_ROOT not set")
command.append("org.apache.sysds.api.PythonDMLScript")
# TODO add an argument parser here
# Find a random port, and hope that no other process
# steals it while we wait for the JVM to startup
port = self.__get_open_port()
command.append(str(port))
process = Popen(command, stdout=PIPE, stdin=PIPE, stderr=PIPE)
first_stdout = process.stdout.readline()
assert (b"Server Started" in first_stdout), "Error JMLC Server not Started"
# Handle Std out from the subprocess.
self.__stdout = Queue()
self.__stderr = Queue()
Thread(target=self.__enqueue_output, args=(
process.stdout, self.__stdout), daemon=True).start()
Thread(target=self.__enqueue_output, args=(
process.stderr, self.__stderr), daemon=True).start()
# Py4j connect to the started process.
gateway_parameters = GatewayParameters(
port=port, eager_load=True, read_timeout=5)
self.java_gateway = JavaGateway(
gateway_parameters=gateway_parameters, java_process=process)
def __build_startup_command(self, port: int):
command = ["java", "-cp"]
root = os.environ.get("SYSTEMDS_ROOT")
if root == None:
# If there is no systemds install default to use the PIP packaged java files.
root = os.path.join(get_module_dir())
# nt means its Windows
cp_separator = ";" if os.name == "nt" else ":"
if os.environ.get("SYSTEMDS_ROOT") != None:
lib_release = os.path.join(root, "lib")
lib_cp = os.path.join(root, "target", "lib")
if os.path.exists(lib_release):
classpath = cp_separator.join([os.path.join(lib_release, '*')])
elif os.path.exists(lib_cp):
systemds_cp = os.path.join(root, "target", "SystemDS.jar")
classpath = cp_separator.join(
[os.path.join(lib_cp, '*'), systemds_cp])
else:
raise ValueError(
"Invalid setup at SYSTEMDS_ROOT env variable path")
else:
lib1 = os.path.join(root, "lib", "*")
lib2 = os.path.join(root, "lib")
classpath = cp_separator.join([lib1, lib2])
command.append(classpath)
files = glob(os.path.join(root, "conf", "log4j*.properties"))
if len(files) > 1:
print(
"WARNING: Multiple logging files found selecting: " + files[0])
if len(files) == 0:
print("WARNING: No log4j file found at: "
+ os.path.join(root, "conf")
+ " therefore using default settings")
else:
command.append("-Dlog4j.configuration=file:" + files[0])
command.append("org.apache.sysds.api.PythonDMLScript")
files = glob(os.path.join(root, "conf", "SystemDS*.xml"))
if len(files) > 1:
print(
"WARNING: Multiple config files found selecting: " + files[0])
if len(files) == 0:
print("WARNING: No log4j file found at: "
+ os.path.join(root, "conf")
+ " therefore using default settings")
else:
command.append("-config")
command.append(files[0])
if port == -1:
actual_port = self.__get_open_port()
else:
actual_port = port
command.append("--python")
command.append(str(actual_port))
return command, actual_port
def __init__(self):
"""Starts a new instance of SystemDSContext, in which the connection to a JVM systemds instance is handled
Any new instance of this SystemDS Context, would start a separate new JVM.
Standard out and standard error form the JVM is also handled in this class, filling up Queues,
that can be read from to get the printed statements from the JVM.
"""
root = os.environ.get("SYSTEMDS_ROOT")
if root == None:
# If there is no systemds install default to use the PIP packaged java files.
root = os.path.join(get_module_dir(), "systemds-java")
# nt means its Windows
cp_separator = ";" if os.name == "nt" else ":"
lib_cp = os.path.join(root, "target","lib", "*")
systemds_cp = os.path.join(root,"target","SystemDS.jar")
classpath = cp_separator.join([lib_cp , systemds_cp])
command = ["java", "-cp", classpath]
if os.environ.get("SYSTEMDS_ROOT") != None:
files = glob(os.path.join(root, "conf", "log4j*.properties"))
if len(files) > 1:
print("WARNING: Multiple logging files found selecting: " + files[0])
if len(files) == 0:
print("WARNING: No log4j file found at: "
+ os.path.join(root, "conf")
+ " therefore using default settings")
else:
command.append("-Dlog4j.configuration=file:" + files[0])
command.append("org.apache.sysds.api.PythonDMLScript")
# TODO add an argument parser here
# Find a random port, and hope that no other process
# steals it while we wait for the JVM to startup
port = self.__get_open_port()
command.append(str(port))
process = Popen(command, stdout=PIPE, stdin=PIPE, stderr=PIPE)
first_stdout = process.stdout.readline()
if(b"GatewayServer Started" in first_stdout):
print("Startup success")
else:
stderr = process.stderr.readline().decode("utf-8")
if(len(stderr) > 1):
raise Exception("Exception in startup of GatewayServer: " + stderr)
outputs = []
outputs.append(first_stdout.decode("utf-8"))
max_tries = 10
for i in range(max_tries):
next_line = process.stdout.readline()
if(b"GatewayServer Started" in next_line):
print("WARNING: Stdout corrupted by prints: " + str(outputs))
print("Startup success")
break
else:
outputs.append(next_line)
if (i == max_tries-1):
raise Exception("Error in startup of systemDS gateway process: \n gateway StdOut: " + str(outputs) + " \n gateway StdErr" + process.stderr.readline().decode("utf-8") )
# Handle Std out from the subprocess.
self.__stdout = Queue()
self.__stderr = Queue()
Thread(target=self.__enqueue_output, args=(
process.stdout, self.__stdout), daemon=True).start()
Thread(target=self.__enqueue_output, args=(
process.stderr, self.__stderr), daemon=True).start()
# Py4j connect to the started process.
gateway_parameters = GatewayParameters(
port=port, eager_load=True, read_timeout=5)
self.java_gateway = JavaGateway(
gateway_parameters=gateway_parameters, java_process=process)