def run(self, cmdline, name=None, cwd=None, print_output=False):
env_name = name if name else self.name
if print_output:
cmd_print_list = ["conda", "list", "-n", env_name]
print("PRINTING LIST OF PACKAGES")
execute_process(cmd_print_list, print_output=True)
if cwd:
# run_command doesn't have cwd
execute_process(
self._add_full_conda_execution(cmdline, env_name),
cwd=cwd,
print_output=print_output,
)
else:
print("CMD: ", " ".join(cmdline))
_, _, return_code = run_command(
Commands.RUN,
self._add_conda_execution(cmdline, env_name),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
use_exception_handler=True,
)
if return_code != 0:
raise Exception(f"Conda run returned {return_code}.")
def create(
self, existence_check=False, name=None, requirements_file=None, python_version=None
):
env_name = name if name else self.name
if self.is_env_exist(env_name):
if existence_check:
print("USING EXISTING ENVIRONMENT")
return
else:
self.remove(env_name)
if python_version:
cmdline_create = [
"conda",
"create",
"--name",
env_name,
f"python={python_version}" if python_version else "",
"-y" if python_version else "",
]
print("CREATING CONDA ENVIRONMENT")
execute_process(cmdline_create, print_output=False)
cmdline = [
"conda",
"env",
"update" if python_version else "create",
"--name",
env_name,
f"--file={requirements_file}" if requirements_file else "",
]
print(f"{'UPDATING' if python_version else 'CREATING'} CONDA ENVIRONMENT")
execute_process(cmdline, print_output=False)
def is_env_exist(self, name=None):
env_name = name if name else self.name
envs_list_cmdline = ["conda", "env", "list"]
_, output = execute_process(envs_list_cmdline)
envs = re.findall(r"[^\s]+", output)
if env_name in envs:
return True
return False
def launch(self):
"Launch OmniSciDB server"
print("Launching server ...")
self.server_process, _ = execute_process(self._server_start_cmdline,
cwd=self._server_cwd,
daemon=True)
print("Server is launched")
try:
pt = threading.Thread(
target=self._print_omnisci_output,
args=(self.server_process.stdout, ),
daemon=True,
)
pt.start()
# Allow server to start up. It has to open TCP port and start
# listening, otherwise the following benchmarks fail.
time.sleep(5)
except Exception as err:
print("Failed", err)
raise
def __init__(
self,
omnisci_executable,
omnisci_port,
database_name,
http_port,
calcite_port,
omnisci_cwd=None,
user="******",
password="******",
max_session_duration=86400,
idle_session_duration=120,
debug_timer=False,
columnar_output=True,
lazy_fetch=None,
multifrag_rs=None,
omnisci_run_kwargs={},
):
if not os.path.isdir(omnisci_executable) and not os.access(
omnisci_executable, os.X_OK):
raise ValueError("Invalid omnisci executable given: " +
omnisci_executable)
self.omnisci_executable = omnisci_executable
self.server_port = omnisci_port
self.user = user
self.password = password
self.database_name = database_name
self._http_port = http_port
self._calcite_port = calcite_port
self._max_session_duration = max_session_duration
self._idle_session_duration = idle_session_duration
if omnisci_cwd is not None:
self._server_cwd = omnisci_cwd
else:
self._server_cwd = pathlib.Path(
self.omnisci_executable).parent.parent
self._data_dir = os.path.join(self._server_cwd, "data")
if not os.path.isdir(self._data_dir):
print("CREATING DATA DIR", self._data_dir)
os.makedirs(self._data_dir)
if not os.path.isdir(os.path.join(self._data_dir, "mapd_data")):
print("INITIALIZING DATA DIR", self._data_dir)
self._initdb_executable = os.path.join(
pathlib.Path(self.omnisci_executable).parent, "initdb")
execute_process(
[self._initdb_executable, "-f", "--data", self._data_dir])
self.omnisci_sql_executable = os.path.join(
pathlib.Path(self.omnisci_executable).parent, "omnisql")
self._server_start_cmdline = [
self.omnisci_executable,
"data",
"--port",
str(omnisci_port),
"--http-port",
str(self._http_port),
"--calcite-port",
str(self._calcite_port),
"--config",
"omnisci.conf",
"--enable-watchdog=false",
"--allow-cpu-retry",
"--max-session-duration",
str(self._max_session_duration),
"--idle-session-duration",
str(self._idle_session_duration),
]
if debug_timer is not None:
self._server_start_cmdline.append(
"--enable-debug-timer=%s" %
("true" if debug_timer else "false"))
if columnar_output is not None:
self._server_start_cmdline.append(
"--enable-columnar-output=%s" %
("true" if columnar_output else "false"))
if lazy_fetch is not None:
self._server_start_cmdline.append(
"--enable-lazy-fetch=%s" % ("true" if lazy_fetch else "false"))
if multifrag_rs is not None:
self._server_start_cmdline.append(
"--enable-multifrag-rs=%s" %
("true" if multifrag_rs else "false"))
for key, value in omnisci_run_kwargs.items():
self._server_start_cmdline.append(f"--{key}={value}")
def remove(self, name=None):
env_name = name if name else self.name
print("REMOVING CONDA ENVIRONMENT")
remove_env_cmdline = ["conda", "env", "remove", "--name", env_name]
execute_process(remove_env_cmdline)