def main():
if "ORCHEST_STEP_UUID" not in os.environ:
raise Exception("No ORCHEST_STEP_UUID passed as environment variable.")
if "ORCHEST_PIPELINE_UUID" not in os.environ:
raise Exception(
"No ORCHEST_PIPELINE_UUID passed as environment variable.")
# index 1 contains filename
if len(sys.argv) < 3:
raise Exception(
"Should pass in the working directory (relative to the project dir) and "
"filename (relative to the project dir) that you want to execute.")
step_uuid = os.environ.get("ORCHEST_STEP_UUID")
pipeline_uuid = os.environ.get("ORCHEST_PIPELINE_UUID")
# sys.argv[1] contains the working directory relative to
# the project dir
working_dir = os.path.join(Config.PROJECT_DIR, sys.argv[1])
# sys.argv[2] contains the relative file path
# (relative to the project directory)
filename = sys.argv[2]
file_extension = get_filename_extension(filename)
file_path = os.path.join(Config.PROJECT_DIR, filename)
# check if file exists in working directory
if not os.path.isfile(file_path):
raise Exception("File doesn't appear to exist in file path '%s'" %
(file_path, ))
if file_extension == "ipynb":
nr = NotebookRunner(pipeline_uuid, step_uuid, working_dir)
nr.run(file_path)
elif file_extension in ["py", "r", "sh", "jl", ""]:
extension_script_mapping = {
"py": "python3",
"r": "Rscript",
"sh": "sh",
"jl": "julia",
"": "sh",
}
pr = ProcessRunner(pipeline_uuid, step_uuid, working_dir)
sys.exit(pr.run(extension_script_mapping[file_extension], file_path))
else:
raise Exception(
"Running files with '%s' extension is not yet supported." %
(file_extension, ))
def test_cell_output_format():
nr = NotebookRunner("pipeline_uuid", "step_uuid", Config.PROJECT_DIR)
# don't write to notebook file in test
nr.write_after_run = False
nr.run(os.path.join(Config.PROJECT_DIR, "../test-files/loop.ipynb"))
# check log file
with open(
os.path.join(Config.PROJECT_DIR, ".orchest/pipeline_uuid/logs/step_uuid.log"),
"r",
) as f:
contents = f.read()
# skip first line as it is a random uuid
contents = "\n".join(contents.split("\n")[1:])
assert contents.strip() == "[1] 0\n1\n2\n3\n4"
def main():
if "STEP_UUID" not in os.environ:
raise Exception("No STEP_UUID passed as environment variable.")
# index 1 contains filename
if len(sys.argv) < 2:
raise Exception(
"Should pass in the filename that you want to execute.")
step_uuid = os.environ.get("STEP_UUID")
filename = sys.argv[1]
file_extension = filename.split(".")[-1].lower()
file_path = os.path.join(Config.WORKING_DIR, filename)
# check if file exists in working directory
if not os.path.isfile(file_path):
raise Exception("File doesn't appear to exist in file path '%s'" %
(file_path, ))
if file_extension == "ipynb":
nr = NotebookRunner(step_uuid)
nr.run(file_path)
elif file_extension in ["py", "r", "sh"]:
extension_script_mapping = {
"py": "python3",
"r": "Rscript",
"sh": "sh"
}
pr = ProcessRunner(step_uuid)
sys.exit(pr.run(extension_script_mapping[file_extension], filename))
else:
raise Exception(
"Running files with '%s' extension is not yet supported." %
(file_extension, ))
def test_cell_output_format():
Config.WORKING_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "test-outputs")
nr = NotebookRunner("")
# don't write to notebook file in test
nr.write_after_run = False
nr.run("tests/test-files/loop.ipynb")
# check log file
with open("tests/test-outputs/.orchest/logs/.log", 'r') as f:
contents = f.read()
# skip first line as it is a random uuid
contents = '\n'.join(contents.split("\n")[1:])
assert contents.strip() == "[1] 0\n1\n2\n3\n4"
# clean up test outputs
shutil.rmtree("tests/test-outputs/")
def main():
if "ORCHEST_STEP_UUID" not in os.environ:
raise Exception("No ORCHEST_STEP_UUID passed as environment variable.")
if "ORCHEST_PIPELINE_UUID" not in os.environ:
raise Exception(
"No ORCHEST_PIPELINE_UUID passed as environment variable.")
# index 1 contains filename
if len(sys.argv) < 3:
raise Exception(
"Should pass in the working directory (relative to the project dir) and "
"filename (relative to the project dir) that you want to execute.")
step_uuid = os.environ.get("ORCHEST_STEP_UUID")
pipeline_uuid = os.environ.get("ORCHEST_PIPELINE_UUID")
# sys.argv[1] contains the working directory relative to
# the project dir
working_dir = os.path.join(Config.PROJECT_DIR, sys.argv[1])
# sys.argv[2] contains the relative file path
# (relative to the project directory)
filename = sys.argv[2]
file_extension = get_filename_extension(filename)
file_path = os.path.join(Config.PROJECT_DIR, filename)
# check if file exists in working directory
if not os.path.isfile(file_path):
raise Exception("File doesn't appear to exist in file path '%s'" %
(file_path, ))
if file_extension == "ipynb":
nr = NotebookRunner(pipeline_uuid, step_uuid, working_dir)
nr.run(file_path)
elif file_extension in ["py", "r", "sh", "jl", ""]:
# Explicitely use the Python that contains the user dependencies
# (otherwise Popen could resolve to the Python inside the
# "orchestdependencies" environment).
python_env = os.environ.get("CONDA_ENV")
if python_env == "base":
python_env = "/opt/conda"
else:
python_env = f"/opt/conda/envs/{python_env}"
extension_script_mapping = {
"py": f"{python_env}/bin/python",
"r": "Rscript",
# Invoke using bash as it is already present in the image
# and users are likely to use bash functionality instead of
# strict sh.
"sh": "bash",
"jl": "julia",
"": "bash",
}
pr = ProcessRunner(pipeline_uuid, step_uuid, working_dir)
sys.exit(pr.run(extension_script_mapping[file_extension], file_path))
else:
raise Exception(
"Running files with '%s' extension is not yet supported." %
(file_extension, ))