def main(): if "ORCHEST_STEP_UUID" not in os.environ: raise Exception("No ORCHEST_STEP_UUID passed as environment variable.") if "ORCHEST_PIPELINE_UUID" not in os.environ: raise Exception( "No ORCHEST_PIPELINE_UUID passed as environment variable.") # index 1 contains filename if len(sys.argv) < 3: raise Exception( "Should pass in the working directory (relative to the project dir) and " "filename (relative to the project dir) that you want to execute.") step_uuid = os.environ.get("ORCHEST_STEP_UUID") pipeline_uuid = os.environ.get("ORCHEST_PIPELINE_UUID") # sys.argv[1] contains the working directory relative to # the project dir working_dir = os.path.join(Config.PROJECT_DIR, sys.argv[1]) # sys.argv[2] contains the relative file path # (relative to the project directory) filename = sys.argv[2] file_extension = get_filename_extension(filename) file_path = os.path.join(Config.PROJECT_DIR, filename) # check if file exists in working directory if not os.path.isfile(file_path): raise Exception("File doesn't appear to exist in file path '%s'" % (file_path, )) if file_extension == "ipynb": nr = NotebookRunner(pipeline_uuid, step_uuid, working_dir) nr.run(file_path) elif file_extension in ["py", "r", "sh", "jl", ""]: extension_script_mapping = { "py": "python3", "r": "Rscript", "sh": "sh", "jl": "julia", "": "sh", } pr = ProcessRunner(pipeline_uuid, step_uuid, working_dir) sys.exit(pr.run(extension_script_mapping[file_extension], file_path)) else: raise Exception( "Running files with '%s' extension is not yet supported." % (file_extension, ))
def test_cell_output_format(): nr = NotebookRunner("pipeline_uuid", "step_uuid", Config.PROJECT_DIR) # don't write to notebook file in test nr.write_after_run = False nr.run(os.path.join(Config.PROJECT_DIR, "../test-files/loop.ipynb")) # check log file with open( os.path.join(Config.PROJECT_DIR, ".orchest/pipeline_uuid/logs/step_uuid.log"), "r", ) as f: contents = f.read() # skip first line as it is a random uuid contents = "\n".join(contents.split("\n")[1:]) assert contents.strip() == "[1] 0\n1\n2\n3\n4"
def main(): if "STEP_UUID" not in os.environ: raise Exception("No STEP_UUID passed as environment variable.") # index 1 contains filename if len(sys.argv) < 2: raise Exception( "Should pass in the filename that you want to execute.") step_uuid = os.environ.get("STEP_UUID") filename = sys.argv[1] file_extension = filename.split(".")[-1].lower() file_path = os.path.join(Config.WORKING_DIR, filename) # check if file exists in working directory if not os.path.isfile(file_path): raise Exception("File doesn't appear to exist in file path '%s'" % (file_path, )) if file_extension == "ipynb": nr = NotebookRunner(step_uuid) nr.run(file_path) elif file_extension in ["py", "r", "sh"]: extension_script_mapping = { "py": "python3", "r": "Rscript", "sh": "sh" } pr = ProcessRunner(step_uuid) sys.exit(pr.run(extension_script_mapping[file_extension], filename)) else: raise Exception( "Running files with '%s' extension is not yet supported." % (file_extension, ))
def test_cell_output_format(): Config.WORKING_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "test-outputs") nr = NotebookRunner("") # don't write to notebook file in test nr.write_after_run = False nr.run("tests/test-files/loop.ipynb") # check log file with open("tests/test-outputs/.orchest/logs/.log", 'r') as f: contents = f.read() # skip first line as it is a random uuid contents = '\n'.join(contents.split("\n")[1:]) assert contents.strip() == "[1] 0\n1\n2\n3\n4" # clean up test outputs shutil.rmtree("tests/test-outputs/")
def main(): if "ORCHEST_STEP_UUID" not in os.environ: raise Exception("No ORCHEST_STEP_UUID passed as environment variable.") if "ORCHEST_PIPELINE_UUID" not in os.environ: raise Exception( "No ORCHEST_PIPELINE_UUID passed as environment variable.") # index 1 contains filename if len(sys.argv) < 3: raise Exception( "Should pass in the working directory (relative to the project dir) and " "filename (relative to the project dir) that you want to execute.") step_uuid = os.environ.get("ORCHEST_STEP_UUID") pipeline_uuid = os.environ.get("ORCHEST_PIPELINE_UUID") # sys.argv[1] contains the working directory relative to # the project dir working_dir = os.path.join(Config.PROJECT_DIR, sys.argv[1]) # sys.argv[2] contains the relative file path # (relative to the project directory) filename = sys.argv[2] file_extension = get_filename_extension(filename) file_path = os.path.join(Config.PROJECT_DIR, filename) # check if file exists in working directory if not os.path.isfile(file_path): raise Exception("File doesn't appear to exist in file path '%s'" % (file_path, )) if file_extension == "ipynb": nr = NotebookRunner(pipeline_uuid, step_uuid, working_dir) nr.run(file_path) elif file_extension in ["py", "r", "sh", "jl", ""]: # Explicitely use the Python that contains the user dependencies # (otherwise Popen could resolve to the Python inside the # "orchestdependencies" environment). python_env = os.environ.get("CONDA_ENV") if python_env == "base": python_env = "/opt/conda" else: python_env = f"/opt/conda/envs/{python_env}" extension_script_mapping = { "py": f"{python_env}/bin/python", "r": "Rscript", # Invoke using bash as it is already present in the image # and users are likely to use bash functionality instead of # strict sh. "sh": "bash", "jl": "julia", "": "bash", } pr = ProcessRunner(pipeline_uuid, step_uuid, working_dir) sys.exit(pr.run(extension_script_mapping[file_extension], file_path)) else: raise Exception( "Running files with '%s' extension is not yet supported." % (file_extension, ))