def write_dax_to_dir(self, output_xml_dir: Optional[Path] = None) -> Path: if not output_xml_dir: output_xml_dir = self._workflow_directory dax_file_name = f"{self.name}.dax" dax_file = output_xml_dir / dax_file_name logging.info("Writing DAX to %s", dax_file) with dax_file.open("w") as dax: self._job_graph.writeXML(dax) build_submit_script(output_xml_dir / "submit.sh", dax_file_name, self._workflow_directory) # We also need to write sites.xml and pegasus.conf sites_xml_path = output_xml_dir / "sites.xml" sites_xml_path.write_text(pkg_resources.read_text( resources, "sites.xml"), encoding="utf-8") pegasus_conf_path = output_xml_dir / "pegasus.conf" pegasus_conf_path.write_text( data=pkg_resources.read_text(resources, "pegasus.conf") + "pegasus.catalog.replica=File\n" + f"pegasus.catalog.replica.file={self._replica_catalog}\n" + self._conf_limits(), encoding="utf-8", ) return dax_file
def write_dax_to_dir(self, output_xml_dir: Optional[Path] = None) -> Path: if not output_xml_dir: output_xml_dir = self._workflow_directory num_jobs = len(self._job_graph.jobs.keys()) num_ckpts = len( [ckpt_file for ckpt_file in output_xml_dir.rglob("___ckpt")]) if num_jobs == num_ckpts: nuke = input( "DAX *may* create a NOOP workflow. Do you want to nuke the checkpoints and regenerate? [y/n]" ) if nuke == "y": self._nuke_checkpoints_and_clear_rc(output_xml_dir) logging.info("Checkpoints cleared!") dax_file_name = f"{self.name}.dax" dax_file = output_xml_dir / dax_file_name logging.info("Writing DAX to %s", dax_file) with dax_file.open("w") as dax: self._job_graph.write(dax) build_submit_script(output_xml_dir / "submit.sh", dax_file_name, self._workflow_directory) # Write Out Sites Catalog sites_yml_path = output_xml_dir / "sites.yml" with sites_yml_path.open("w") as sites: self._sites_catalog.write(sites) self._properties["pegasus.catalog.site.file"] = str( sites_yml_path.absolute()) # Write Out Replica Catalog replica_yml_path = output_xml_dir / "replicas.yml" with replica_yml_path.open("w") as replicas: self._replica_catalog.write(replicas) self._properties["pegasus.catalog.replica"] = "YAML" self._properties["pegasus.catalog.replica.file"] = str( replica_yml_path.absolute()) # Write Out Transformation Catalog transformation_yml_path = output_xml_dir / "transformations.yml" with transformation_yml_path.open("w") as transformations: self._transformation_catalog.write(transformations) self._properties["pegasus.catalog.transformation"] = "YAML" self._properties["pegasus.catalog.transformation.file"] = str( transformation_yml_path.absolute()) # Write Out Pegasus Properties self._conf_limits() pegasus_conf_path = output_xml_dir / "pegasus.properties" with pegasus_conf_path.open("w") as properties: self._properties.write(properties) return dax_file
def test_dax_with_job_on_saga(tmp_path): workflow_params = Parameters.from_mapping({ "workflow_name": "Test", "workflow_created": "Testing", "workflow_log_dir": str(tmp_path / "log"), "workflow_directory": str(tmp_path / "working"), "site": "saga", "namespace": "test", "partition": "scavenge", }) slurm_params = Parameters.from_mapping({ "partition": "scavenge", "num_cpus": 1, "num_gpus": 0, "memory": "4G" }) multiply_input_file = tmp_path / "raw_nums.txt" random = Random() random.seed(0) nums = immutableset(int(random.random() * 100) for _ in range(0, 25)) multiply_output_file = tmp_path / "multiplied_nums.txt" sorted_output_file = tmp_path / "sorted_nums.txt" with multiply_input_file.open("w") as mult_file: mult_file.writelines(f"{num}\n" for num in nums) multiply_params = Parameters.from_mapping({ "input_file": multiply_input_file, "output_file": multiply_output_file, "x": 4 }) sort_params = Parameters.from_mapping({ "input_file": multiply_output_file, "output_file": sorted_output_file }) resources = SlurmResourceRequest.from_parameters(slurm_params) workflow_builder = WorkflowBuilder.from_parameters(workflow_params) multiply_job_name = Locator(_parse_parts("jobs/multiply")) multiply_artifact = ValueArtifact( multiply_output_file, depends_on=workflow_builder.run_python_on_parameters( multiply_job_name, multiply_by_x_main, multiply_params, depends_on=[]), locator=Locator("multiply"), ) multiple_dir = workflow_builder.directory_for(multiply_job_name) assert (multiple_dir / "___run.sh").exists() assert (multiple_dir / "____params.params").exists() sort_job_name = Locator(_parse_parts("jobs/sort")) sort_dir = workflow_builder.directory_for(sort_job_name) workflow_builder.run_python_on_parameters( sort_job_name, sort_nums_main, sort_params, depends_on=[multiply_artifact], resource_request=resources, ) assert (sort_dir / "___run.sh").exists() assert (sort_dir / "____params.params").exists() dax_file_one = workflow_builder.write_dax_to_dir(tmp_path) dax_file_two = workflow_builder.write_dax_to_dir() assert dax_file_one.exists() assert dax_file_two.exists() submit_script_one = tmp_path / "submit_script_one.sh" submit_script_two = tmp_path / "submit_script_two.sh" build_submit_script( submit_script_one, str(dax_file_one), workflow_builder._workflow_directory, # pylint:disable=protected-access ) build_submit_script( submit_script_two, str(dax_file_two), workflow_builder._workflow_directory, # pylint:disable=protected-access ) assert submit_script_one.exists() assert submit_script_two.exists() submit_script_process = subprocess.Popen( ["sh", str(submit_script_one)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8", ) stdout, stderr = submit_script_process.communicate() print(stdout) print(stderr)
def example_workflow(params: Parameters): """ An example script to generate a workflow for submission to Pegasus. """ tmp_path = params.creatable_directory("example_root_dir") # Generating parameters for initializing a workflow # We recommend making workflow directory, site, and partition parameters # in an research workflow workflow_params = Parameters.from_mapping({ "workflow_name": "Test", "workflow_created": "Testing", "workflow_log_dir": str(tmp_path / "log"), "workflow_directory": str(tmp_path / "working"), "site": "saga", "namespace": "test", }) workflow_params = workflow_params.unify(params) # Our source input for the sample jobs multiply_input_file = tmp_path / "raw_nums.txt" random = Random() random.seed(0) nums = [int(random.random() * 100) for _ in range(0, 25)] multiply_output_file = tmp_path / "multiplied_nums.txt" sorted_output_file = tmp_path / "sorted_nums.txt" # Base Job Locator job_locator = Locator(("jobs", )) # Write a list of numbers out to be able to run the workflow with multiply_input_file.open("w") as mult_file: mult_file.writelines(f"{num}\n" for num in nums) initialize_vista_pegasus_wrapper(workflow_params) multiply_artifact = ValueArtifact( multiply_output_file, depends_on=run_python_on_parameters( job_locator / "multiply", multiply_by_x, { "input_file": multiply_input_file, "output_file": multiply_output_file, "x": 4, "logfile": str(tmp_path / "multiply_log.txt"), }, depends_on=[], ), locator=Locator("multiply"), ) run_python_on_parameters( job_locator / "sort", sort_nums_in_file, { "input_file": multiply_output_file, "output_file": sorted_output_file }, depends_on=[multiply_artifact], # if you want to use a different resource for some task, you can do this way # resource_request=SlurmResourceRequest.from_parameters(slurm_params), ) # Generate the Pegasus DAX file dax_file = write_workflow_description(tmp_path) submit_script = tmp_path / "submit_script.sh" # Our attempt at an easy submit file, it MAY NOT be accurate for more complicated # workflows but it # does work for this simple example. # See https://github.com/isi-vista/vista-pegasus-wrapper/issues/27 build_submit_script( submit_script, str(dax_file), experiment_directory(), # pylint:disable=protected-access )
def test_dax_with_job_on_saga_with_dict_as_params(tmp_path): workflow_params = Parameters.from_mapping({ "workflow_name": "Test", "workflow_created": "Testing", "workflow_log_dir": str(tmp_path / "log"), "workflow_directory": str(tmp_path / "working"), "site": "saga", "namespace": "test", "partition": "gaia", "experiment_name": "fred", "home_dir": str(tmp_path), }) slurm_params = Parameters.from_mapping({ "partition": "gaia", "num_cpus": 1, "num_gpus": 0, "memory": "4G" }) multiply_input_file = tmp_path / "raw_nums.txt" random = Random() random.seed(0) nums = immutableset(int(random.random() * 100) for _ in range(25)) multiply_output_file = tmp_path / "multiplied_nums.txt" sorted_output_file = tmp_path / "sorted_nums.txt" add_output_file = tmp_path / "add_nums.txt" with multiply_input_file.open("w") as mult_file: mult_file.writelines(f"{num}\n" for num in nums) multiply_params = { "input_file": multiply_input_file, "output_file": multiply_output_file, "x": 4, } sort_params = { "input_file": multiply_output_file, "output_file": sorted_output_file } add_args = f"{sorted_output_file} {add_output_file} --y 10" job_profile = PegasusProfile(namespace="pegasus", key="transfer.bypass.input.staging", value="True") resources = SlurmResourceRequest.from_parameters(slurm_params) initialize_vista_pegasus_wrapper(workflow_params) multiply_job_name = Locator(_parse_parts("jobs/multiply")) multiply_artifact = ValueArtifact( multiply_output_file, depends_on=run_python_on_parameters( multiply_job_name, multiply_by_x_main, multiply_params, depends_on=[], job_profiles=[job_profile], ), locator=Locator("multiply"), ) multiple_dir = directory_for(multiply_job_name) assert (multiple_dir / "___run.sh").exists() assert (multiple_dir / "____params.params").exists() sort_job_name = Locator(_parse_parts("jobs/sort")) sort_dir = directory_for(sort_job_name) sort_artifact = run_python_on_parameters( sort_job_name, sort_nums_main, sort_params, depends_on=[multiply_artifact], resource_request=resources, category="add", ) assert (sort_dir / "___run.sh").exists() assert (sort_dir / "____params.params").exists() add_job_name = Locator(_parse_parts("jobs/add")) add_dir = directory_for(add_job_name) run_python_on_args(add_job_name, "add_job_main.py", add_args, depends_on=[sort_artifact]) assert (add_dir / "___run.sh").exists() dax_file_one = write_workflow_description(tmp_path) dax_file_two = write_workflow_description() assert dax_file_one.exists() assert dax_file_two.exists() submit_script_one = tmp_path / "submit_script_one.sh" submit_script_two = tmp_path / "submit_script_two.sh" build_submit_script(submit_script_one, str(dax_file_one), experiment_directory()) build_submit_script(submit_script_two, str(dax_file_two), experiment_directory()) assert submit_script_one.exists() assert submit_script_two.exists() site_catalog = workflow_params.existing_directory( "workflow_directory") / "sites.yml" assert site_catalog.exists() replica_catalog = ( workflow_params.existing_directory("workflow_directory") / "replicas.yml") assert replica_catalog.exists() transformations_catalog = ( workflow_params.existing_directory("workflow_directory") / "transformations.yml") assert transformations_catalog.exists() properties_file = ( workflow_params.existing_directory("workflow_directory") / "pegasus.properties") assert properties_file.exists()
def test_dax_with_python_into_container_jobs(tmp_path): docker_tar = Path(f"{tmp_path}/docker/tar.tar") docker_build_dir = tmp_path docker_image_name = "pegasus_wrapper_container_demo" docker_image_tag = "0.2" # Generating parameters for initializing a workflow # We recommend making workflow directory, site, and partition parameters # in an research workflow workflow_params = Parameters.from_mapping({ "workflow_name": "Test", "workflow_created": "Testing", "workflow_log_dir": str(tmp_path / "log"), "workflow_directory": str(tmp_path / "working"), "site": "saga", "namespace": "test", "home_dir": str(tmp_path), "partition": "scavenge", }) saga31_request = SlurmResourceRequest.from_parameters( Parameters.from_mapping({ "run_on_single_node": "saga31", "partition": "gaia" })) # Our source input for the sample jobs input_file = tmp_path / "raw_nums.txt" add_y_output_file_nas = tmp_path / "nums_y.txt" sorted_output_file_nas = tmp_path / "sorted.txt" random = Random() random.seed(0) nums = [int(random.random() * 100) for _ in range(0, 25)] # Base Job Locator job_locator = Locator(("jobs", )) docker_python_root = Path("/home/app/") # Write a list of numbers out to be able to run the workflow with input_file.open("w") as mult_file: mult_file.writelines(f"{num}\n" for num in nums) workflow_builder = WorkflowBuilder.from_parameters(workflow_params) build_container_locator = job_locator / "build_docker" build_container = workflow_builder.run_bash( build_container_locator, command=[ "mkdir -p /scratch/dockermount/pegasus_wrapper_tmp", f"cd {docker_build_dir}", f"docker build . -t {docker_image_name}:{docker_image_tag}", f"docker save -o /scratch/dockermount/pegasus_wrapper_tmp/{docker_tar.name} {docker_image_name}:{docker_image_tag}", f"cp /scratch/dockermount/pegasus_wrapper_tmp/{docker_tar.name} {docker_tar.absolute()}", f"chmod go+r {docker_tar.absolute()}", ], depends_on=[], resource_request=saga31_request, ) build_container_dir = workflow_builder.directory_for( build_container_locator) assert (build_container_dir / "script.sh").exists() python36 = workflow_builder.add_container( f"{docker_image_name}:{docker_image_tag}", "docker", str(docker_tar.absolute()), image_site="saga", bypass_staging=True, ) job_profile = PegasusProfile(namespace="pegasus", key="transfer.bypass.input.staging", value="True") mongo4_4 = workflow_builder.add_container("mongo:4.4", "docker", "path/to/tar.tar", image_site="saga", bypass_staging=True) with pytest.raises(RuntimeError): _ = workflow_builder.stop_docker_as_service( mongo4_4, depends_on=[], resource_request=saga31_request) start_mongo = workflow_builder.start_docker_as_service( mongo4_4, depends_on=[build_container], docker_args=f"-v /scratch/mongo/data/db:/data/db", resource_request=saga31_request, ) mongo4_4_dir = workflow_builder.directory_for( Locator(("containers", mongo4_4.name))) assert (mongo4_4_dir / "start.sh").exists() assert (mongo4_4_dir / "stop.sh").exists() add_y_locator = job_locator / "add" add_y_job = workflow_builder.run_python_on_args( add_y_locator, docker_python_root / "add_y.py", set_args=f"{input_file} {add_y_output_file_nas} --y 10", depends_on=[build_container], job_profiles=[job_profile], resource_request=saga31_request, container=python36, input_file_paths=[input_file], output_file_paths=[add_y_output_file_nas], ) add_y_dir = workflow_builder.directory_for(add_y_locator) assert (add_y_dir / "___run.sh").exists() with pytest.raises(RuntimeError): _ = workflow_builder.run_python_on_args( add_y_locator, docker_python_root / "add_y.py", set_args=f"{input_file} {add_y_output_file_nas} --y 10", depends_on=[build_container], job_profiles=[job_profile], resource_request=saga31_request, container=python36, input_file_paths=[input_file, input_file], output_file_paths=[add_y_output_file_nas], ) sort_job_locator = job_locator / "sort" sort_job = workflow_builder.run_python_on_parameters( sort_job_locator, sort_nums_main, { "input_file": add_y_output_file_nas, "output_file": sorted_output_file_nas }, depends_on=[add_y_job], container=python36, job_profiles=[job_profile], resource_request=saga31_request, input_file_paths=add_y_output_file_nas, output_file_paths=sorted_output_file_nas, ) assert sort_job == workflow_builder.run_python_on_parameters( sort_job_locator, sort_nums_main, { "input_file": add_y_output_file_nas, "output_file": sorted_output_file_nas }, depends_on=[add_y_job], container=python36, job_profiles=[job_profile], resource_request=saga31_request, input_file_paths=add_y_output_file_nas, output_file_paths=sorted_output_file_nas, ) sort_job_dir = workflow_builder.directory_for(sort_job_locator) assert (sort_job_dir / "___run.sh").exists() assert (sort_job_dir / "____params.params").exists() with pytest.raises(RuntimeError): _ = workflow_builder.run_python_on_parameters( sort_job_locator, sort_nums_main, { "input_file": add_y_output_file_nas, "output_file": sorted_output_file_nas }, depends_on=[add_y_job], container=python36, job_profiles=[job_profile], resource_request=saga31_request, input_file_paths=add_y_output_file_nas, output_file_paths=[sorted_output_file_nas, sorted_output_file_nas], ) celebration_bash_locator = job_locator / "celebrate" celebration_bash = workflow_builder.run_bash( celebration_bash_locator, 'echo "Jobs Runs Successfully"', depends_on=[sort_job], job_profiles=[job_profile], ) assert celebration_bash == workflow_builder.run_bash( celebration_bash_locator, 'echo "Jobs Runs Successfully"', depends_on=[sort_job], job_profiles=[job_profile], ) celebration_bash_dir = workflow_builder.directory_for( celebration_bash_locator) assert (celebration_bash_dir / "script.sh").exists() _ = workflow_builder.stop_docker_as_service( mongo4_4, depends_on=[start_mongo, sort_job], resource_request=saga31_request) dax_file_one = workflow_builder.write_dax_to_dir(tmp_path) assert dax_file_one.exists() submit_script_one = tmp_path / "submit_script_one.sh" build_submit_script( submit_script_one, str(dax_file_one), workflow_builder._workflow_directory, # pylint:disable=protected-access ) assert submit_script_one.exists()