def write_dax_to_dir(self, output_xml_dir: Optional[Path] = None) -> Path:
if not output_xml_dir:
output_xml_dir = self._workflow_directory
dax_file_name = f"{self.name}.dax"
dax_file = output_xml_dir / dax_file_name
logging.info("Writing DAX to %s", dax_file)
with dax_file.open("w") as dax:
self._job_graph.writeXML(dax)
build_submit_script(output_xml_dir / "submit.sh", dax_file_name,
self._workflow_directory)
# We also need to write sites.xml and pegasus.conf
sites_xml_path = output_xml_dir / "sites.xml"
sites_xml_path.write_text(pkg_resources.read_text(
resources, "sites.xml"),
encoding="utf-8")
pegasus_conf_path = output_xml_dir / "pegasus.conf"
pegasus_conf_path.write_text(
data=pkg_resources.read_text(resources, "pegasus.conf") +
"pegasus.catalog.replica=File\n" +
f"pegasus.catalog.replica.file={self._replica_catalog}\n" +
self._conf_limits(),
encoding="utf-8",
)
return dax_file
def write_dax_to_dir(self, output_xml_dir: Optional[Path] = None) -> Path:
if not output_xml_dir:
output_xml_dir = self._workflow_directory
num_jobs = len(self._job_graph.jobs.keys())
num_ckpts = len(
[ckpt_file for ckpt_file in output_xml_dir.rglob("___ckpt")])
if num_jobs == num_ckpts:
nuke = input(
"DAX *may* create a NOOP workflow. Do you want to nuke the checkpoints and regenerate? [y/n]"
)
if nuke == "y":
self._nuke_checkpoints_and_clear_rc(output_xml_dir)
logging.info("Checkpoints cleared!")
dax_file_name = f"{self.name}.dax"
dax_file = output_xml_dir / dax_file_name
logging.info("Writing DAX to %s", dax_file)
with dax_file.open("w") as dax:
self._job_graph.write(dax)
build_submit_script(output_xml_dir / "submit.sh", dax_file_name,
self._workflow_directory)
# Write Out Sites Catalog
sites_yml_path = output_xml_dir / "sites.yml"
with sites_yml_path.open("w") as sites:
self._sites_catalog.write(sites)
self._properties["pegasus.catalog.site.file"] = str(
sites_yml_path.absolute())
# Write Out Replica Catalog
replica_yml_path = output_xml_dir / "replicas.yml"
with replica_yml_path.open("w") as replicas:
self._replica_catalog.write(replicas)
self._properties["pegasus.catalog.replica"] = "YAML"
self._properties["pegasus.catalog.replica.file"] = str(
replica_yml_path.absolute())
# Write Out Transformation Catalog
transformation_yml_path = output_xml_dir / "transformations.yml"
with transformation_yml_path.open("w") as transformations:
self._transformation_catalog.write(transformations)
self._properties["pegasus.catalog.transformation"] = "YAML"
self._properties["pegasus.catalog.transformation.file"] = str(
transformation_yml_path.absolute())
# Write Out Pegasus Properties
self._conf_limits()
pegasus_conf_path = output_xml_dir / "pegasus.properties"
with pegasus_conf_path.open("w") as properties:
self._properties.write(properties)
return dax_file
def test_dax_with_job_on_saga(tmp_path):
workflow_params = Parameters.from_mapping({
"workflow_name":
"Test",
"workflow_created":
"Testing",
"workflow_log_dir":
str(tmp_path / "log"),
"workflow_directory":
str(tmp_path / "working"),
"site":
"saga",
"namespace":
"test",
"partition":
"scavenge",
})
slurm_params = Parameters.from_mapping({
"partition": "scavenge",
"num_cpus": 1,
"num_gpus": 0,
"memory": "4G"
})
multiply_input_file = tmp_path / "raw_nums.txt"
random = Random()
random.seed(0)
nums = immutableset(int(random.random() * 100) for _ in range(0, 25))
multiply_output_file = tmp_path / "multiplied_nums.txt"
sorted_output_file = tmp_path / "sorted_nums.txt"
with multiply_input_file.open("w") as mult_file:
mult_file.writelines(f"{num}\n" for num in nums)
multiply_params = Parameters.from_mapping({
"input_file": multiply_input_file,
"output_file": multiply_output_file,
"x": 4
})
sort_params = Parameters.from_mapping({
"input_file": multiply_output_file,
"output_file": sorted_output_file
})
resources = SlurmResourceRequest.from_parameters(slurm_params)
workflow_builder = WorkflowBuilder.from_parameters(workflow_params)
multiply_job_name = Locator(_parse_parts("jobs/multiply"))
multiply_artifact = ValueArtifact(
multiply_output_file,
depends_on=workflow_builder.run_python_on_parameters(
multiply_job_name,
multiply_by_x_main,
multiply_params,
depends_on=[]),
locator=Locator("multiply"),
)
multiple_dir = workflow_builder.directory_for(multiply_job_name)
assert (multiple_dir / "___run.sh").exists()
assert (multiple_dir / "____params.params").exists()
sort_job_name = Locator(_parse_parts("jobs/sort"))
sort_dir = workflow_builder.directory_for(sort_job_name)
workflow_builder.run_python_on_parameters(
sort_job_name,
sort_nums_main,
sort_params,
depends_on=[multiply_artifact],
resource_request=resources,
)
assert (sort_dir / "___run.sh").exists()
assert (sort_dir / "____params.params").exists()
dax_file_one = workflow_builder.write_dax_to_dir(tmp_path)
dax_file_two = workflow_builder.write_dax_to_dir()
assert dax_file_one.exists()
assert dax_file_two.exists()
submit_script_one = tmp_path / "submit_script_one.sh"
submit_script_two = tmp_path / "submit_script_two.sh"
build_submit_script(
submit_script_one,
str(dax_file_one),
workflow_builder._workflow_directory, # pylint:disable=protected-access
)
build_submit_script(
submit_script_two,
str(dax_file_two),
workflow_builder._workflow_directory, # pylint:disable=protected-access
)
assert submit_script_one.exists()
assert submit_script_two.exists()
submit_script_process = subprocess.Popen(
["sh", str(submit_script_one)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding="utf-8",
)
stdout, stderr = submit_script_process.communicate()
print(stdout)
print(stderr)
def example_workflow(params: Parameters):
"""
An example script to generate a workflow for submission to Pegasus.
"""
tmp_path = params.creatable_directory("example_root_dir")
# Generating parameters for initializing a workflow
# We recommend making workflow directory, site, and partition parameters
# in an research workflow
workflow_params = Parameters.from_mapping({
"workflow_name":
"Test",
"workflow_created":
"Testing",
"workflow_log_dir":
str(tmp_path / "log"),
"workflow_directory":
str(tmp_path / "working"),
"site":
"saga",
"namespace":
"test",
})
workflow_params = workflow_params.unify(params)
# Our source input for the sample jobs
multiply_input_file = tmp_path / "raw_nums.txt"
random = Random()
random.seed(0)
nums = [int(random.random() * 100) for _ in range(0, 25)]
multiply_output_file = tmp_path / "multiplied_nums.txt"
sorted_output_file = tmp_path / "sorted_nums.txt"
# Base Job Locator
job_locator = Locator(("jobs", ))
# Write a list of numbers out to be able to run the workflow
with multiply_input_file.open("w") as mult_file:
mult_file.writelines(f"{num}\n" for num in nums)
initialize_vista_pegasus_wrapper(workflow_params)
multiply_artifact = ValueArtifact(
multiply_output_file,
depends_on=run_python_on_parameters(
job_locator / "multiply",
multiply_by_x,
{
"input_file": multiply_input_file,
"output_file": multiply_output_file,
"x": 4,
"logfile": str(tmp_path / "multiply_log.txt"),
},
depends_on=[],
),
locator=Locator("multiply"),
)
run_python_on_parameters(
job_locator / "sort",
sort_nums_in_file,
{
"input_file": multiply_output_file,
"output_file": sorted_output_file
},
depends_on=[multiply_artifact],
# if you want to use a different resource for some task, you can do this way
# resource_request=SlurmResourceRequest.from_parameters(slurm_params),
)
# Generate the Pegasus DAX file
dax_file = write_workflow_description(tmp_path)
submit_script = tmp_path / "submit_script.sh"
# Our attempt at an easy submit file, it MAY NOT be accurate for more complicated
# workflows but it
# does work for this simple example.
# See https://github.com/isi-vista/vista-pegasus-wrapper/issues/27
build_submit_script(
submit_script,
str(dax_file),
experiment_directory(), # pylint:disable=protected-access
)
def test_dax_with_job_on_saga_with_dict_as_params(tmp_path):
workflow_params = Parameters.from_mapping({
"workflow_name":
"Test",
"workflow_created":
"Testing",
"workflow_log_dir":
str(tmp_path / "log"),
"workflow_directory":
str(tmp_path / "working"),
"site":
"saga",
"namespace":
"test",
"partition":
"gaia",
"experiment_name":
"fred",
"home_dir":
str(tmp_path),
})
slurm_params = Parameters.from_mapping({
"partition": "gaia",
"num_cpus": 1,
"num_gpus": 0,
"memory": "4G"
})
multiply_input_file = tmp_path / "raw_nums.txt"
random = Random()
random.seed(0)
nums = immutableset(int(random.random() * 100) for _ in range(25))
multiply_output_file = tmp_path / "multiplied_nums.txt"
sorted_output_file = tmp_path / "sorted_nums.txt"
add_output_file = tmp_path / "add_nums.txt"
with multiply_input_file.open("w") as mult_file:
mult_file.writelines(f"{num}\n" for num in nums)
multiply_params = {
"input_file": multiply_input_file,
"output_file": multiply_output_file,
"x": 4,
}
sort_params = {
"input_file": multiply_output_file,
"output_file": sorted_output_file
}
add_args = f"{sorted_output_file} {add_output_file} --y 10"
job_profile = PegasusProfile(namespace="pegasus",
key="transfer.bypass.input.staging",
value="True")
resources = SlurmResourceRequest.from_parameters(slurm_params)
initialize_vista_pegasus_wrapper(workflow_params)
multiply_job_name = Locator(_parse_parts("jobs/multiply"))
multiply_artifact = ValueArtifact(
multiply_output_file,
depends_on=run_python_on_parameters(
multiply_job_name,
multiply_by_x_main,
multiply_params,
depends_on=[],
job_profiles=[job_profile],
),
locator=Locator("multiply"),
)
multiple_dir = directory_for(multiply_job_name)
assert (multiple_dir / "___run.sh").exists()
assert (multiple_dir / "____params.params").exists()
sort_job_name = Locator(_parse_parts("jobs/sort"))
sort_dir = directory_for(sort_job_name)
sort_artifact = run_python_on_parameters(
sort_job_name,
sort_nums_main,
sort_params,
depends_on=[multiply_artifact],
resource_request=resources,
category="add",
)
assert (sort_dir / "___run.sh").exists()
assert (sort_dir / "____params.params").exists()
add_job_name = Locator(_parse_parts("jobs/add"))
add_dir = directory_for(add_job_name)
run_python_on_args(add_job_name,
"add_job_main.py",
add_args,
depends_on=[sort_artifact])
assert (add_dir / "___run.sh").exists()
dax_file_one = write_workflow_description(tmp_path)
dax_file_two = write_workflow_description()
assert dax_file_one.exists()
assert dax_file_two.exists()
submit_script_one = tmp_path / "submit_script_one.sh"
submit_script_two = tmp_path / "submit_script_two.sh"
build_submit_script(submit_script_one, str(dax_file_one),
experiment_directory())
build_submit_script(submit_script_two, str(dax_file_two),
experiment_directory())
assert submit_script_one.exists()
assert submit_script_two.exists()
site_catalog = workflow_params.existing_directory(
"workflow_directory") / "sites.yml"
assert site_catalog.exists()
replica_catalog = (
workflow_params.existing_directory("workflow_directory") /
"replicas.yml")
assert replica_catalog.exists()
transformations_catalog = (
workflow_params.existing_directory("workflow_directory") /
"transformations.yml")
assert transformations_catalog.exists()
properties_file = (
workflow_params.existing_directory("workflow_directory") /
"pegasus.properties")
assert properties_file.exists()
def test_dax_with_python_into_container_jobs(tmp_path):
docker_tar = Path(f"{tmp_path}/docker/tar.tar")
docker_build_dir = tmp_path
docker_image_name = "pegasus_wrapper_container_demo"
docker_image_tag = "0.2"
# Generating parameters for initializing a workflow
# We recommend making workflow directory, site, and partition parameters
# in an research workflow
workflow_params = Parameters.from_mapping({
"workflow_name":
"Test",
"workflow_created":
"Testing",
"workflow_log_dir":
str(tmp_path / "log"),
"workflow_directory":
str(tmp_path / "working"),
"site":
"saga",
"namespace":
"test",
"home_dir":
str(tmp_path),
"partition":
"scavenge",
})
saga31_request = SlurmResourceRequest.from_parameters(
Parameters.from_mapping({
"run_on_single_node": "saga31",
"partition": "gaia"
}))
# Our source input for the sample jobs
input_file = tmp_path / "raw_nums.txt"
add_y_output_file_nas = tmp_path / "nums_y.txt"
sorted_output_file_nas = tmp_path / "sorted.txt"
random = Random()
random.seed(0)
nums = [int(random.random() * 100) for _ in range(0, 25)]
# Base Job Locator
job_locator = Locator(("jobs", ))
docker_python_root = Path("/home/app/")
# Write a list of numbers out to be able to run the workflow
with input_file.open("w") as mult_file:
mult_file.writelines(f"{num}\n" for num in nums)
workflow_builder = WorkflowBuilder.from_parameters(workflow_params)
build_container_locator = job_locator / "build_docker"
build_container = workflow_builder.run_bash(
build_container_locator,
command=[
"mkdir -p /scratch/dockermount/pegasus_wrapper_tmp",
f"cd {docker_build_dir}",
f"docker build . -t {docker_image_name}:{docker_image_tag}",
f"docker save -o /scratch/dockermount/pegasus_wrapper_tmp/{docker_tar.name} {docker_image_name}:{docker_image_tag}",
f"cp /scratch/dockermount/pegasus_wrapper_tmp/{docker_tar.name} {docker_tar.absolute()}",
f"chmod go+r {docker_tar.absolute()}",
],
depends_on=[],
resource_request=saga31_request,
)
build_container_dir = workflow_builder.directory_for(
build_container_locator)
assert (build_container_dir / "script.sh").exists()
python36 = workflow_builder.add_container(
f"{docker_image_name}:{docker_image_tag}",
"docker",
str(docker_tar.absolute()),
image_site="saga",
bypass_staging=True,
)
job_profile = PegasusProfile(namespace="pegasus",
key="transfer.bypass.input.staging",
value="True")
mongo4_4 = workflow_builder.add_container("mongo:4.4",
"docker",
"path/to/tar.tar",
image_site="saga",
bypass_staging=True)
with pytest.raises(RuntimeError):
_ = workflow_builder.stop_docker_as_service(
mongo4_4, depends_on=[], resource_request=saga31_request)
start_mongo = workflow_builder.start_docker_as_service(
mongo4_4,
depends_on=[build_container],
docker_args=f"-v /scratch/mongo/data/db:/data/db",
resource_request=saga31_request,
)
mongo4_4_dir = workflow_builder.directory_for(
Locator(("containers", mongo4_4.name)))
assert (mongo4_4_dir / "start.sh").exists()
assert (mongo4_4_dir / "stop.sh").exists()
add_y_locator = job_locator / "add"
add_y_job = workflow_builder.run_python_on_args(
add_y_locator,
docker_python_root / "add_y.py",
set_args=f"{input_file} {add_y_output_file_nas} --y 10",
depends_on=[build_container],
job_profiles=[job_profile],
resource_request=saga31_request,
container=python36,
input_file_paths=[input_file],
output_file_paths=[add_y_output_file_nas],
)
add_y_dir = workflow_builder.directory_for(add_y_locator)
assert (add_y_dir / "___run.sh").exists()
with pytest.raises(RuntimeError):
_ = workflow_builder.run_python_on_args(
add_y_locator,
docker_python_root / "add_y.py",
set_args=f"{input_file} {add_y_output_file_nas} --y 10",
depends_on=[build_container],
job_profiles=[job_profile],
resource_request=saga31_request,
container=python36,
input_file_paths=[input_file, input_file],
output_file_paths=[add_y_output_file_nas],
)
sort_job_locator = job_locator / "sort"
sort_job = workflow_builder.run_python_on_parameters(
sort_job_locator,
sort_nums_main,
{
"input_file": add_y_output_file_nas,
"output_file": sorted_output_file_nas
},
depends_on=[add_y_job],
container=python36,
job_profiles=[job_profile],
resource_request=saga31_request,
input_file_paths=add_y_output_file_nas,
output_file_paths=sorted_output_file_nas,
)
assert sort_job == workflow_builder.run_python_on_parameters(
sort_job_locator,
sort_nums_main,
{
"input_file": add_y_output_file_nas,
"output_file": sorted_output_file_nas
},
depends_on=[add_y_job],
container=python36,
job_profiles=[job_profile],
resource_request=saga31_request,
input_file_paths=add_y_output_file_nas,
output_file_paths=sorted_output_file_nas,
)
sort_job_dir = workflow_builder.directory_for(sort_job_locator)
assert (sort_job_dir / "___run.sh").exists()
assert (sort_job_dir / "____params.params").exists()
with pytest.raises(RuntimeError):
_ = workflow_builder.run_python_on_parameters(
sort_job_locator,
sort_nums_main,
{
"input_file": add_y_output_file_nas,
"output_file": sorted_output_file_nas
},
depends_on=[add_y_job],
container=python36,
job_profiles=[job_profile],
resource_request=saga31_request,
input_file_paths=add_y_output_file_nas,
output_file_paths=[sorted_output_file_nas, sorted_output_file_nas],
)
celebration_bash_locator = job_locator / "celebrate"
celebration_bash = workflow_builder.run_bash(
celebration_bash_locator,
'echo "Jobs Runs Successfully"',
depends_on=[sort_job],
job_profiles=[job_profile],
)
assert celebration_bash == workflow_builder.run_bash(
celebration_bash_locator,
'echo "Jobs Runs Successfully"',
depends_on=[sort_job],
job_profiles=[job_profile],
)
celebration_bash_dir = workflow_builder.directory_for(
celebration_bash_locator)
assert (celebration_bash_dir / "script.sh").exists()
_ = workflow_builder.stop_docker_as_service(
mongo4_4,
depends_on=[start_mongo, sort_job],
resource_request=saga31_request)
dax_file_one = workflow_builder.write_dax_to_dir(tmp_path)
assert dax_file_one.exists()
submit_script_one = tmp_path / "submit_script_one.sh"
build_submit_script(
submit_script_one,
str(dax_file_one),
workflow_builder._workflow_directory, # pylint:disable=protected-access
)
assert submit_script_one.exists()