def _make_singularity_command(self):
params = self._submission_group.submitter_params.singularity_params
if not params.run_command:
raise InvalidConfiguration("Singularity command cannot be empty.")
container_path = Path(params.container)
cmd = f"{params.run_command} {container_path} {self._run_script}"
sing_script = self._run_script.parent / self._run_script.name.replace(
"run", "singularity", 1)
text = "#!/bin/bash\n" + params.load_command + "\n" + cmd + "\n"
create_script(str(sing_script), text)
return sing_script
def _create_job(container, spark_conf, script):
if script is None:
script = Path("/tmp/sleep.sh")
# TODO: Can we get the walltime value?
create_script(script, "sleep 10d\n", executable=True)
return GenericCommandParameters(
name="1",
command=f"bash {script}",
spark_config=SparkConfigModel(
conf_dir=spark_conf,
container=SparkContainerModel(path=container),
enabled=True,
run_user_script_inside_container=False,
),
)
def _create_run_script(self, config_file, filename, submission_group):
text = ["#!/bin/bash"]
sing_params = submission_group.submitter_params.singularity_params
if sing_params and sing_params.enabled:
text += sing_params.setup_commands.split("\n")
if submission_group.submitter_params.distributed_submitter:
dsub = "--distributed-submitter"
else:
dsub = "--no-distributed-submitter"
command = f"jade-internal run-jobs {config_file} --output={self._output} {dsub}"
if submission_group.submitter_params.num_processes is not None:
command += f" --num-processes={submission_group.submitter_params.num_processes}"
if submission_group.submitter_params.verbose:
command += " --verbose"
text.append(command)
create_script(filename, "\n".join(text) + "\n")
def _create_run_script(config_file, filename, num_processes, output,
verbose):
text = ["#!/bin/bash"]
if shutil.which("module") is not None:
# Required for HPC systems.
text.append("module load conda")
text.append("conda activate jade")
command = f"jade-internal run-jobs {config_file} " \
f"--output={output}"
if num_processes is not None:
command += f" --num-processes={num_processes}"
if verbose:
command += " --verbose"
text.append(command)
create_script(filename, "\n".join(text))
def create_submission_script(self, name, script, filename, path="."):
feature = self._config.get("feature")
if feature is None:
feature = ""
else:
feature = "#PBS -l feature={}".format("feature")
script = PBS_SCRIPT.format(
name=name,
alloc=self._config["allocation"],
qos=self._config["qos"],
queue=self._config["queue"],
path=path,
feature=feature,
script=script,
)
utils.create_script(filename, script)
def create_submission_script(self, name, script, filename, path):
text = self._create_submission_script_text(name, script, path)
utils.create_script(filename, "\n".join(text) + "\n")
def create_submission_script(self, name, script, filename, path):
lines = [
"#!/bin/bash",
str(script),
]
create_script(filename, "\n".join(lines))
