def test_SGEScript_set_tasks():
"""check different methods of adding tasks work"""
task_int = cookiecutter.SGEScript(user="******", tasks=42)
assert "#$ -t 1-42" in task_int.template
task_list = cookiecutter.SGEScript(user="******", tasks=[12, 32])
assert "#$ -t 12-32" in task_list.template
# check that tasks are inferred is tasks are missing set
# if the input_file is readable
task_infer = cookiecutter.SGEScript(user="******")
task_infer.loop_through_file(COMMANDS_LOC)
assert task_infer.array is True
assert task_infer.tasks == "#$ -t 1-{}".format(N_TASKS)
def test_SGEScript_mock_cluster():
"""Test auto-detecting user if we're on the cluster."""
os.environ["SGE_CLUSTER_NAME"] = "idrs"
my_script = cookiecutter.SGEScript(name="mock_cluster")
assert my_script.user == os.environ["USER"]
user = os.environ["USER"]
expected_output_location = "/scratch/{}/".format(user)
assert my_script.output == expected_output_location
def test_SGEScript_loop_through_file():
"""test creating array job script"""
script_tasks = cookiecutter.SGEScript(user="******", tasks="1-100")
script_tasks.loop_through_file(input_file="commands.txt")
output = script_tasks.template.split("\n")
assert output[-4] == 'CP_COMMAND_LIST="commands.txt"'
assert output[-3] == 'CP_COMMAND=$(awk "NR==$SGE_TASK_ID" "$CP_COMMAND_LIST")'
assert output[-2] == "$CP_COMMAND"
assert "#$ -t 1-100" in output
assert script_tasks.array is True
def make_qsub_scripts(commands_location, commands_count_dict,
logfile_location):
"""
Create and save qsub submission scripts in the same location as the
commands.
Parameters:
-----------
commands_location: string
path to directory that contains staging, cp_commands, and destaging
command files.
commands_count_dict: dictionary
dictionary of the number of commands contain in each of the jobs
logfile_location: string
where to store the log files. By default this will store them
in a directory alongside the results.
Returns:
---------
Nothing, writes files to `commands_location`
"""
cmd_path = make_command_paths(commands_location)
time_now = datetime.now().replace(microsecond=0)
time_now = str(time_now).replace(" ", "-")
# append random hex to job names - this allows you to run multiple jobs
# without the -hold_jid flags fron clashing
job_hex = cookiecutter.generate_random_hex()
n_tasks = commands_count_dict["cp_commands"]
analysis_script = cookiecutter.SGEScript(
name="analysis_{}".format(job_hex),
tasks=n_tasks,
output=os.path.join(logfile_location, "analysis"))
analysis_script += "\n"
analysis_script += "module load jdk/1.8.0_25"
analysis_script += "module load singularity"
analysis_script += 'CP_CONTAINER="/HPC_projets/CPCB-AI/singularity_containers/cellprofiler_319.simg"'
analysis_script.loop_through_file(cmd_path["cp_commands"],
prefix="singularity exec $CP_CONTAINER")
analysis_loc = os.path.join(commands_location,
"{}_analysis_script.sh".format(time_now))
analysis_script += make_logfile_text(logfile_location,
job_file=job_hex,
n_tasks=n_tasks)
analysis_script.save(analysis_loc)
def test_SGEScript():
"""test setting attributes"""
my_script = cookiecutter.SGEScript(
name="test_script", user="******", runtime="06:00:00"
)
assert my_script.name == "test_script"
assert my_script.runtime == "06:00:00"
assert my_script.queue == "lungo.q"
assert my_script.user == "test_user"
assert isinstance(my_script.template, str)
my_script += "#$ -another_option"
# add __iadd__ adds a final "\n" to the added text, when splitting by "\n"
# we need to look for the second to last element, as the last one if just
# and empty string
assert my_script.template.split("\n")[-2] == "#$ -another_option"
# test that the __iadd__ method works as expected
my_script += "python example.py"
assert my_script.template.split("\n")[-2] == "python example.py"