def test_sub_3(self):
jobs = [make_script(["sleep 1"]) for _ in range(5)]
array_script, array_jobs = prep_array_script(jobs, os.getcwd(),
SunGridEngine.TASK_ID)
jobid = SunGridEngine.sub(array_script,
array=[1, 5],
hold=True,
name=inspect.stack()[0][3],
shell="/bin/sh")
time.sleep(5)
self.assertTrue(SunGridEngine.stat(jobid))
SunGridEngine.kill(jobid)
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
def test_sub_3(self):
jobs = [make_script(["sleep 1"]) for _ in range(5)]
array_script, array_jobs = prep_array_script(
jobs, os.getcwd(), LoadSharingFacility.TASK_ID)
jobid = LoadSharingFacility.sub(array_script,
array=[1, 5],
hold=True,
log=os.devnull,
name=inspect.stack()[0][3],
shell="/bin/sh")
time.sleep(5)
self.assertTrue(LoadSharingFacility.stat(jobid))
LoadSharingFacility.kill(jobid)
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
def submit(self, script, *args, **kwargs):
"""Submit a job to the job management platform
Parameters
----------
script : str, list
A list of one or more scripts with absolute paths
"""
# Only allow one submission per job
if self._lock:
logger.debug(
"This Job instance is locked, for further submissions create a new"
)
return
# Define a directory if not already done
if not ('directory' in kwargs and kwargs['directory']):
kwargs['directory'] = os.getcwd()
# Quick check if all scripts are sound - Also keep copy of logs and scripts
self._script, self._log = Job.check_script(script)
# See if we need to prepare an array
if len(self._script) > 1:
if any(base.__name__ == "ClusterPlatform"
for base in self._platform.__bases__):
script, _ = prep_array_script(self._script,
kwargs['directory'],
self._platform.TASK_ID)
kwargs["array"] = [
1,
len(self._script), kwargs['max_array_jobs']
if 'max_array_jobs' in kwargs else len(self._script)
]
kwargs["shell"] = "/bin/sh"
kwargs["log"] = os.devnull
script = [script]
else:
script = self._script
else:
script = self._script
# Get the submission function and submit the job
self._pid = self._platform.sub(script, **kwargs)
# Lock this Job so we cannot submit another
self._lock = True
def test_prep_array_scripts_3(self):
scripts = [
os.path.join(os.getcwd(), "script1.sh"),
os.path.join(os.getcwd(), "script2.sh")
]
array_script, array_jobs = prep_array_script(scripts, os.getcwd(),
"RANDOM_TEXT")
with open(array_jobs) as f_in:
array_jobs_content = [l.strip() for l in f_in.readlines()]
self.assertEqual(scripts, array_jobs_content)
with open(array_script) as f_in:
array_script_content = [l.strip() for l in f_in.readlines()]
self.assertEqual([
"#!/bin/sh", 'script=$(awk "NR==$RANDOM_TEXT" ' + array_jobs + ')',
"log=$(echo $script | sed 's/\.sh/\.log/')", "$script > $log 2>&1"
], array_script_content)
for f in [array_script, array_jobs]:
os.unlink(f)
def test_sub_6(self):
jobs = [
make_script(["echo $SGE_ROOT"], directory=os.getcwd())
for _ in range(2)
]
array_script, array_jobs = prep_array_script(jobs, os.getcwd(),
SunGridEngine.TASK_ID)
jobid = SunGridEngine.sub(array_script,
array=[1, 2],
log=os.devnull,
name=inspect.stack()[0][3],
shell="/bin/sh")
while SunGridEngine.stat(jobid):
time.sleep(1)
for i, j in enumerate(jobs):
f = j.replace(".sh", ".log")
self.assertTrue(os.path.isfile(f))
self.assertEqual(os.environ["SGE_ROOT"], open(f).read().strip())
os.unlink(f)
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
def test_stat_2(self):
jobs = [make_script(["sleep 100"]) for _ in range(5)]
array_script, array_jobs = prep_array_script(jobs, os.getcwd(),
SunGridEngine.TASK_ID)
jobid = SunGridEngine.sub(array_script,
array=[1, 5],
hold=True,
name=inspect.stack()[0][3],
shell="/bin/sh")
time.sleep(5)
data = SunGridEngine.stat(jobid)
SunGridEngine.kill(jobid)
self.assertTrue(data)
self.assertEqual(jobid, int(data['job_number']))
self.assertTrue('sge_o_shell' in data)
self.assertTrue('sge_o_workdir' in data)
self.assertTrue('sge_o_host' in data)
self.assertTrue('job-array tasks' in data)
self.assertEqual("1-5:1", data['job-array tasks'].strip())
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
def test_sub_5(self):
directory = os.getcwd()
jobs = [
make_script(['echo "file {0}"'.format(i)], directory=directory)
for i in range(100)
]
array_script, array_jobs = prep_array_script(jobs, directory,
SunGridEngine.TASK_ID)
jobid = SunGridEngine.sub(array_script,
array=[1, 100],
log=os.devnull,
name=inspect.stack()[0][3],
shell="/bin/sh")
while SunGridEngine.stat(jobid):
time.sleep(1)
for i, j in enumerate(jobs):
f = j.replace(".sh", ".log")
self.assertTrue(os.path.isfile(f))
self.assertEqual("file {0}".format(i), open(f).read().strip())
os.unlink(f)
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
def test_sub_7(self):
assert "PYJOB_ENV1" not in os.environ
os.environ["PYJOB_ENV1"] = "pyjob_random1"
jobs = [
make_script(["echo $PYJOB_ENV1"], directory=os.getcwd())
for _ in range(2)
]
array_script, array_jobs = prep_array_script(
jobs, os.getcwd(), LoadSharingFacility.TASK_ID)
jobid = LoadSharingFacility.sub(array_script,
array=[1, 2],
directory=os.getcwd(),
log=os.devnull,
name=inspect.stack()[0][3],
shell="/bin/sh")
while LoadSharingFacility.stat(jobid):
time.sleep(1)
for i, j in enumerate(jobs):
f = j.replace(".sh", ".log")
self.assertTrue(os.path.isfile(f))
self.assertEqual(os.environ["PYJOB_ENV1"], open(f).read().strip())
os.unlink(f)
for f in jobs + [array_script, array_jobs]:
os.unlink(f)
