def test_sub_3(self): jobs = [make_script(["sleep 1"]) for _ in range(5)] array_script, array_jobs = prep_array_script(jobs, os.getcwd(), SunGridEngine.TASK_ID) jobid = SunGridEngine.sub(array_script, array=[1, 5], hold=True, name=inspect.stack()[0][3], shell="/bin/sh") time.sleep(5) self.assertTrue(SunGridEngine.stat(jobid)) SunGridEngine.kill(jobid) for f in jobs + [array_script, array_jobs]: os.unlink(f)
def test_sub_3(self): jobs = [make_script(["sleep 1"]) for _ in range(5)] array_script, array_jobs = prep_array_script( jobs, os.getcwd(), LoadSharingFacility.TASK_ID) jobid = LoadSharingFacility.sub(array_script, array=[1, 5], hold=True, log=os.devnull, name=inspect.stack()[0][3], shell="/bin/sh") time.sleep(5) self.assertTrue(LoadSharingFacility.stat(jobid)) LoadSharingFacility.kill(jobid) for f in jobs + [array_script, array_jobs]: os.unlink(f)
def submit(self, script, *args, **kwargs): """Submit a job to the job management platform Parameters ---------- script : str, list A list of one or more scripts with absolute paths """ # Only allow one submission per job if self._lock: logger.debug( "This Job instance is locked, for further submissions create a new" ) return # Define a directory if not already done if not ('directory' in kwargs and kwargs['directory']): kwargs['directory'] = os.getcwd() # Quick check if all scripts are sound - Also keep copy of logs and scripts self._script, self._log = Job.check_script(script) # See if we need to prepare an array if len(self._script) > 1: if any(base.__name__ == "ClusterPlatform" for base in self._platform.__bases__): script, _ = prep_array_script(self._script, kwargs['directory'], self._platform.TASK_ID) kwargs["array"] = [ 1, len(self._script), kwargs['max_array_jobs'] if 'max_array_jobs' in kwargs else len(self._script) ] kwargs["shell"] = "/bin/sh" kwargs["log"] = os.devnull script = [script] else: script = self._script else: script = self._script # Get the submission function and submit the job self._pid = self._platform.sub(script, **kwargs) # Lock this Job so we cannot submit another self._lock = True
def test_prep_array_scripts_3(self): scripts = [ os.path.join(os.getcwd(), "script1.sh"), os.path.join(os.getcwd(), "script2.sh") ] array_script, array_jobs = prep_array_script(scripts, os.getcwd(), "RANDOM_TEXT") with open(array_jobs) as f_in: array_jobs_content = [l.strip() for l in f_in.readlines()] self.assertEqual(scripts, array_jobs_content) with open(array_script) as f_in: array_script_content = [l.strip() for l in f_in.readlines()] self.assertEqual([ "#!/bin/sh", 'script=$(awk "NR==$RANDOM_TEXT" ' + array_jobs + ')', "log=$(echo $script | sed 's/\.sh/\.log/')", "$script > $log 2>&1" ], array_script_content) for f in [array_script, array_jobs]: os.unlink(f)
def test_sub_6(self): jobs = [ make_script(["echo $SGE_ROOT"], directory=os.getcwd()) for _ in range(2) ] array_script, array_jobs = prep_array_script(jobs, os.getcwd(), SunGridEngine.TASK_ID) jobid = SunGridEngine.sub(array_script, array=[1, 2], log=os.devnull, name=inspect.stack()[0][3], shell="/bin/sh") while SunGridEngine.stat(jobid): time.sleep(1) for i, j in enumerate(jobs): f = j.replace(".sh", ".log") self.assertTrue(os.path.isfile(f)) self.assertEqual(os.environ["SGE_ROOT"], open(f).read().strip()) os.unlink(f) for f in jobs + [array_script, array_jobs]: os.unlink(f)
def test_stat_2(self): jobs = [make_script(["sleep 100"]) for _ in range(5)] array_script, array_jobs = prep_array_script(jobs, os.getcwd(), SunGridEngine.TASK_ID) jobid = SunGridEngine.sub(array_script, array=[1, 5], hold=True, name=inspect.stack()[0][3], shell="/bin/sh") time.sleep(5) data = SunGridEngine.stat(jobid) SunGridEngine.kill(jobid) self.assertTrue(data) self.assertEqual(jobid, int(data['job_number'])) self.assertTrue('sge_o_shell' in data) self.assertTrue('sge_o_workdir' in data) self.assertTrue('sge_o_host' in data) self.assertTrue('job-array tasks' in data) self.assertEqual("1-5:1", data['job-array tasks'].strip()) for f in jobs + [array_script, array_jobs]: os.unlink(f)
def test_sub_5(self): directory = os.getcwd() jobs = [ make_script(['echo "file {0}"'.format(i)], directory=directory) for i in range(100) ] array_script, array_jobs = prep_array_script(jobs, directory, SunGridEngine.TASK_ID) jobid = SunGridEngine.sub(array_script, array=[1, 100], log=os.devnull, name=inspect.stack()[0][3], shell="/bin/sh") while SunGridEngine.stat(jobid): time.sleep(1) for i, j in enumerate(jobs): f = j.replace(".sh", ".log") self.assertTrue(os.path.isfile(f)) self.assertEqual("file {0}".format(i), open(f).read().strip()) os.unlink(f) for f in jobs + [array_script, array_jobs]: os.unlink(f)
def test_sub_7(self): assert "PYJOB_ENV1" not in os.environ os.environ["PYJOB_ENV1"] = "pyjob_random1" jobs = [ make_script(["echo $PYJOB_ENV1"], directory=os.getcwd()) for _ in range(2) ] array_script, array_jobs = prep_array_script( jobs, os.getcwd(), LoadSharingFacility.TASK_ID) jobid = LoadSharingFacility.sub(array_script, array=[1, 2], directory=os.getcwd(), log=os.devnull, name=inspect.stack()[0][3], shell="/bin/sh") while LoadSharingFacility.stat(jobid): time.sleep(1) for i, j in enumerate(jobs): f = j.replace(".sh", ".log") self.assertTrue(os.path.isfile(f)) self.assertEqual(os.environ["PYJOB_ENV1"], open(f).read().strip()) os.unlink(f) for f in jobs + [array_script, array_jobs]: os.unlink(f)