def launch_jobs(launcher, pbs_filenames, cluster_name, path_job): # pragma: no cover
''' Invokes launcher on a set of PBS files.
Parameters
----------
launcher : str
launcher name
pbs_filenames : list of str
a list of PBS files to launch
cluster_name : str
cluster name
path_job : str
path to the job folder
'''
jobs_id = []
for pbs_filename in pbs_filenames:
launcher_output = check_output('PBS_FILENAME={pbs_filename} {launcher} {pbs_filename}'.format(
launcher=launcher, pbs_filename=pbs_filename), shell=True)
jobs_id += [launcher_output.strip()]
# On some clusters, SRMJID and PBS_JOBID don't match
if cluster_name in ['helios']:
launcher_output = check_output(['qstat', '-f']).split('Job Id: ')
for job in launcher_output:
if re.search(r"SRMJID:{job_id}".format(job_id=jobs_id[-1]), job):
pbs_job_id = re.match(r"[0-9a-zA-Z.-]*", job).group()
jobs_id[-1] = '{pbs}'.format(pbs=pbs_job_id)
with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file:
jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
jobs_id_file.writelines("\n".join(jobs_id) + "\n")
print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))
def test_lock(self):
command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir]
# Lock the commands file before running 'base_worker.py'
with open_with_lock(self.command_manager._commands_filename, 'r+'):
process = Popen(command, stdout=PIPE, stderr=PIPE)
time.sleep(1)
stdout, stderr = process.communicate()
assert_equal(stdout, "")
assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
assert_true("Traceback" not in stderr) # Check that there are no errors.
def log_command_line(path_job, command_line):
""" Logs a command line in a job folder.
The command line is append to a file named 'command_line.log' that resides
in the given job folder. The current date and time is also added along
each command line logged.
Notes
-----
Commands save in log file might differ from sys.argv since we want to make sure
we can paste the command line as-is in the terminal. This means that the quotes
symbole " and the square brackets will be escaped.
"""
with open_with_lock(pjoin(path_job, "command_line.log"), 'a') as command_line_log:
command_line_log.write(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
command_line = command_line.replace('"', r'\"') # Make sure we can paste the command line as-is
command_line = re.sub(r'(\[)([^\[\]]*\\ [^\[\]]*)(\])', r'"\1\2\3"', command_line) # Make sure we can paste the command line as-is
command_line_log.write(command_line + "\n\n")
def log_command_line(path_job, command_line):
""" Logs a command line in a job folder.
The command line is append to a file named 'command_line.log' that resides
in the given job folder. The current date and time is also added along
each command line logged.
Notes
-----
Commands save in log file might differ from sys.argv since we want to make sure
we can paste the command line as-is in the terminal. This means that the quotes
symbole " and the square brackets will be escaped.
"""
with open_with_lock(pjoin(path_job, "command_line.log"), 'a') as command_line_log:
command_line_log.write(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
command_line = command_line.replace('"', r'\"') # Make sure we can paste the command line as-is
command_line = re.sub(r'(\[)([^\[\]]*\\ [^\[\]]*)(\])', r'"\1\2\3"', command_line) # Make sure we can paste the command line as-is
command_line_log.write(command_line + "\n\n")
def test_lock(self):
command = [
'python2', self.base_worker_script,
self.command_manager._commands_filename, self.logs_dir
]
# Lock the commands file before running 'base_worker.py'
with open_with_lock(self.command_manager._commands_filename, 'r+'):
process = Popen(command, stdout=PIPE, stderr=PIPE)
time.sleep(1)
stdout, stderr = process.communicate()
assert_equal(stdout, "")
assert_true(
"write-lock" in stderr,
msg="Forcing a race condition, try increasing sleeping time above."
)
assert_true("Traceback"
not in stderr) # Check that there are no errors.