def test_job_parameters(tmpdir):
input_file = path.join(path.dirname(path.abspath(__file__)),
'job_scripts/job_parameters.txt')
folder = str(tmpdir.realpath())
params = JobParameters(cores=3,
wall_time='00-00:02:00') # command line _params
manager = Submitted(input_file,
folder,
TEST_PROFILE,
max_running_jobs=1,
cli_params=params)
for i, id_ in enumerate(manager.get_jobs()):
job = manager.get(id_)
if id_ == 1:
print(job.params)
assert job.params['cores'] == 6 or job.params['cores'] == '6'
assert job.params['queue'] == 'myq'
else:
assert len(job.params) == 0
def load_all(tmp_folder):
test_number = 0
input_files_dir = path.join(path.dirname(path.abspath(__file__)), 'job_scripts')
for in_file in os.listdir(input_files_dir):
input_file = path.join(input_files_dir, in_file)
folder = str(tmp_folder.mkdir('test{}_qmap_output'.format(test_number)))
test_number += 1
manager = Submitted(input_file, folder, TEST_PROFILE, max_running_jobs=1)
yield manager
def test_not_empty_output_folder(tmpdir):
input_file = path.join(path.dirname(path.abspath(__file__)),
'job_scripts/simple.txt')
folder = str(tmpdir.realpath())
copy_file(input_file, path.join(folder, 'tst.txt'))
try:
Submitted(input_file, folder, TEST_PROFILE)
except QMapError:
assert 1
else:
assert 0
def test_grouping(tmpdir):
input_file = path.join(path.dirname(path.abspath(__file__)),
'job_scripts/10_jobs.txt')
for i, g in enumerate([1, 2, 3, 5, 7, 11]):
folder = str(tmpdir.mkdir('test{}_qmap_output'.format(i)))
Submitted(input_file,
folder,
TEST_PROFILE,
max_running_jobs=1,
group_size=g)
scripts = len([f for f in os.listdir(folder)
if f.endswith('.sh')]) # count number of script files
assert scripts == math.ceil(10 / g)
def submit(jobs_file, logs_folder, max_running, group, no_console, cores,
memory, wall_time, working_directory, prefix, profile):
"""
Submit a set of jobs (using a jobs file)
\b
The following values will be extended
${QMAP_LINE}: for line number in the input file (this provides a unique identifier for each line)
"""
params = {}
if cores is not None:
params['cores'] = cores
if memory is not None:
if memory[-1].isdigit():
memory += 'G'
params['memory'] = memory
if wall_time is not None:
if wall_time.isdigit():
wall_time += 'h'
params['time'] = wall_time
if prefix is not None:
params['prefix'] = prefix
else:
if 'STY' in os.environ: # use screen name as prefix
params['prefix'] = os.environ['STY'].split('.')[1]
# Pass always the working directory to prevent issues when resubmitting the jobs in reattached
if working_directory is None:
params['working_directory'] = os.getcwd().strip()
else:
params['working_directory'] = path.abspath(
path.expanduser(working_directory))
profile_conf = Profile(profile)
if logs_folder is None:
name = path.splitext(path.basename(jobs_file))[0]
logs_folder = '{}_{}'.format(name, datetime.now().strftime("%Y%m%d"))
manager = Submitted(jobs_file,
logs_folder,
profile_conf,
max_running_jobs=max_running,
group_size=group,
cli_params=JobParameters(**params))
if no_console:
run_plain(manager)
else:
run_console(manager)
def test_submission(tmpdir):
test_number = 0
input_files_dir = path.join(path.dirname(path.abspath(__file__)),
'job_scripts')
for in_file in os.listdir(input_files_dir):
input_file = path.join(input_files_dir, in_file)
folder = str(tmpdir.mkdir('test{}_qmap_output'.format(test_number)))
test_number += 1
Submitted(input_file, folder, TEST_PROFILE)
files = os.listdir(folder)
# Check the input file has been copied
if INPUT_FILE_NAME in files and cmp_files(
input_file, path.join(folder, INPUT_FILE_NAME)):
assert 1
else:
assert 0
pre_commands, commands, post_commands, params = parse_input(input_file)
jobs = len(commands)
pre_processing_cmds = len(pre_commands)
post_processing_commands = len(post_commands)
# Check one file (per job) has been created with the shell script
assert jobs == len([f for f in files if f.endswith('.sh')])
# Check one file (per job) has been created with the job metadata
assert jobs == len([f for f in files if f.endswith('.info')
]) - 1 # + execution.info
# Check that the pre and post commands are in one of the shell scripts
shell_file = next(file for file in files if file.endswith('.sh'))
lines = 0
with open(path.join(folder, shell_file), "rt") as fd:
for line in fd:
l = line.strip()
if l:
if l.startswith('#'):
continue
lines += 1
assert lines == pre_processing_cmds + post_processing_commands + 6
def load_simple(tmp_folder):
input_file = path.join(path.dirname(path.abspath(__file__)), 'job_scripts/simple.txt')
folder = str(tmp_folder.realpath())
execution = Submitted(input_file, folder, TEST_PROFILE, max_running_jobs=1)
return execution