def test_job_script_extra_options() -> None: job_desc = JobDescription() job_desc.extra_scheduler_options = '--gres=gpu:1' script = _job_desc_to_job_script(job_desc) assert '#SBATCH --gres=gpu:1' in script
def test_num_nodes(scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs if isinstance(sched, DirectGnuScheduler): # this scheduler runs everything on the same node # and ignores the num_nodes attribute return job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.num_nodes = 2 if isinstance(sched, TorqueScheduler): job_desc.command = 'wc' job_desc.arguments = ['-l', '$PBS_NODEFILE'] elif isinstance(sched, SlurmScheduler): job_desc.command = 'echo' job_desc.arguments = ['$SLURM_JOB_NUM_NODES'] job_desc.queue_name = 'batch' job_desc.stdout_file = '/home/cerulean/test_num_nodes.out' job_id = sched.submit(job_desc) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) outfile = fs / 'home/cerulean/test_num_nodes.out' num_nodes_output = outfile.read_text() assert '2' in outfile.read_text() outfile.unlink()
def test_job_script_name() -> None: job_desc = JobDescription() job_desc.name = 'test_name' script = _job_desc_to_job_script(job_desc) assert '#SBATCH --job-name=test_name' in script
def test_job_script_processes_per_node() -> None: job_desc = JobDescription() job_desc.mpi_processes_per_node = 4 script = _job_desc_to_job_script(job_desc) assert '#SBATCH --ntasks-per-node=4' in script
def test_job_script_extra_scheduler_options() -> None: job_desc = JobDescription() job_desc.extra_scheduler_options = '-p 10' script = _job_desc_to_job_script(job_desc) assert '#PBS -p 10' in script
def test_job_script_processes_per_node() -> None: job_desc = JobDescription() job_desc.mpi_processes_per_node = 4 script = _job_desc_to_job_script(job_desc) assert 'ppn=4' in script
def test_job_script_name() -> None: job_desc = JobDescription() job_desc.name = 'test_name' script = _job_desc_to_job_script(job_desc) assert '#PBS -N test_name' in script
def test_job_script_num_nodes() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.num_nodes = 42 script = _job_desc_to_job_script(job_desc) assert '#SBATCH --nodes=42' in script
def test_job_script_time_reserved() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.time_reserved = 70 script = _job_desc_to_job_script(job_desc) assert '00-00:01:10' in script
def test_job_script_queue_name() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.queue_name = 'testing_queue' script = _job_desc_to_job_script(job_desc) assert '#SBATCH --partition=testing_queue' in script
def test_job_script_stderr_file() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.stderr_file = '/home/user/test.err' script = _job_desc_to_job_script(job_desc) assert '/home/user/test.err' in script
def test_job_script_working_directory() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.working_directory = '/home/user/workdir' script = _job_desc_to_job_script(job_desc) assert '/home/user/workdir' in script
def test_job_script_command_args() -> None: # Note: doesn't test that it works, that's what test_scheduler is for job_desc = JobDescription() job_desc.command = 'echo' job_desc.arguments = ['-n', 'Hello world', 'testing'] script = _job_desc_to_job_script(job_desc) assert "echo -n Hello world testing" in script
def test_scheduler_no_command( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched = scheduler_and_fs[0] job_desc = JobDescription() with pytest.raises(ValueError): sched.submit(job_desc)
def test_scheduler_exit_code(scheduler_and_fs: Tuple[Scheduler, FileSystem], caplog: Any) -> None: caplog.set_level(logging.DEBUG) sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'exit' job_desc.arguments = ['5'] job_id = sched.submit(job_desc) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) retval = sched.get_exit_code(job_id) assert retval == 5
def test_queue_name(scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs if isinstance(sched, DirectGnuScheduler): # this scheduler ignores queues return job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'echo' job_desc.arguments = ['$SLURM_JOB_PARTITION', '$PBS_QUEUE'] job_desc.queue_name = 'batch' job_desc.stdout_file = '/home/cerulean/test_queue_name.out' job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) retval = sched.get_exit_code(job_id) assert retval == 0 outfile = fs / 'home/cerulean/test_queue_name.out' assert 'batch' in outfile.read_text() outfile.unlink()
def test_scheduler_timeout( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = '/usr/local/bin/endless-job.sh' job_desc.time_reserved = 2 job_id = sched.submit(job_desc) while sched.get_status(job_id) != JobStatus.RUNNING: time.sleep(0.1) t = 0.0 while sched.get_status(job_id) != JobStatus.DONE: time.sleep(1.0) t += 1.0 assert t < 100.0
def test_prefix(request: Any, scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, fixture_id = scheduler_and_fs # We have tests running in parallel, so use a unique name prefix_file = 'prefixtest_{}.txt'.format(fixture_id) prefix_path = fs / 'home' / 'cerulean' / prefix_file print(prefix_path) setattr(sched, '_{}__prefix'.format(sched.__class__.__name__), 'echo prefixtest >>{} ;'.format(prefix_path)) job_desc = JobDescription() job_desc.working_directory = '/home' job_desc.command = 'ls' job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) output_lines = len(prefix_path.read_text().splitlines()) assert output_lines >= 1 print('getting status') while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) print('getting status') output_lines = len(prefix_path.read_text().splitlines()) - output_lines assert output_lines >= 1 print('getting exit code') retval = sched.get_exit_code(job_id) assert retval == 0 output_lines = len(prefix_path.read_text().splitlines()) - output_lines assert output_lines >= 1 print('cancelling') sched.cancel(job_id) output_lines = len(prefix_path.read_text().splitlines()) - output_lines assert output_lines >= 1 prefix_path.unlink() setattr(sched, '_{}__prefix'.format(sched.__class__.__name__), '')
def test_scheduler(scheduler_and_fs: Tuple[Scheduler, FileSystem], caplog: Any) -> None: caplog.set_level(logging.DEBUG) sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home' job_desc.command = 'ls' job_desc.arguments = ['-l'] job_desc.stdout_file = '/home/cerulean/test_scheduler.out' job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) retval = sched.get_exit_code(job_id) assert retval == 0 try: output = (fs / 'home/cerulean/test_scheduler.out').read_text() except FileNotFoundError: msg = '' for path in (fs / 'home/cerulean').iterdir(): msg += '{}\n'.format(path) pytest.xfail('Output file not found, to be investigated.' ' Debug output: {}'.format(msg)) assert 'cerulean' in output (fs / 'home/cerulean/test_scheduler.out').unlink()
def test_scheduler_cancel(scheduler_and_fs: Tuple[Scheduler, FileSystem], caplog: Any) -> None: caplog.set_level(logging.DEBUG) sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'sleep' job_desc.arguments = ['15'] job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) while sched.get_status(job_id) != JobStatus.RUNNING: time.sleep(1.0) sched.cancel(job_id) t = 0.0 while sched.get_status(job_id) != JobStatus.DONE: time.sleep(1.0) t += 1.0 assert t < 10.0
def test_system_err_redirect( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'bash' job_desc.arguments = ['-c', 'for i in x ; do something invalid'] job_desc.time_reserved = 1 job_desc.stderr_file = '/dev/null' job_desc.system_err_file = '/home/cerulean/test_sys_redirect.err' job_id = sched.submit(job_desc) sched.wait(job_id) syserr = (fs / 'home/cerulean/test_sys_redirect.err').read_text() retval = sched.get_exit_code(job_id) assert retval != 0 assert 'syntax error' in syserr
def test_system_err_redirect2( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'while [ a = a ] ; do echo bla >/dev/null; done' job_desc.time_reserved = 1 job_desc.stderr_file = '/dev/null' job_desc.system_err_file = '/home/cerulean/test_sys_redirect.err' job_id = sched.submit(job_desc) sched.wait(job_id) syserr = (fs / 'home/cerulean/test_sys_redirect.err').read_text() print('Sys err: {}'.format(syserr)) assert 'CANCELLED' in syserr or 'killed' in syserr or 'Killed' in syserr
def test_system_out_redirect( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'ls' job_desc.time_reserved = 1 job_desc.stdout_file = '/dev/null' job_desc.system_out_file = '/home/cerulean/test_sys_redirect.out' job_id = sched.submit(job_desc) sched.wait(job_id) sysout = (fs / 'home/cerulean/test_sys_redirect.out').read_text() retval = sched.get_exit_code(job_id) assert retval == 0 assert sysout == ''
def test_environment(scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.environment['ENVIRONMENT_TEST1'] = 'test_environment_value1' job_desc.environment['ENVIRONMENT_TEST2'] = 'test_environment_value2' job_desc.command = 'echo' job_desc.arguments = ['$ENVIRONMENT_TEST1', '$ENVIRONMENT_TEST2'] job_desc.stdout_file = '/home/cerulean/test_environment.out' job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) retval = sched.get_exit_code(job_id) assert retval == 0 outfile = fs / 'home/cerulean/test_environment.out' assert 'test_environment_value1' in outfile.read_text() assert 'test_environment_value2' in outfile.read_text() outfile.unlink()
def test_scheduler_wait(scheduler_and_fs: Tuple[Scheduler, FileSystem], caplog: Any) -> None: caplog.set_level(logging.DEBUG) sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home/cerulean' job_desc.command = 'ls' job_desc.time_reserved = 60 job_id = sched.submit(job_desc) exit_code = sched.wait(job_id) assert exit_code == 0 job_desc.command = '/usr/local/bin/endless-job.sh' job_id = sched.submit(job_desc) exit_code = sched.wait(job_id, 1.0) assert exit_code is None sched.cancel(job_id)
def test_stderr_redirect( scheduler_and_fs: Tuple[Scheduler, FileSystem]) -> None: sched, fs, _ = scheduler_and_fs job_desc = JobDescription() job_desc.working_directory = '/home' job_desc.command = 'ls' job_desc.arguments = ['--non-existing-option'] job_desc.stderr_file = '/home/cerulean/test_stderr_redirect.out' job_id = sched.submit(job_desc) print('Job id: {}'.format(job_id)) while sched.get_status(job_id) != JobStatus.DONE: time.sleep(10.0) retval = sched.get_exit_code(job_id) assert retval == 2 outfile = fs / 'home/cerulean/test_stderr_redirect.out' assert 'unrecognized option' in outfile.read_text() outfile.unlink()