def perform_ini_save_load_cycle(self, args): """ Performs a full cycle of saving the ini file from the provided args, then loading and parsing the ini file """ from core.submit import bilby_ini_to_args from bilby_pipe.data_generation import DataGenerationInput ini = args_to_bilby_ini(args) args = bilby_ini_to_args(ini.decode('utf-8')) args.idx = 1 args.ini = None input_args = DataGenerationInput(args, [], create_data=False) input_args.create_data(args) return input_args
def test_submit_simulated_data_job_condor(self, *args, **kwargs): # Generate a minimal ini file ini = args_to_bilby_ini({ 'label': 'test-simulated', 'detectors': ['H1', 'V1'], 'trigger-time': '87654321', 'n-simulation': 1, 'gaussian_noise': True, 'injection-numbers': [] }).decode('utf-8') details = {'job_id': 1} with TemporaryDirectory() as td: global working_directory_mock_return, get_unique_job_id_mock_return, submit_mock_return, \ update_job_result update_job_result = None working_directory_mock_return = os.path.join(td, 'job') # Local imports so that the mocks work as expected from core.submit import submit submit_mock_return = 1234 get_unique_job_id_mock_return = 4321 params = dict(name='test-real', description='Some description', ini_string=ini) result = submit(details, json.dumps(params)) # Check that the return value (The internal bundle submission id) is correct self.assertEqual(result, get_unique_job_id_mock_return) # Check that the internal job object was correctly created self.assertEqual(update_job_result['job_id'], get_unique_job_id_mock_return) self.assertEqual(update_job_result['submit_id'], submit_mock_return) self.assertEqual(update_job_result['working_directory'], td) self.assertEqual(update_job_result['submit_directory'], 'job/submit') # Check that the master slurm script was correctly modified with open( os.path.join(td, 'job', 'submit', 'dag_test-simulated.submit'), 'r') as f: self.assertEqual(f.read( ), """JOB test-simulated_data0_87654321-0_generation_arg_0 job/submit/test-simulated_data0_87654321-0_generation.submit VARS test-simulated_data0_87654321-0_generation_arg_0 ARGS="job/test-simulated_config_complete.ini --label test-simulated_data0_87654321-0_generation --idx 0 --trigger-time 87654321.0" JOB test-simulated_data0_87654321-0_analysis_H1V1_dynesty_arg_0 job/submit/test-simulated_data0_87654321-0_analysis_H1V1_dynesty.submit VARS test-simulated_data0_87654321-0_analysis_H1V1_dynesty_arg_0 ARGS="job/test-simulated_config_complete.ini --detectors H1 --detectors V1 --label test-simulated_data0_87654321-0_analysis_H1V1_dynesty --data-dump-file job/data/test-simulated_data0_87654321-0_generation_data_dump.pickle --sampler dynesty" JOB test-simulated_data0_87654321-0_analysis_H1V1_dynesty_final_result_arg_0 job/submit/test-simulated_data0_87654321-0_analysis_H1V1_dynesty_final_result.submit VARS test-simulated_data0_87654321-0_analysis_H1V1_dynesty_final_result_arg_0 ARGS="--result job/result/test-simulated_data0_87654321-0_analysis_H1V1_dynesty_result.json --outdir job/final_result --extension json --max-samples 20000 --lightweight --save" JOB test-simulated_data0_87654321-0_analysis_H1V1_dynesty_plot_arg_0 job/submit/test-simulated_data0_87654321-0_analysis_H1V1_dynesty_plot.submit VARS test-simulated_data0_87654321-0_analysis_H1V1_dynesty_plot_arg_0 ARGS="--result job/result/test-simulated_data0_87654321-0_analysis_H1V1_dynesty_result.json --outdir job/result --corner --marginal --skymap --waveform --format png" #Inter-job dependencies Parent test-simulated_data0_87654321-0_generation_arg_0 Child test-simulated_data0_87654321-0_analysis_H1V1_dynesty_arg_0 Parent test-simulated_data0_87654321-0_analysis_H1V1_dynesty_arg_0 Child test-simulated_data0_87654321-0_analysis_H1V1_dynesty_final_result_arg_0 Parent test-simulated_data0_87654321-0_analysis_H1V1_dynesty_arg_0 Child test-simulated_data0_87654321-0_analysis_H1V1_dynesty_plot_arg_0""" # noqa ) # Check that the ini file was correctly updated with open( os.path.join(td, 'job', 'test-simulated_config_complete.ini'), 'r') as f: from core.submit import bilby_ini_to_args args = bilby_ini_to_args(f.read()) self.assertEqual(args.label, 'test-simulated') self.assertEqual(args.detectors, ["'H1'", "'V1'"]) self.assertEqual(args.trigger_time, '87654321') self.assertEqual(args.n_simulation, 1) self.assertEqual(args.gaussian_noise, True) self.assertEqual(args.outdir, os.path.join(td, 'job')) self.assertEqual(args.periodic_restart_time, 28800) self.assertEqual(args.scheduler, settings.scheduler.value) self.assertEqual(args.scheduler_env, settings.scheduler_env) self.assertEqual(args.accounting, 'no.group') self.assertEqual(args.transfer_files, False)
def test_submit_real_data_job_slurm(self, *args, **kwargs): # Generate a minimal ini file ini = args_to_bilby_ini({ 'label': 'test-real', 'detectors': ['H1'], 'trigger-time': '12345678', 'injection-numbers': [] }).decode('utf-8') details = {'job_id': 1} with TemporaryDirectory() as td: global working_directory_mock_return, get_unique_job_id_mock_return, submit_mock_return, \ update_job_result update_job_result = None working_directory_mock_return = os.path.join(td, 'job') # Configure the popen data generation mock popen_command = f'/bin/bash {td}/job/submit/test-real_data0_12345678-0_generation.sh' self.popen.set_command(popen_command, stdout=b'stdout test', stderr=b'stderr test') # Local imports so that the mocks work as expected from core.submit import submit submit_mock_return = 1234 get_unique_job_id_mock_return = 4321 params = dict(name='test-real', description='Some description', ini_string=ini) result = submit(details, json.dumps(params)) # Check that the return value (The internal bundle submission id) is correct self.assertEqual(result, get_unique_job_id_mock_return) # Check that the internal job object was correctly created self.assertEqual(update_job_result['job_id'], get_unique_job_id_mock_return) self.assertEqual(update_job_result['submit_id'], submit_mock_return) self.assertEqual(update_job_result['working_directory'], td) self.assertEqual(update_job_result['submit_directory'], 'job/submit') # Check that the job script generation successfully called the the popen command process = call.Popen(popen_command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=td, shell=True) compare(self.popen.all_calls, expected=[ process, process.wait(), process.communicate(), process.wait() ]) # Check the stdout and stderr logs for the data generation step are correctly written to their respective # log files with open( os.path.join(td, 'job', 'log_data_generation', 'test-real_data0_12345678-0_generation.out'), 'rb') as f: self.assertEqual(f.read(), b'stdout test') with open( os.path.join(td, 'job', 'log_data_generation', 'test-real_data0_12345678-0_generation.err'), 'rb') as f: self.assertEqual(f.read(), b'stderr test') # Check that the master slurm script was correctly modified with open( os.path.join(td, 'job', 'submit', 'slurm_test-real_master.sh'), 'r') as f: self.assertEqual(f.read(), """#!/bin/bash #SBATCH --time=00:10:00 #SBATCH --output=job/submit/test-real_master_slurm.out #SBATCH --error=job/submit/test-real_master_slurm.err jid1=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=4G --time=7-00:00:00 --job-name=test-real_data0_12345678-0_analysis_H1_dynesty --output=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty.out --error=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty.err job/submit/test-real_data0_12345678-0_analysis_H1_dynesty.sh)) echo "jid1 ${jid1[-1]}" >> job/submit/slurm_ids jid2=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=4G --time=1:00:00 --job-name=test-real_data0_12345678-0_analysis_H1_dynesty_final_result --dependency=afterok:${jid1[-1]} --output=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty_final_result.out --error=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty_final_result.err job/submit/test-real_data0_12345678-0_analysis_H1_dynesty_final_result.sh)) echo "jid2 ${jid2[-1]}" >> job/submit/slurm_ids jid3=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=32G --time=1:00:00 --job-name=test-real_data0_12345678-0_analysis_H1_dynesty_plot --dependency=afterok:${jid1[-1]} --output=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty_plot.out --error=job/log_data_analysis/test-real_data0_12345678-0_analysis_H1_dynesty_plot.err job/submit/test-real_data0_12345678-0_analysis_H1_dynesty_plot.sh)) echo "jid3 ${jid3[-1]}" >> job/submit/slurm_ids """ # noqa ) # Check that the ini file was correctly updated with open( os.path.join(td, 'job', 'test-real_config_complete.ini'), 'r') as f: from core.submit import bilby_ini_to_args args = bilby_ini_to_args(f.read()) self.assertEqual(args.label, 'test-real') self.assertEqual(args.detectors, ["'H1'"]) self.assertEqual(args.trigger_time, '12345678') self.assertEqual(args.outdir, os.path.join(td, 'job')) self.assertEqual(args.periodic_restart_time, 2147483647) self.assertEqual(args.scheduler, settings.scheduler.value) self.assertEqual(args.scheduler_env, settings.scheduler_env) self.assertEqual(args.transfer_files, False)
def test_submit_simulated_data_job_submission_error_slurm( self, *args, **kwargs): # Generate a minimal ini file ini = args_to_bilby_ini({ 'label': 'test-simulated-submission-failure', 'detectors': ['V1', 'L1'], 'trigger-time': '11111111', 'n-simulation': 1, 'gaussian_noise': True, 'injection-numbers': [] }).decode('utf-8') details = {'job_id': 1} with TemporaryDirectory() as td: global working_directory_mock_return, submit_mock_return, \ update_job_result update_job_result = None working_directory_mock_return = os.path.join(td, 'job') # Configure the popen data generation mock popen_command = f'/bin/bash {td}/submit/test-simulated_data0_12345678-0_generation.sh' self.popen.set_command(popen_command, stdout=b'stdout test', stderr=b'stderr test') # Local imports so that the mocks work as expected from core.submit import submit submit_mock_return = None params = dict(name='test-simulated-submission-failure', description='Some description', ini_string=ini) result = submit(details, json.dumps(params)) # Check that the return value (The internal bundle submission id) is correct self.assertEqual(result, None) # Check that the internal job object not created self.assertEqual(update_job_result, None) # Check that the job script generation did not call the the popen command compare(self.popen.all_calls, expected=[]) # Check the stdout and stderr logs for the data generation step do not exist self.assertFalse( os.path.exists( os.path.join( td, 'log_data_generation', 'test-simulated_data0_12345678-0_generation.out'))) self.assertFalse( os.path.exists( os.path.join( td, 'log_data_generation', 'test-simulated_data0_12345678-0_generation.err'))) # Check that the master slurm script was correctly modified with open( os.path.join( td, 'job', 'submit', 'slurm_test-simulated-submission-failure_master.sh'), 'r') as f: self.assertEqual(f.read(), """#!/bin/bash #SBATCH --time=00:10:00 #SBATCH --output=job/submit/test-simulated-submission-failure_master_slurm.out #SBATCH --error=job/submit/test-simulated-submission-failure_master_slurm.err jid0=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=8G --time=1:00:00 --job-name=test-simulated-submission-failure_data0_11111111-0_generation --output=job/log_data_generation/test-simulated-submission-failure_data0_11111111-0_generation.out --error=job/log_data_generation/test-simulated-submission-failure_data0_11111111-0_generation.err job/submit/test-simulated-submission-failure_data0_11111111-0_generation.sh)) echo "jid0 ${jid0[-1]}" >> job/submit/slurm_ids jid1=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=4G --time=7-00:00:00 --job-name=test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty --dependency=afterok:${jid0[-1]} --output=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty.out --error=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty.err job/submit/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty.sh)) echo "jid1 ${jid1[-1]}" >> job/submit/slurm_ids jid2=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=4G --time=1:00:00 --job-name=test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_final_result --dependency=afterok:${jid1[-1]} --output=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_final_result.out --error=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_final_result.err job/submit/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_final_result.sh)) echo "jid2 ${jid2[-1]}" >> job/submit/slurm_ids jid3=($(sbatch --nodes=1 --ntasks-per-node=1 --mem=32G --time=1:00:00 --job-name=test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_plot --dependency=afterok:${jid1[-1]} --output=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_plot.out --error=job/log_data_analysis/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_plot.err job/submit/test-simulated-submission-failure_data0_11111111-0_analysis_L1V1_dynesty_plot.sh)) echo "jid3 ${jid3[-1]}" >> job/submit/slurm_ids """ # noqa ) # Check that the ini file was correctly updated with open( os.path.join( td, 'job', 'test-simulated-submission-failure_config_complete.ini' ), 'r') as f: from core.submit import bilby_ini_to_args args = bilby_ini_to_args(f.read()) self.assertEqual(args.label, 'test-simulated-submission-failure') self.assertEqual(args.detectors, ["'V1'", "'L1'"]) self.assertEqual(args.trigger_time, '11111111') self.assertEqual(args.n_simulation, 1) self.assertEqual(args.gaussian_noise, True) self.assertEqual(args.outdir, os.path.join(td, 'job')) self.assertEqual(args.periodic_restart_time, 2147483647) self.assertEqual(args.scheduler, settings.scheduler.value) self.assertEqual(args.scheduler_env, settings.scheduler_env) self.assertEqual(args.transfer_files, False)