def test_psydoit_remerges_if_result_is_missing(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'result.h5')) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 1 assert 'merge' in scheduler.joblist[0]['name']
def test_psydoit_resubmits_for_missing_job_output(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'result.h5')) os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'out', '0.h5')) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 2 assert 'process:0' in scheduler.joblist[0]['name'] assert 'merge' in scheduler.joblist[1]['name']
def test_psydoit_resubmits_jobs_if_necessary(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() shutil.rmtree(taskenv.workdir) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 6 assert 'split' in scheduler.joblist[0]['name'] for i in range(4): assert 'process:{0}'.format(i) in scheduler.joblist[i + 1]['name'] assert 'merge' in scheduler.joblist[5]['name']
def test_psydoit_no_resubmits_if_result_is_uptodate(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() for dirpath, dirnames, filenames in os.walk(taskenv.workdir): for filename in filenames: if filename == 'result.h5': continue os.remove(os.path.join(dirpath, filename)) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 0
def test_psydoit_workdir_contents(taskenv): workdir = os.path.join('psywork', 'square') os.remove(os.path.join(taskenv.taskdir, 'psyconf.py')) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'square']) assert os.path.exists(os.path.join(workdir, 'in', '0.h5')) assert os.path.exists(os.path.join(workdir, 'out', '0.h5')) assert os.path.exists(os.path.join(workdir, 'result.h5')) assert os.path.exists(os.path.join(workdir, 'square:split.py')) assert os.path.exists(os.path.join(workdir, 'square:process:0.py')) assert os.path.exists(os.path.join(workdir, 'square:merge.py')) assert os.path.exists(os.path.join(workdir, 'square:split.log')) assert os.path.exists(os.path.join(workdir, 'square:process:0.log')) assert os.path.exists(os.path.join(workdir, 'square:merge.log'))
def test_psydoit_resubmits_all_if_infile_is_outdated( taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() time.sleep(1) t = time.time() os.utime( os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'), (t, t)) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 6 assert 'split' in scheduler.joblist[0]['name'] for i in range(4): assert 'process:{0}'.format(i) in scheduler.joblist[i + 1]['name'] assert 'merge' in scheduler.joblist[5]['name']
def test_psydoit_resubmits_merge_if_result_is_outdated(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() time.sleep(1) t = time.time() os.utime( os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'), (t, t)) for i in range(4): os.utime(os.path.join( taskenv.workdir, 'mocked_scheduler', 'in', str(i) + '.h5'), (t, t)) os.utime(os.path.join( taskenv.workdir, 'mocked_scheduler', 'out', str(i) + '.h5'), (t, t)) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 1 assert 'merge' in scheduler.joblist[0]['name']
def test_psydoit_resubmits_process_and_merge_if_outfile_is_outdated( taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() time.sleep(1) t = time.time() os.utime( os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'), (t, t)) for i in range(4): os.utime(os.path.join( taskenv.workdir, 'mocked_scheduler', 'in', str(i) + '.h5'), (t, t)) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(scheduler.joblist) == 5 for i in range(4): assert 'process:{0}'.format(i) in scheduler.joblist[i]['name'] assert 'merge' in scheduler.joblist[4]['name']
def test_psydoit_file_dep(taskenv): with open(os.path.join(taskenv.taskdir, 'in.txt'), 'w') as f: f.write('2') psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'file_dep']) result = load_dict_h5(os.path.join( taskenv.workdir, 'file_dep', 'result.h5')) assert sorted(result['y']) == [4] # Ensure that modification time changes as some file systems only support # 1s resolution. time.sleep(1) with open(os.path.join(taskenv.taskdir, 'in.txt'), 'w') as f: f.write('3') psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'file_dep']) result = load_dict_h5(os.path.join( taskenv.workdir, 'file_dep', 'result.h5')) assert sorted(result['y']) == [8]
def test_psydoit_does_not_resubmit_split_if_infiles_uptodate( taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) scheduler.consume() for dirpath, dirnames, filenames in os.walk(taskenv.workdir): for filename in filenames: if os.path.basename(dirpath) == 'out': continue os.remove(os.path.join(dirpath, filename)) psydoit(taskenv.taskdir, [ 'list', '-s', '--all', '--db-file', taskenv.dbfile, 'mocked_scheduler']) psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) for job in scheduler.joblist: assert 'split' not in job['name']
def test_psydoit(taskenv): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'square']) result = load_dict_h5(os.path.join(taskenv.workdir, 'square', 'result.h5')) assert sorted(result['y']) == [0, 1, 4, 9]
def test_working_directory(taskenv): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, '-v', '2', 'workingdir'])
def test_psydoit_does_not_resubmit_queued_jobs(taskenv, scheduler): psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) expected = scheduler.joblist psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler']) assert len(expected) == len(scheduler.joblist) assert all(x['id'] == y['id'] for x, y in zip(expected, scheduler.joblist))