def test_psydoit_remerges_if_result_is_missing(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'result.h5'))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 1
assert 'merge' in scheduler.joblist[0]['name']
def test_psydoit_resubmits_for_missing_job_output(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'result.h5'))
os.remove(os.path.join(taskenv.workdir, 'mocked_scheduler', 'out', '0.h5'))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 2
assert 'process:0' in scheduler.joblist[0]['name']
assert 'merge' in scheduler.joblist[1]['name']
def test_psydoit_resubmits_jobs_if_necessary(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
shutil.rmtree(taskenv.workdir)
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 6
assert 'split' in scheduler.joblist[0]['name']
for i in range(4):
assert 'process:{0}'.format(i) in scheduler.joblist[i + 1]['name']
assert 'merge' in scheduler.joblist[5]['name']
def test_psydoit_no_resubmits_if_result_is_uptodate(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
for dirpath, dirnames, filenames in os.walk(taskenv.workdir):
for filename in filenames:
if filename == 'result.h5':
continue
os.remove(os.path.join(dirpath, filename))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 0
def test_psydoit_workdir_contents(taskenv):
workdir = os.path.join('psywork', 'square')
os.remove(os.path.join(taskenv.taskdir, 'psyconf.py'))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'square'])
assert os.path.exists(os.path.join(workdir, 'in', '0.h5'))
assert os.path.exists(os.path.join(workdir, 'out', '0.h5'))
assert os.path.exists(os.path.join(workdir, 'result.h5'))
assert os.path.exists(os.path.join(workdir, 'square:split.py'))
assert os.path.exists(os.path.join(workdir, 'square:process:0.py'))
assert os.path.exists(os.path.join(workdir, 'square:merge.py'))
assert os.path.exists(os.path.join(workdir, 'square:split.log'))
assert os.path.exists(os.path.join(workdir, 'square:process:0.log'))
assert os.path.exists(os.path.join(workdir, 'square:merge.log'))
def test_psydoit_resubmits_all_if_infile_is_outdated(
taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
time.sleep(1)
t = time.time()
os.utime(
os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'),
(t, t))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 6
assert 'split' in scheduler.joblist[0]['name']
for i in range(4):
assert 'process:{0}'.format(i) in scheduler.joblist[i + 1]['name']
assert 'merge' in scheduler.joblist[5]['name']
def test_psydoit_resubmits_merge_if_result_is_outdated(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
time.sleep(1)
t = time.time()
os.utime(
os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'),
(t, t))
for i in range(4):
os.utime(os.path.join(
taskenv.workdir, 'mocked_scheduler', 'in', str(i) + '.h5'), (t, t))
os.utime(os.path.join(
taskenv.workdir, 'mocked_scheduler', 'out', str(i) + '.h5'), (t, t))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 1
assert 'merge' in scheduler.joblist[0]['name']
def test_psydoit_resubmits_process_and_merge_if_outfile_is_outdated(
taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
time.sleep(1)
t = time.time()
os.utime(
os.path.join(taskenv.taskdir, 'task_mocked_scheduler.py'),
(t, t))
for i in range(4):
os.utime(os.path.join(
taskenv.workdir, 'mocked_scheduler', 'in', str(i) + '.h5'), (t, t))
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(scheduler.joblist) == 5
for i in range(4):
assert 'process:{0}'.format(i) in scheduler.joblist[i]['name']
assert 'merge' in scheduler.joblist[4]['name']
def test_psydoit_file_dep(taskenv):
with open(os.path.join(taskenv.taskdir, 'in.txt'), 'w') as f:
f.write('2')
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'file_dep'])
result = load_dict_h5(os.path.join(
taskenv.workdir, 'file_dep', 'result.h5'))
assert sorted(result['y']) == [4]
# Ensure that modification time changes as some file systems only support
# 1s resolution.
time.sleep(1)
with open(os.path.join(taskenv.taskdir, 'in.txt'), 'w') as f:
f.write('3')
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'file_dep'])
result = load_dict_h5(os.path.join(
taskenv.workdir, 'file_dep', 'result.h5'))
assert sorted(result['y']) == [8]
def test_psydoit_does_not_resubmit_split_if_infiles_uptodate(
taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
scheduler.consume()
for dirpath, dirnames, filenames in os.walk(taskenv.workdir):
for filename in filenames:
if os.path.basename(dirpath) == 'out':
continue
os.remove(os.path.join(dirpath, filename))
psydoit(taskenv.taskdir, [
'list', '-s', '--all', '--db-file', taskenv.dbfile,
'mocked_scheduler'])
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
for job in scheduler.joblist:
assert 'split' not in job['name']
def test_psydoit(taskenv):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'square'])
result = load_dict_h5(os.path.join(taskenv.workdir, 'square', 'result.h5'))
assert sorted(result['y']) == [0, 1, 4, 9]
def test_working_directory(taskenv):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, '-v', '2', 'workingdir'])
def test_psydoit_does_not_resubmit_queued_jobs(taskenv, scheduler):
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
expected = scheduler.joblist
psydoit(taskenv.taskdir, ['--db-file', taskenv.dbfile, 'mocked_scheduler'])
assert len(expected) == len(scheduler.joblist)
assert all(x['id'] == y['id'] for x, y in zip(expected, scheduler.joblist))
