def test_init(self, tmpdir):
temp_path = os.path.join(str(tmpdir), 'temp')
log_path = os.path.join(str(tmpdir), 'log')
os.makedirs(log_path)
kwargs = {
'paths': {
'temp': temp_path,
'log': log_path
},
'steps': [1, 2, 3, 4],
'next_id': 5,
'run_steps': [2, 4],
'run_warnings': 'dummy value',
'run_step_idx': 1,
'dummy_key': 'added me too!'
}
with pytest.raises(pipeline.PipelineError):
pipe = pipeline.Pipeline(**kwargs)
pipe = pipeline.Pipeline(create_paths=True,
pipeline_name='test_pipeline',
**kwargs)
for k, v in kwargs.items():
assert getattr(pipe, k) == v
assert pipe.name == 'test_pipeline'
def test_run_swarp(tmpdir):
paths = {
'temp': os.path.join(str(tmpdir), 'temp'),
'log': os.path.join(str(tmpdir), 'log')
}
setattr(builtins, 'raw_input', mock_raw_input('y'))
setattr(builtins, 'input', mock_raw_input('y'))
pipe = pipeline.Pipeline(paths=paths, build_paths={})
result = api.run_swarp(pipe, 0, ['img1.fits', 'img2.fits'], {})
cmd = 'swarp img1.fits img2.fits -RESAMPLE_DIR {0} '.format(paths['temp'])
cmd += '-WRITE_XML Y -XML_NAME {0}/0.swarp.log.xml'.format(paths['log'])
cmd_result = {
'args': (cmd, False, '{0}/0.swarp.log.xml'.format(paths['log']), True),
'kwargs': {},
'status': 'error'
}
assert result == cmd_result
result = api.run_swarp(pipe, 0, ['img1.fits', 'img2.fits'], {}, frames=[1])
cmd = 'swarp img1.fits[1] img2.fits[1] -RESAMPLE_DIR {0} '.format(
paths['temp'])
cmd += '-WRITE_XML Y -XML_NAME {0}/0.swarp.log-1.xml'.format(paths['log'])
cmd_result = {
'args':
(cmd, False, '{0}/0.swarp.log.xml'.format(paths['log']), False),
'kwargs': {
'frame': '1'
},
'status': 'error',
'warnings': None
}
assert result == cmd_result
def test_empty_init(self):
pipe = pipeline.Pipeline()
assert pipe.create_paths == False
assert pipe.steps == []
assert pipe.next_id == 0
assert pipe.run_steps == None
assert pipe.run_warnings == None
assert pipe.run_step_idx == 0
assert pipe.paths == {}
def test_run_basic(self, tmpdir):
temp_path = os.path.join(str(tmpdir), 'temp')
log_path = os.path.join(str(tmpdir), 'log')
paths = {'temp': temp_path, 'log': log_path}
pipe = pipeline.Pipeline(paths=paths, create_paths=True)
pipe.add_step(test_func1, ['func1'], var1=3, var2=4)
pipe.add_step(test_func2, ['func2'], var1=25, var2=10)
pipe.add_step(test_func2, ['func2'], var1=1, var2=0)
pipe.add_step(test_func3, var1=1, var2=0)
result = pipe.run(ignore_errors=True, ignore_exceptions=True)
assert result['status'] == 'success'
assert pipe.steps[0].results == {
'next_id': 4,
'status': 'success',
'step_id': 0,
'sum': 7
}
assert pipe.steps[1].results == {'diff': 2.5, 'status': 'success'}
assert pipe.steps[2].results == {
'error': 'Division by 0',
'status': 'error'
}
assert pipe.steps[3].results['status'] == 'error'
for step in pipe.steps:
step.results = None
result = pipe.run(['func2'],
ignore_errors=True,
ignore_exceptions=True)
assert pipe.steps[0].results == None
assert pipe.steps[1].results == {'diff': 2.5, 'status': 'success'}
assert pipe.steps[2].results == {
'error': 'Division by 0',
'status': 'error'
}
assert pipe.steps[3].results == None
for step in pipe.steps:
step.results = None
result = pipe.run(ignore_tags=['func2'],
ignore_errors=True,
ignore_exceptions=True)
assert pipe.steps[0].results == {
'next_id': 4,
'status': 'success',
'step_id': 0,
'sum': 7
}
assert pipe.steps[1].results == None
assert pipe.steps[2].results == None
assert pipe.steps[3].results['status'] == 'error'
def test_run_psfex(tmpdir):
paths = {
'temp': os.path.join(str(tmpdir), 'temp'),
'log': os.path.join(str(tmpdir), 'log')
}
setattr(builtins, 'raw_input', mock_raw_input('y'))
setattr(builtins, 'input', mock_raw_input('y'))
pipe = pipeline.Pipeline(paths=paths, build_paths={})
result = api.run_psfex(pipe, 0, ['cat1.fits', 'cat2.fits'], {})
cmd = 'psfex cat1.fits cat2.fits -PSF_DIR {0} '.format(paths['temp'])
cmd += '-WRITE_XML Y -XML_NAME {0}/0.psfex.log.xml'.format(paths['log'])
cmd_result = {
'args': (cmd, False, '{0}/0.psfex.log.xml'.format(paths['log']), True),
'kwargs': {},
'status': 'error'
}
assert result == cmd_result
def test_run_scamp(tmpdir):
paths = {
'temp': os.path.join(str(tmpdir), 'temp'),
'log': os.path.join(str(tmpdir), 'log')
}
setattr(builtins, 'raw_input', mock_raw_input('y'))
setattr(builtins, 'input', mock_raw_input('y'))
pipe = pipeline.Pipeline(paths=paths, build_paths={})
result = api.run_scamp(pipe, 0, ['cat1.fits', 'cat2.fits'], {},
'new_cat.fits')
cmd = 'scamp cat1.fits cat2.fits -SAVE_REFCATALOG Y -REFOUT_CATPATH new_cat.fits '
cmd += '-WRITE_XML Y -XML_NAME {0}/0.scamp.log.xml'.format(paths['log'])
cmd_result = {
'args': (cmd, False, '{0}/0.scamp.log.xml'.format(paths['log']), True),
'kwargs': {},
'status': 'error'
}
assert result == cmd_result
def test_add_step(self):
def test_function(pipeline, step_id, var1, var2):
result = {
'status': success,
'next_id': pipeline.next_id,
'step_id': step_id,
'sum': var1 + var2
}
return result
pipe = pipeline.Pipeline()
pipe.add_step(test_function, ['tag1', 'tag2'], var1=5, var2=10)
step = pipe.steps[0]
assert len(pipe.steps) == 1
assert pipe.next_id == 1
assert step.func == test_function
assert set(step.tags) == set(['tag1', 'tag2'])
assert step.func_kwargs == {'var1': 5, 'var2': 10}
assert step.results == None
def test_run_sex(tmpdir):
paths = {
'temp': os.path.join(str(tmpdir), 'temp'),
'log': os.path.join(str(tmpdir), 'log')
}
setattr(builtins, 'raw_input', mock_raw_input('y'))
setattr(builtins, 'input', mock_raw_input('y'))
pipe = pipeline.Pipeline(paths=paths, build_paths={})
files = {
'image': 'img.fits',
'dqmask': 'img.dqmask.fits',
'wtmap': 'img.wtmap.fits'
}
kwargs = {
'config': OrderedDict([
('PARAMETERS_NAME', 'default.path'),
])
}
result = api.run_sex(pipe, 0, files, kwargs)
cmd = 'sex img.fits -PARAMETERS_NAME default.path -CATALOG_NAME img.cat '
cmd += '-FLAG_IMAGE img.dqmask.fits -WEIGHT_IMAGE img.wtmap.fits '
cmd += '-WRITE_XML Y -XML_NAME {0}/0.sex.log.xml'.format(paths['log'])
cmd_result = {
'args': (cmd, False, '{0}/0.sex.log.xml'.format(paths['log']), True),
'kwargs': {},
'status': 'error'
}
assert result == cmd_result
result = api.run_sex(pipe, 0, files, kwargs, [1])
cmd = 'sex img.fits[1] -PARAMETERS_NAME default.path -CATALOG_NAME img.cat '
cmd += '-FLAG_IMAGE img.dqmask.fits[1] -WEIGHT_IMAGE img.wtmap.fits[1] '
cmd += '-WRITE_XML Y -XML_NAME {0}/0.sex.log-1.xml'.format(paths['log'])
cmd_result = {
'args': (cmd, False, '{0}/0.sex.log.xml'.format(paths['log']), False),
'kwargs': {
'frame': '1'
},
'status': 'error',
'warnings': None
}
assert result == cmd_result
def test_run_advanced(self, tmpdir):
temp_path = os.path.join(str(tmpdir), 'temp')
log_path = os.path.join(str(tmpdir), 'log')
paths = {'temp': temp_path, 'log': log_path}
pipe = pipeline.Pipeline(paths=paths, create_paths=True)
pipe.add_step(test_func1, ['func1'], var1=3, var2=4)
pipe.add_step(test_func2, ['func2'], var1=25, var2=10)
pipe.add_step(test_func2, ['func2'], var1=1, var2=0)
pipe.add_step(test_func3, var1=1, var2=0)
with pytest.raises(pipeline.PipelineError):
pipe.run()
pipe = dill.load(open(os.path.join(paths['log'], 'pipeline.p'), 'rb'))
assert pipe.run_step_idx == 2
assert pipe.steps[0].results == {
'next_id': 4,
'status': 'success',
'step_id': 0,
'sum': 7
}
with pytest.raises(ZeroDivisionError):
for step in pipe.steps:
step.results = None
pipe.run(resume=True, ignore_errors=True)
new_pipe = dill.load(
open(os.path.join(paths['log'], 'pipeline.p'), 'rb'))
result = new_pipe.run(start_idx=1,
ignore_errors=True,
ignore_exceptions=True)
assert result['status'] == 'success'
assert new_pipe.steps[0].results == None
assert new_pipe.steps[1].results == {'diff': 2.5, 'status': 'success'}
assert new_pipe.steps[2].results == {
'error': 'Division by 0',
'status': 'error'
}
assert new_pipe.steps[3].results['status'] == 'error'