def ad(request, ad_factory, path_to_outputs):
"""
Loads existing input FITS files as AstroData objects, runs the
`extract1DSpectra` primitive on it, and return the output object containing
the extracted 1d spectrum. This makes tests more efficient because the
primitive is run only once, instead of N x Numbes of tests.
If the input file does not exist, this fixture raises a IOError.
If the input file does not exist and PyTest is called with the
`--force-preprocess-data`, this fixture looks for cached raw data and
process it. If the raw data does not exist, it is then cached via download
from the Gemini Archive.
Parameters
----------
request : fixture
PyTest's built-in fixture with information about the test itself.
ad_factory : fixture
Custom fixture defined in the `conftest.py` file that loads cached data,
or download and/or process it if needed.
path_to_outputs : fixture
Custom fixture defined in `astrodata.testing` containing the path to the
output folder.
Returns
-------
AstroData
Object containing Wavelength Solution table.
Raises
------
IOError
If the input file does not exist and if --force-preprocess-data is False.
"""
fname, arc_name, ap_center = request.param
p = primitives_gmos_spect.GMOSSpect([])
p.viewer = geminidr.dormantViewer(p, None)
print('\n\n Running test inside folder:\n {}'.format(path_to_outputs))
arc_ad = ad_factory(arc_name, preprocess_arc_recipe)
_ad = ad_factory(fname, preprocess_recipe, center=ap_center, arc=arc_ad)
ad_out = p.extract1DSpectra([_ad], method="standard", width=None,
grow=10)[0]
tests_failed_before_module = request.session.testsfailed
yield ad_out
if request.session.testsfailed > tests_failed_before_module:
_dir = os.path.join(path_to_outputs, os.path.dirname(fname))
os.makedirs(_dir, exist_ok=True)
fname_out = os.path.join(_dir, ad_out.filename)
ad_out.write(filename=fname_out, overwrite=True)
print('\n Saved file to:\n {}\n'.format(fname_out))
del ad_out
def test_regression_for_determine_distortion_using_models_coefficients(
ad, change_working_dir, ref_ad_factory, request):
"""
Runs the `determineDistortion` primitive on a preprocessed data and compare
its model with the one in the reference file.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
request : pytest.fixture
PyTest built-in containing command line options.
"""
with change_working_dir():
logutils.config(file_name='log_model_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(distortion_determined_ad.filename)
for ext, ext_ref in zip(distortion_determined_ad, ref_ad):
c = np.ma.masked_invalid(ext.FITCOORD["coefficients"])
c_ref = np.ma.masked_invalid(ext_ref.FITCOORD["coefficients"])
np.testing.assert_allclose(c, c_ref, atol=2)
if request.config.getoption("--do-plots"):
do_plots(distortion_determined_ad, ref_ad)
def test_fitcoord_table_and_gwcs_match(ad, change_working_dir):
"""
Runs determineDistortion and checks that the model in the gWCS is the same
as the model in the FITCOORD table. The FITCOORD table is never used by
DRAGONS.
Parameters
----------
ad: pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
"""
with change_working_dir():
logutils.config(file_name='log_match_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
model = distortion_determined_ad[0].wcs.get_transform(
"pixels", "distortion_corrected")
fitcoord = distortion_determined_ad[0].FITCOORD
fitcoord_model = am.table_to_model(fitcoord[0])
fitcoord_inv = am.table_to_model(fitcoord[1])
np.testing.assert_allclose(model[1].parameters, fitcoord_model.parameters)
np.testing.assert_allclose(model.inverse[1].parameters,
fitcoord_inv.parameters)
def test_regression_on_extract_1d_spectra(ad, ref_ad_factory,
change_working_dir):
"""
Regression test for the :func:`~geminidr.gmos.GMOSSpect.extract1DSpectra`
primitive.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(
file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.extract1DSpectra(method="standard", width=None, grow=10)
extracted_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(extracted_ad.filename)
for ext, ref_ext in zip(extracted_ad, ref_ad):
assert ext.data.ndim == 1
np.testing.assert_allclose(ext.data, ref_ext.data, atol=1e-3)
def test_regression_for_determine_distortion_using_wcs(ad, change_working_dir,
ref_ad_factory):
"""
Runs the `determineDistortion` primitive on a preprocessed data and compare
its model with the one in the reference file. The distortion model needs to
be reconstructed because different coefficients might return same results.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(
file_name='log_fitcoord_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(distortion_determined_ad.filename)
model = distortion_determined_ad[0].wcs.get_transform(
"pixels", "distortion_corrected")[1]
ref_model = ref_ad[0].wcs.get_transform("pixels",
"distortion_corrected")[1]
X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]]
np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def ad(request, ad_factory, path_to_outputs):
"""
Loads existing input FITS files as AstroData objects, runs the
`traceApertures` primitive on it, and return the output object containing
a `.APERTURE` table.
This makes tests more efficient because the primitive is run only once,
instead of N x Number of tests.
If the input file does not exist, this fixture raises a IOError.
If the input file does not exist and PyTest is called with the
`--force-preprocess-data`, this fixture looks for cached raw data and
process it. If the raw data does not exist, it is then cached via download
from the Gemini Archive.
Parameters
----------
request : fixture
PyTest's built-in fixture with information about the test itself.
ad_factory : fixture
Custom fixture defined in the `conftest.py` file that loads cached data,
or download and/or process it if needed.
path_to_outputs : fixture
Custom fixture defined in `astrodata.testing` containing the path to the
output folder.
Returns
-------
AstroData
Object containing Wavelength Solution table.
"""
fname, ap_center = request.param
p = primitives_gmos_spect.GMOSSpect([])
p.viewer = geminidr.dormantViewer(p, None)
print('\n\n Running test inside folder:\n {}'.format(path_to_outputs))
_ad = ad_factory(fname, preprocess_recipe, **{'center': ap_center})
ad_out = p.traceApertures([_ad],
trace_order=2,
nsum=20,
step=10,
max_shift=0.09,
max_missed=5,
debug=False)[0]
tests_failed_before_module = request.session.testsfailed
yield ad_out
if request.session.testsfailed > tests_failed_before_module:
_dir = os.path.join(path_to_outputs, os.path.dirname(fname))
os.makedirs(_dir, exist_ok=True)
fname_out = os.path.join(_dir, ad_out.filename)
ad_out.write(filename=fname_out, overwrite=True)
print('\n Saved file to:\n {}\n'.format(fname_out))
del ad_out
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr,
caplog, change_working_dir,
ref_ad_factory):
"""
Make sure that the wavelength solution gives same results on different
runs.
"""
caplog.set_level(logging.INFO, logger="geminidr")
with change_working_dir():
logutils.config(
file_name='log_regress_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineWavelengthSolution(
order=order,
min_snr=min_snr,
fwidth=fwidth,
**determine_wavelength_solution_parameters)
wcalibrated_ad = p.writeOutputs().pop()
for record in caplog.records:
if record.levelname == "WARNING":
assert "No acceptable wavelength solution found" not in record.message
ref_ad = ref_ad_factory(wcalibrated_ad.filename)
table = wcalibrated_ad[0].WAVECAL
table_ref = ref_ad[0].WAVECAL
model = astromodels.dict_to_chebyshev(
dict(zip(table["name"], table["coefficients"])))
ref_model = astromodels.dict_to_chebyshev(
dict(zip(table_ref["name"], table_ref["coefficients"])))
x = np.arange(wcalibrated_ad[0].shape[1])
wavelength = model(x)
ref_wavelength = ref_model(x)
pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px
slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace(
'arcsec', ''))
slit_size_in_px = slit_size_in_arcsec / pixel_scale
dispersion = abs(
wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px
tolerance = 0.5 * (slit_size_in_px * dispersion)
np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance)
def reduce(filename):
_p = primitives_gmos_spect.GMOSSpect([astrodata.open(filename)])
_p.viewer = geminidr.dormantViewer(_p, None)
_p.prepare()
_p.addDQ(static_bpm=None)
_p.addVAR(read_noise=True)
_p.overscanCorrect()
_p.ADUToElectrons()
_p.addVAR(poisson_noise=True)
_p.mosaicDetectors()
_p.makeIRAFCompatible()
_p.determineWavelengthSolution(suffix="_arc")
return _p
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr,
caplog, change_working_dir,
ref_ad_factory, request):
"""
Make sure that the wavelength solution gives same results on different
runs.
"""
caplog.set_level(logging.INFO, logger="geminidr")
with change_working_dir():
logutils.config(
file_name='log_regress_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineWavelengthSolution(
order=order,
min_snr=min_snr,
fwidth=fwidth,
**determine_wavelength_solution_parameters)
wcalibrated_ad = p.writeOutputs().pop()
for record in caplog.records:
if record.levelname == "WARNING":
assert "No acceptable wavelength solution found" not in record.message
ref_ad = ref_ad_factory(wcalibrated_ad.filename)
model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform,
"WAVE")
ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE")
x = np.arange(wcalibrated_ad[0].shape[1])
wavelength = model(x)
ref_wavelength = ref_model(x)
pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px
slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace(
'arcsec', ''))
slit_size_in_px = slit_size_in_arcsec / pixel_scale
dispersion = abs(
wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px
tolerance = 0.5 * (slit_size_in_px * dispersion)
np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance)
if request.config.getoption("--do-plots"):
do_plots(wcalibrated_ad)
def test_consistent_air_and_vacuum_solutions(ad, fwidth, order, min_snr):
p = GMOSLongslit([])
p.viewer = geminidr.dormantViewer(p, None)
ad_air = p.determineWavelengthSolution(
[deepcopy(ad)], order=order, min_snr=min_snr, fwidth=fwidth,
in_vacuo=False, **determine_wavelength_solution_parameters).pop()
ad_vac = p.determineWavelengthSolution(
[ad], order=order, min_snr=min_snr, fwidth=fwidth,
in_vacuo=True, **determine_wavelength_solution_parameters).pop()
wave_air = am.get_named_submodel(ad_air[0].wcs.forward_transform, "WAVE")
wave_vac = am.get_named_submodel(ad_vac[0].wcs.forward_transform, "WAVE")
x = np.arange(ad_air[0].shape[1])
wair = wave_air(x)
wvac = air_to_vac(wair * u.nm).to(u.nm).value
dw = wvac - wave_vac(x)
assert abs(dw).max() < 0.001
def test_regression_extract_1d_spectra(ad, change_working_dir,
ref_ad_factory):
with change_working_dir():
logutils.config(
file_name='log_regression_{}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.skyCorrectFromSlit(order=5, grow=0)
sky_subtracted_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(sky_subtracted_ad.filename)
for ext, ref_ext in zip(sky_subtracted_ad, ref_ad):
np.testing.assert_allclose(ext.data, ref_ext.data, atol=0.01)
def test_regression_in_distortion_correct(ad, change_working_dir, ref_ad_factory):
"""
Runs the `distortionCorrect` primitive on a preprocessed data and compare
its model with the one in the reference file.
"""
with change_working_dir():
logutils.config(
file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([deepcopy(ad)])
p.viewer = geminidr.dormantViewer(p, None)
p.distortionCorrect(arc=deepcopy(ad), order=3, subsample=1)
dist_corrected_ad = p.writeOutputs()[0]
ref_ad = ref_ad_factory(dist_corrected_ad.filename)
for ext, ext_ref in zip(dist_corrected_ad, ref_ad):
data = np.ma.masked_invalid(ext.data)
ref_data = np.ma.masked_invalid(ext_ref.data)
np.testing.assert_allclose(data, ref_data, atol=1)
def test_regression_trace_apertures(ad, change_working_dir, ref_ad_factory):
with change_working_dir():
logutils.config(file_name="log_regression_{}.txt".format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.traceApertures()
aperture_traced_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(aperture_traced_ad.filename)
for ext, ref_ext in zip(aperture_traced_ad, ref_ad):
input_table = ext.APERTURE
reference_table = ref_ext.APERTURE
assert input_table['aper_lower'][0] <= 0
assert input_table['aper_upper'][0] >= 0
keys = ext.APERTURE.colnames
actual = np.array([input_table[k] for k in keys])
desired = np.array([reference_table[k] for k in keys])
np.testing.assert_allclose(desired, actual, atol=0.05)
def test_reduced_arcs_contain_wavelength_solution_model_with_expected_rms(
ad, caplog, change_working_dir, fwidth, min_snr, order, request):
"""
Make sure that the WAVECAL model was fitted with an RMS smaller than 0.2
times the FWHM of the arc lines (i.e., less than half of the standard deviation).
"""
caplog.set_level(logging.INFO, logger="geminidr")
with change_working_dir():
# logutils.config(file_name='log_rms_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineWavelengthSolution(
order=order,
min_snr=min_snr,
fwidth=fwidth,
**determine_wavelength_solution_parameters)
wcalibrated_ad = p.writeOutputs().pop()
for record in caplog.records:
if record.levelname == "WARNING":
assert "No acceptable wavelength solution found" not in record.message
if request.config.getoption("--do-plots"):
do_plots(wcalibrated_ad)
table = wcalibrated_ad[0].WAVECAL
tdict = dict(zip(table['name'], table['coefficients']))
rms = tdict['rms']
fwidth = tdict['fwidth']
dispersion = abs(
wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px
required_rms = 0.2 * fwidth * dispersion
np.testing.assert_array_less(rms, required_rms)
def test_regression_for_determine_distortion_using_fitcoord_table(
ad, change_working_dir, ref_ad_factory):
"""
Runs the `determineDistortion` primitive on a preprocessed data and compare
its model with the one in the reference file. The distortion model needs to
be reconstructed because different coefficients might return same results.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(file_name='log_fitcoord_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(distortion_determined_ad.filename)
table = ad[0].FITCOORD
model_dict = dict(zip(table['name'], table['coefficients']))
model = astromodels.dict_to_chebyshev(model_dict)
ref_table = ref_ad[0].FITCOORD
ref_model_dict = dict(zip(ref_table['name'], ref_table['coefficients']))
ref_model = astromodels.dict_to_chebyshev(ref_model_dict)
X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]]
np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def test_regression_sky_correct_from_slit(filename, params, refname,
change_working_dir, path_to_inputs,
path_to_refs):
func = params.get('function', 'spline')
if not func.startswith('spline') and ASTROPY_LT_42:
pytest.skip('Astropy 4.2 is required to use the linear fitter '
'with weights')
path = os.path.join(path_to_inputs, filename)
ad = astrodata.open(path)
with change_working_dir():
logutils.config(file_name=f'log_regression_{ad.data_label()}.txt')
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.skyCorrectFromSlit(**params)
sky_subtracted_ad = p.writeOutputs(outfilename=refname).pop()
ref_ad = astrodata.open(os.path.join(path_to_refs, refname))
for ext, ref_ext in zip(sky_subtracted_ad, ref_ad):
np.testing.assert_allclose(ext.data, ref_ext.data, atol=0.01)
def ad(request, ad_factory, path_to_outputs, path_to_refs):
"""
Loads existing input FITS files as AstroData objects, runs the
`distortionCorrect` primitive on it, and return the output object free of
distortion.
This makes tests more efficient because the primitive is run only once,
instead of N x Numbes of tests.
If the input file does not exist, this fixture raises a IOError.
If the input file does not exist **and** PyTest is called with the
`--force-preprocess-data`, this fixture looks for cached raw data and
process it. If the raw data does not exist, it is then cached via download
from the Gemini Archive.
Parameters
----------
request : fixture
PyTest's built-in fixture with information about the test itself.
ad_factory : fixture
Custom fixture defined in the `conftest.py` file that loads cached data,
or download and/or process it if needed.
path_to_outputs : fixture
Custom fixture defined in `astrodata.testing` containing the path to the
output folder.
path_to_refs : fixture
Custom fixture defined in `astrodata.testing` containing the path to the
cached reference files.
Returns
-------
AstroData
Object containing Wavelength Solution table.
Raises
------
IOError
If the input file does not exist and if --force-preprocess-data is False.
"""
fname = request.param
p = primitives_gmos_spect.GMOSSpect([])
p.viewer = geminidr.dormantViewer(p, None)
print('\n\n Running test inside folder:\n {}'.format(path_to_outputs))
_ad = ad_factory(fname, preprocess_recipe)
ad_out = p.distortionCorrect([_ad], arc=_ad, order=3, subsample=1)[0]
tests_failed_before_module = request.session.testsfailed
yield ad_out
_dir = os.path.join(path_to_outputs, os.path.dirname(request.param))
_ref_dir = os.path.join(path_to_refs, os.path.dirname(request.param))
os.makedirs(_dir, exist_ok=True)
if request.config.getoption("--do-plots"):
do_plots(ad_out, _dir, _ref_dir)
if request.session.testsfailed > tests_failed_before_module:
fname_out = os.path.join(_dir, ad_out.filename)
ad_out.write(filename=fname_out, overwrite=True)
print('\n Saved file to:\n {}\n'.format(fname_out))
del ad_out
