def ad(request, ad_factory, path_to_outputs): """ Loads existing input FITS files as AstroData objects, runs the `extract1DSpectra` primitive on it, and return the output object containing the extracted 1d spectrum. This makes tests more efficient because the primitive is run only once, instead of N x Numbes of tests. If the input file does not exist, this fixture raises a IOError. If the input file does not exist and PyTest is called with the `--force-preprocess-data`, this fixture looks for cached raw data and process it. If the raw data does not exist, it is then cached via download from the Gemini Archive. Parameters ---------- request : fixture PyTest's built-in fixture with information about the test itself. ad_factory : fixture Custom fixture defined in the `conftest.py` file that loads cached data, or download and/or process it if needed. path_to_outputs : fixture Custom fixture defined in `astrodata.testing` containing the path to the output folder. Returns ------- AstroData Object containing Wavelength Solution table. Raises ------ IOError If the input file does not exist and if --force-preprocess-data is False. """ fname, arc_name, ap_center = request.param p = primitives_gmos_spect.GMOSSpect([]) p.viewer = geminidr.dormantViewer(p, None) print('\n\n Running test inside folder:\n {}'.format(path_to_outputs)) arc_ad = ad_factory(arc_name, preprocess_arc_recipe) _ad = ad_factory(fname, preprocess_recipe, center=ap_center, arc=arc_ad) ad_out = p.extract1DSpectra([_ad], method="standard", width=None, grow=10)[0] tests_failed_before_module = request.session.testsfailed yield ad_out if request.session.testsfailed > tests_failed_before_module: _dir = os.path.join(path_to_outputs, os.path.dirname(fname)) os.makedirs(_dir, exist_ok=True) fname_out = os.path.join(_dir, ad_out.filename) ad_out.write(filename=fname_out, overwrite=True) print('\n Saved file to:\n {}\n'.format(fname_out)) del ad_out
def test_regression_for_determine_distortion_using_models_coefficients( ad, change_working_dir, ref_ad_factory, request): """ Runs the `determineDistortion` primitive on a preprocessed data and compare its model with the one in the reference file. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). request : pytest.fixture PyTest built-in containing command line options. """ with change_working_dir(): logutils.config(file_name='log_model_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(distortion_determined_ad.filename) for ext, ext_ref in zip(distortion_determined_ad, ref_ad): c = np.ma.masked_invalid(ext.FITCOORD["coefficients"]) c_ref = np.ma.masked_invalid(ext_ref.FITCOORD["coefficients"]) np.testing.assert_allclose(c, c_ref, atol=2) if request.config.getoption("--do-plots"): do_plots(distortion_determined_ad, ref_ad)
def test_fitcoord_table_and_gwcs_match(ad, change_working_dir): """ Runs determineDistortion and checks that the model in the gWCS is the same as the model in the FITCOORD table. The FITCOORD table is never used by DRAGONS. Parameters ---------- ad: pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). """ with change_working_dir(): logutils.config(file_name='log_match_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() model = distortion_determined_ad[0].wcs.get_transform( "pixels", "distortion_corrected") fitcoord = distortion_determined_ad[0].FITCOORD fitcoord_model = am.table_to_model(fitcoord[0]) fitcoord_inv = am.table_to_model(fitcoord[1]) np.testing.assert_allclose(model[1].parameters, fitcoord_model.parameters) np.testing.assert_allclose(model.inverse[1].parameters, fitcoord_inv.parameters)
def test_regression_on_extract_1d_spectra(ad, ref_ad_factory, change_working_dir): """ Regression test for the :func:`~geminidr.gmos.GMOSSpect.extract1DSpectra` primitive. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). """ with change_working_dir(): logutils.config( file_name='log_regression_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.extract1DSpectra(method="standard", width=None, grow=10) extracted_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(extracted_ad.filename) for ext, ref_ext in zip(extracted_ad, ref_ad): assert ext.data.ndim == 1 np.testing.assert_allclose(ext.data, ref_ext.data, atol=1e-3)
def test_regression_for_determine_distortion_using_wcs(ad, change_working_dir, ref_ad_factory): """ Runs the `determineDistortion` primitive on a preprocessed data and compare its model with the one in the reference file. The distortion model needs to be reconstructed because different coefficients might return same results. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). """ with change_working_dir(): logutils.config( file_name='log_fitcoord_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(distortion_determined_ad.filename) model = distortion_determined_ad[0].wcs.get_transform( "pixels", "distortion_corrected")[1] ref_model = ref_ad[0].wcs.get_transform("pixels", "distortion_corrected")[1] X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]] np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def ad(request, ad_factory, path_to_outputs): """ Loads existing input FITS files as AstroData objects, runs the `traceApertures` primitive on it, and return the output object containing a `.APERTURE` table. This makes tests more efficient because the primitive is run only once, instead of N x Number of tests. If the input file does not exist, this fixture raises a IOError. If the input file does not exist and PyTest is called with the `--force-preprocess-data`, this fixture looks for cached raw data and process it. If the raw data does not exist, it is then cached via download from the Gemini Archive. Parameters ---------- request : fixture PyTest's built-in fixture with information about the test itself. ad_factory : fixture Custom fixture defined in the `conftest.py` file that loads cached data, or download and/or process it if needed. path_to_outputs : fixture Custom fixture defined in `astrodata.testing` containing the path to the output folder. Returns ------- AstroData Object containing Wavelength Solution table. """ fname, ap_center = request.param p = primitives_gmos_spect.GMOSSpect([]) p.viewer = geminidr.dormantViewer(p, None) print('\n\n Running test inside folder:\n {}'.format(path_to_outputs)) _ad = ad_factory(fname, preprocess_recipe, **{'center': ap_center}) ad_out = p.traceApertures([_ad], trace_order=2, nsum=20, step=10, max_shift=0.09, max_missed=5, debug=False)[0] tests_failed_before_module = request.session.testsfailed yield ad_out if request.session.testsfailed > tests_failed_before_module: _dir = os.path.join(path_to_outputs, os.path.dirname(fname)) os.makedirs(_dir, exist_ok=True) fname_out = os.path.join(_dir, ad_out.filename) ad_out.write(filename=fname_out, overwrite=True) print('\n Saved file to:\n {}\n'.format(fname_out)) del ad_out
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr, caplog, change_working_dir, ref_ad_factory): """ Make sure that the wavelength solution gives same results on different runs. """ caplog.set_level(logging.INFO, logger="geminidr") with change_working_dir(): logutils.config( file_name='log_regress_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineWavelengthSolution( order=order, min_snr=min_snr, fwidth=fwidth, **determine_wavelength_solution_parameters) wcalibrated_ad = p.writeOutputs().pop() for record in caplog.records: if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message ref_ad = ref_ad_factory(wcalibrated_ad.filename) table = wcalibrated_ad[0].WAVECAL table_ref = ref_ad[0].WAVECAL model = astromodels.dict_to_chebyshev( dict(zip(table["name"], table["coefficients"]))) ref_model = astromodels.dict_to_chebyshev( dict(zip(table_ref["name"], table_ref["coefficients"]))) x = np.arange(wcalibrated_ad[0].shape[1]) wavelength = model(x) ref_wavelength = ref_model(x) pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace( 'arcsec', '')) slit_size_in_px = slit_size_in_arcsec / pixel_scale dispersion = abs( wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px tolerance = 0.5 * (slit_size_in_px * dispersion) np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance)
def reduce(filename): _p = primitives_gmos_spect.GMOSSpect([astrodata.open(filename)]) _p.viewer = geminidr.dormantViewer(_p, None) _p.prepare() _p.addDQ(static_bpm=None) _p.addVAR(read_noise=True) _p.overscanCorrect() _p.ADUToElectrons() _p.addVAR(poisson_noise=True) _p.mosaicDetectors() _p.makeIRAFCompatible() _p.determineWavelengthSolution(suffix="_arc") return _p
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr, caplog, change_working_dir, ref_ad_factory, request): """ Make sure that the wavelength solution gives same results on different runs. """ caplog.set_level(logging.INFO, logger="geminidr") with change_working_dir(): logutils.config( file_name='log_regress_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineWavelengthSolution( order=order, min_snr=min_snr, fwidth=fwidth, **determine_wavelength_solution_parameters) wcalibrated_ad = p.writeOutputs().pop() for record in caplog.records: if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message ref_ad = ref_ad_factory(wcalibrated_ad.filename) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") x = np.arange(wcalibrated_ad[0].shape[1]) wavelength = model(x) ref_wavelength = ref_model(x) pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace( 'arcsec', '')) slit_size_in_px = slit_size_in_arcsec / pixel_scale dispersion = abs( wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px tolerance = 0.5 * (slit_size_in_px * dispersion) np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance) if request.config.getoption("--do-plots"): do_plots(wcalibrated_ad)
def test_consistent_air_and_vacuum_solutions(ad, fwidth, order, min_snr): p = GMOSLongslit([]) p.viewer = geminidr.dormantViewer(p, None) ad_air = p.determineWavelengthSolution( [deepcopy(ad)], order=order, min_snr=min_snr, fwidth=fwidth, in_vacuo=False, **determine_wavelength_solution_parameters).pop() ad_vac = p.determineWavelengthSolution( [ad], order=order, min_snr=min_snr, fwidth=fwidth, in_vacuo=True, **determine_wavelength_solution_parameters).pop() wave_air = am.get_named_submodel(ad_air[0].wcs.forward_transform, "WAVE") wave_vac = am.get_named_submodel(ad_vac[0].wcs.forward_transform, "WAVE") x = np.arange(ad_air[0].shape[1]) wair = wave_air(x) wvac = air_to_vac(wair * u.nm).to(u.nm).value dw = wvac - wave_vac(x) assert abs(dw).max() < 0.001
def test_regression_extract_1d_spectra(ad, change_working_dir, ref_ad_factory): with change_working_dir(): logutils.config( file_name='log_regression_{}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.skyCorrectFromSlit(order=5, grow=0) sky_subtracted_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(sky_subtracted_ad.filename) for ext, ref_ext in zip(sky_subtracted_ad, ref_ad): np.testing.assert_allclose(ext.data, ref_ext.data, atol=0.01)
def test_regression_in_distortion_correct(ad, change_working_dir, ref_ad_factory): """ Runs the `distortionCorrect` primitive on a preprocessed data and compare its model with the one in the reference file. """ with change_working_dir(): logutils.config( file_name='log_regression_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_longslit.GMOSLongslit([deepcopy(ad)]) p.viewer = geminidr.dormantViewer(p, None) p.distortionCorrect(arc=deepcopy(ad), order=3, subsample=1) dist_corrected_ad = p.writeOutputs()[0] ref_ad = ref_ad_factory(dist_corrected_ad.filename) for ext, ext_ref in zip(dist_corrected_ad, ref_ad): data = np.ma.masked_invalid(ext.data) ref_data = np.ma.masked_invalid(ext_ref.data) np.testing.assert_allclose(data, ref_data, atol=1)
def test_regression_trace_apertures(ad, change_working_dir, ref_ad_factory): with change_working_dir(): logutils.config(file_name="log_regression_{}.txt".format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.traceApertures() aperture_traced_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(aperture_traced_ad.filename) for ext, ref_ext in zip(aperture_traced_ad, ref_ad): input_table = ext.APERTURE reference_table = ref_ext.APERTURE assert input_table['aper_lower'][0] <= 0 assert input_table['aper_upper'][0] >= 0 keys = ext.APERTURE.colnames actual = np.array([input_table[k] for k in keys]) desired = np.array([reference_table[k] for k in keys]) np.testing.assert_allclose(desired, actual, atol=0.05)
def test_reduced_arcs_contain_wavelength_solution_model_with_expected_rms( ad, caplog, change_working_dir, fwidth, min_snr, order, request): """ Make sure that the WAVECAL model was fitted with an RMS smaller than 0.2 times the FWHM of the arc lines (i.e., less than half of the standard deviation). """ caplog.set_level(logging.INFO, logger="geminidr") with change_working_dir(): # logutils.config(file_name='log_rms_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineWavelengthSolution( order=order, min_snr=min_snr, fwidth=fwidth, **determine_wavelength_solution_parameters) wcalibrated_ad = p.writeOutputs().pop() for record in caplog.records: if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message if request.config.getoption("--do-plots"): do_plots(wcalibrated_ad) table = wcalibrated_ad[0].WAVECAL tdict = dict(zip(table['name'], table['coefficients'])) rms = tdict['rms'] fwidth = tdict['fwidth'] dispersion = abs( wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px required_rms = 0.2 * fwidth * dispersion np.testing.assert_array_less(rms, required_rms)
def test_regression_for_determine_distortion_using_fitcoord_table( ad, change_working_dir, ref_ad_factory): """ Runs the `determineDistortion` primitive on a preprocessed data and compare its model with the one in the reference file. The distortion model needs to be reconstructed because different coefficients might return same results. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). """ with change_working_dir(): logutils.config(file_name='log_fitcoord_{:s}.txt'.format(ad.data_label())) p = primitives_gmos_spect.GMOSSpect([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(distortion_determined_ad.filename) table = ad[0].FITCOORD model_dict = dict(zip(table['name'], table['coefficients'])) model = astromodels.dict_to_chebyshev(model_dict) ref_table = ref_ad[0].FITCOORD ref_model_dict = dict(zip(ref_table['name'], ref_table['coefficients'])) ref_model = astromodels.dict_to_chebyshev(ref_model_dict) X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]] np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def test_regression_sky_correct_from_slit(filename, params, refname, change_working_dir, path_to_inputs, path_to_refs): func = params.get('function', 'spline') if not func.startswith('spline') and ASTROPY_LT_42: pytest.skip('Astropy 4.2 is required to use the linear fitter ' 'with weights') path = os.path.join(path_to_inputs, filename) ad = astrodata.open(path) with change_working_dir(): logutils.config(file_name=f'log_regression_{ad.data_label()}.txt') p = primitives_gmos_longslit.GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.skyCorrectFromSlit(**params) sky_subtracted_ad = p.writeOutputs(outfilename=refname).pop() ref_ad = astrodata.open(os.path.join(path_to_refs, refname)) for ext, ref_ext in zip(sky_subtracted_ad, ref_ad): np.testing.assert_allclose(ext.data, ref_ext.data, atol=0.01)
def ad(request, ad_factory, path_to_outputs, path_to_refs): """ Loads existing input FITS files as AstroData objects, runs the `distortionCorrect` primitive on it, and return the output object free of distortion. This makes tests more efficient because the primitive is run only once, instead of N x Numbes of tests. If the input file does not exist, this fixture raises a IOError. If the input file does not exist **and** PyTest is called with the `--force-preprocess-data`, this fixture looks for cached raw data and process it. If the raw data does not exist, it is then cached via download from the Gemini Archive. Parameters ---------- request : fixture PyTest's built-in fixture with information about the test itself. ad_factory : fixture Custom fixture defined in the `conftest.py` file that loads cached data, or download and/or process it if needed. path_to_outputs : fixture Custom fixture defined in `astrodata.testing` containing the path to the output folder. path_to_refs : fixture Custom fixture defined in `astrodata.testing` containing the path to the cached reference files. Returns ------- AstroData Object containing Wavelength Solution table. Raises ------ IOError If the input file does not exist and if --force-preprocess-data is False. """ fname = request.param p = primitives_gmos_spect.GMOSSpect([]) p.viewer = geminidr.dormantViewer(p, None) print('\n\n Running test inside folder:\n {}'.format(path_to_outputs)) _ad = ad_factory(fname, preprocess_recipe) ad_out = p.distortionCorrect([_ad], arc=_ad, order=3, subsample=1)[0] tests_failed_before_module = request.session.testsfailed yield ad_out _dir = os.path.join(path_to_outputs, os.path.dirname(request.param)) _ref_dir = os.path.join(path_to_refs, os.path.dirname(request.param)) os.makedirs(_dir, exist_ok=True) if request.config.getoption("--do-plots"): do_plots(ad_out, _dir, _ref_dir) if request.session.testsfailed > tests_failed_before_module: fname_out = os.path.join(_dir, ad_out.filename) ad_out.write(filename=fname_out, overwrite=True) print('\n Saved file to:\n {}\n'.format(fname_out)) del ad_out