def create_quartz_inputs(): """ Creates input data for tests using pre-processed twilight flat data and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ associated_calibrations = { "N20200715S0059.fits": { "quartz": ["N20200715S0059.fits"], }, "S20130601S0121.fits": { "quartz": ["S20130601S0121.fits"], }, "S20190204S0081.fits": { "quartz": ["S20190204S0081.fits"], }, } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/longslit/test_slit_illum_correct/inputs" path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) cwd = os.getcwd() os.chdir(path) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Download raw files') quartz_path = [download_from_archive(f) for f in cals['quartz']] quartz_ad = astrodata.open(quartz_path[0]) data_label = quartz_ad.data_label() print('Reducing quartz lamp:') logutils.config(file_name='log_quartz_{}.txt'.format(data_label)) with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit([astrodata.open(f) for f in quartz_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() # p.biasCorrect(bias=bias_master) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.stackFrames() # Write non-mosaicked data p.writeOutputs(suffix="_quartz", strip=True) os.chdir(cwd)
def test_fitcoord_table_and_gwcs_match(ad, change_working_dir): """ Runs determineDistortion and checks that the model in the gWCS is the same as the model in the FITCOORD table. The FITCOORD table is never used by DRAGONS. Parameters ---------- ad: pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). """ with change_working_dir(): logutils.config(file_name='log_match_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() model = distortion_determined_ad[0].wcs.get_transform( "pixels", "distortion_corrected") fitcoord = distortion_determined_ad[0].FITCOORD fitcoord_model = am.table_to_model(fitcoord[0]) fitcoord_inv = am.table_to_model(fitcoord[1]) np.testing.assert_allclose(model[1].parameters, fitcoord_model.parameters) np.testing.assert_allclose(model.inverse[1].parameters, fitcoord_inv.parameters)
def test_regression_for_determine_distortion_using_wcs(ad, change_working_dir, ref_ad_factory): """ Runs the `determineDistortion` primitive on a preprocessed data and compare its model with the one in the reference file. The distortion model needs to be reconstructed because different coefficients might return same results. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). """ with change_working_dir(): logutils.config( file_name='log_fitcoord_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(distortion_determined_ad.filename) model = distortion_determined_ad[0].wcs.get_transform( "pixels", "distortion_corrected")[1] ref_model = ref_ad[0].wcs.get_transform("pixels", "distortion_corrected")[1] X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]] np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def test_regression_for_determine_distortion_using_models_coefficients( ad, change_working_dir, ref_ad_factory, request): """ Runs the `determineDistortion` primitive on a preprocessed data and compare its model with the one in the reference file. Parameters ---------- ad : pytest.fixture (AstroData) Fixture that reads the filename and loads as an AstroData object. change_working_dir : pytest.fixture Fixture that changes the working directory (see :mod:`astrodata.testing`). reference_ad : pytest.fixture Fixture that contains a function used to load the reference AstroData object (see :mod:`recipe_system.testing`). request : pytest.fixture PyTest built-in containing command line options. """ with change_working_dir(): logutils.config(file_name='log_model_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineDistortion(**fixed_parameters_for_determine_distortion) distortion_determined_ad = p.writeOutputs().pop() ref_ad = ref_ad_factory(distortion_determined_ad.filename) assert_have_same_distortion(distortion_determined_ad, ref_ad, atol=1) if request.config.getoption("--do-plots"): do_plots(distortion_determined_ad, ref_ad)
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr, caplog, change_working_dir, ref_ad_factory, request): """ Make sure that the wavelength solution gives same results on different runs. """ caplog.set_level(logging.INFO, logger="geminidr") with change_working_dir(): logutils.config( file_name='log_regress_{:s}.txt'.format(ad.data_label())) p = GMOSLongslit([ad]) p.viewer = geminidr.dormantViewer(p, None) p.determineWavelengthSolution( order=order, min_snr=min_snr, fwidth=fwidth, **determine_wavelength_solution_parameters) wcalibrated_ad = p.writeOutputs().pop() for record in caplog.records: if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message ref_ad = ref_ad_factory(wcalibrated_ad.filename) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") x = np.arange(wcalibrated_ad[0].shape[1]) wavelength = model(x) ref_wavelength = ref_model(x) pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace( 'arcsec', '')) slit_size_in_px = slit_size_in_arcsec / pixel_scale dispersion = abs( wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px tolerance = 0.5 * (slit_size_in_px * dispersion) np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance) if request.config.getoption("--do-plots"): do_plots(wcalibrated_ad)
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from geminidr.gmos.tests.spect import CREATED_INPUTS_PATH_FOR_TESTS module_name, _ = os.path.splitext(os.path.basename(__file__)) path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("inputs/", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename in datasets: print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() os.chdir("inputs/") processed_ad = p.writeOutputs().pop() os.chdir("..") print("Wrote pre-processed file to:\n" " {:s}".format(processed_ad.filename))
def create_twilight_inputs(): """ Creates input data for tests using pre-processed twilight flat data and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ associated_calibrations = { "S20190204S0006.fits": { "bias": [ "S20190203S0110.fits", "S20190203S0109.fits", "S20190203S0108.fits", "S20190203S0107.fits", "S20190203S0106.fits" ], "twilight": ["S20190204S0006.fits"], }, "N20190103S0462.fits": { "bias": [ "N20190102S0531.fits", "N20190102S0530.fits", "N20190102S0529.fits", "N20190102S0528.fits", "N20190102S0527.fits" ], "twilight": ["N20190103S0462.fits", "N20190103S0463.fits"], }, "N20190327S0056.fits": { "bias": [ "N20190327S0098.fits", "N20190327S0099.fits", "N20190327S0100.fits", "N20190327S0101.fits", "N20190327S0102.fits" ], "twilight": ["N20190327S0056.fits"], }, "S20130602S0005.fits": { "bias": [ "S20130601S0161.fits", "S20130601S0160.fits", "S20130601S0159.fits", "S20130601S0158.fits", "S20130601S0157.fits", ], "twilight": ["S20130602S0005.fits"], }, "N20190602S0306.fits": { "bias": [ "N20190601S0648.fits", "N20190601S0647.fits", "N20190601S0646.fits", "N20190601S0645.fits", "N20190601S0644.fits", ], "twilight": ["N20190602S0306.fits"], } } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/longslit/test_slit_illumination_correct/inputs" path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) cwd = os.getcwd() os.chdir(path) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Download raw files') bias_path = [download_from_archive(f) for f in cals['bias']] twilight_path = [download_from_archive(f) for f in cals['twilight']] twilight_ad = astrodata.open(twilight_path[0]) data_label = twilight_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) logutils.config(file_name='log_bias_{}.txt'.format(data_label)) bias_reduce = Reduce() bias_reduce.files.extend(bias_path) bias_reduce.runr() bias_master = bias_reduce.output_filenames.pop() del bias_reduce print('Reducing twilight flat:') logutils.config(file_name='log_twilight_{}.txt'.format(data_label)) with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit([astrodata.open(f) for f in twilight_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.biasCorrect(bias=bias_master) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.stackFrames() # Write non-mosaicked data twilight = p.writeOutputs(suffix="_twilight", strip=True)[0] # Write mosaicked data p = GMOSLongslit([twilight]) p.makeSlitIllum() p.writeOutputs() os.chdir(cwd) return
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils associated_calibrations = { "S20190808S0048.fits": { "arcs": ["S20190808S0167.fits"] }, "S20190808S0049.fits": { "arcs": ["S20190808S0168.fits"] }, # "S20190808S0052.fits": {"arcs": ["S20190808S0165.fits"]}, # Can't find aperture "S20190808S0053.fits": { "arcs": ["S20190808S0169.fits"] }, } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Downloading files...') sci_path = download_from_archive(filename) arc_path = [download_from_archive(f) for f in cals['arcs']] sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) arc_reduce = Reduce() arc_reduce.files.extend(arc_path) arc_reduce.runr() arc = arc_reduce.output_filenames.pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")
def create_inputs(): """ Create inputs for `test_plot_spectra_for_qa_single_frame`. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import glob import os from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit from gempy.utils import logutils from recipe_system.reduction.coreReduce import Reduce from recipe_system.utils.reduce_utils import normalize_ucals cwd = os.getcwd() path = f"./dragons_test_inputs/geminidr/core/{__file__.split('.')[0]}/" os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("inputs/", exist_ok=True) for raw_list, bias_list, quartz_list, arc_list in single_aperture_data: if all([ os.path.exists(f"inputs/{s.split('.')[0]}_extracted.fits") for s in raw_list ]): print("Skipping already created input.") continue raw_paths = [download_from_archive(f) for f in raw_list] bias_paths = [download_from_archive(f) for f in bias_list] quartz_paths = [download_from_archive(f) for f in quartz_list] arc_paths = [download_from_archive(f) for f in arc_list] cals = [] raw_ads = [astrodata.open(p) for p in raw_paths] data_label = raw_ads[0].data_label() print('Current working directory:\n {:s}'.format(os.getcwd())) if len(bias_paths): logutils.config(file_name='log_bias_{}.txt'.format(data_label)) r = Reduce() r.files.extend(bias_paths) r.runr() master_bias = r.output_filenames.pop() cals.append(f"processed_bias:{master_bias}") del r else: master_bias = None if len(quartz_paths): logutils.config(file_name='log_quartz_{}.txt'.format(data_label)) r = Reduce() r.files.extend(quartz_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_quartz = r.output_filenames.pop() cals.append(f"processed_flat:{master_quartz}") del r else: master_quartz = None logutils.config(file_name='log_arc_{}.txt'.format(data_label)) r = Reduce() r.files.extend(arc_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_arc = r.output_filenames.pop() logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit(raw_ads) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.biasCorrect(do_bias=master_bias is not None, bias=master_bias) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.flatCorrect(do_flat=master_quartz is not None, flat=master_quartz) p.QECorrect(arc=master_arc) p.distortionCorrect(arc=master_arc) p.findSourceApertures(max_apertures=3) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() p.linearizeSpectra() [os.remove(s) for s in glob.glob("*_arc.fits")] [os.remove(s) for s in glob.glob("*_bias.fits")] [os.remove(s) for s in glob.glob("*_flat.fits")] [os.remove(s) for s in glob.glob("*_mosaic.fits")] os.chdir("inputs/") print("\n\n Writing processed files for tests into:\n" " {:s}\n\n".format(os.getcwd())) _ = p.writeOutputs() os.chdir("../") os.chdir(cwd)
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils from astrodata.testing import get_associated_calibrations associated_calibrations = { "S20190808S0048.fits": 'S20190808S0167.fits', "S20190808S0049.fits": 'S20190808S0168.fits', "S20190808S0053.fits": 'S20190808S0169.fits', "N20180106S0025.fits": 'N20180115S0264.fits', "N20180106S0026.fits": 'N20180115S0264.fits', "N20180106S0028.fits": 'N20180115S0264.fits', "N20180106S0029.fits": 'N20180115S0264.fits', } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for fname, arc_fname in associated_calibrations.items(): sci_path = download_from_archive(fname) arc_path = download_from_archive(arc_fname) sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) if os.path.exists( arc_fname.replace('.fits', '_distortionDetermined.fits')): arc = astrodata.open( arc_fname.replace('.fits', '_distortionDetermined.fits')) else: p = GMOSLongslit([astrodata.open(arc_path)]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() p.determineWavelengthSolution() p.determineDistortion() arc = p.writeOutputs().pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")