def process_object(filename, arc, suffix="linearized"): """ Helper recipe to reduce the object file. Returns ------- AstroData Processed arc. """ from astrodata.testing import download_from_archive from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit processed_filename, ext = os.path.splitext(filename) processed_filename += "_{:s}{:s}".format(suffix, ext) if os.path.exists(processed_filename): ad = astrodata.open(processed_filename) else: if os.path.exists(filename): ad = astrodata.open(filename) else: ad = astrodata.open( download_from_archive(filename, path='', env_var='DRAGONS_TEST')) p = GMOSLongslit([ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() # p.biasCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) # p.flatCorrect() # p.applyQECorrection() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() p.linearizeSpectra() # TODO: needed? p.calculateSensitivity() ad = p.streams['main'][0] ad.write(overwrite=True) return ad
def test_distortion_correct(config): """ Corrects distortion on both output and reference files using the distortion model stored in themselves. Previous tests assures that these data are similar and that distortion correct is applied the same way. Now, this one tests the model itself. """ # Recover name of the distortion corrected arc files --- basename = os.path.basename(config.filename) filename, extension = os.path.splitext(basename) filename = filename.split("_")[0] + "_distortionDetermined" + extension output = os.path.join(config.output_dir, filename) reference = os.path.join(config.ref_dir, filename) if not os.path.exists(output): pytest.fail("Processed arc file not found: {}".format(output)) if not os.path.exists(reference): pytest.fail( "Processed reference file not found: {}".format(reference)) p = GMOSLongslit([]) ad_out = astrodata.open(output) ad_out_corrected_with_out = p.distortionCorrect([ad_out], arc=output)[0] ad_out_corrected_with_ref = p.distortionCorrect([ad_out], arc=reference)[0] ad_out_corrected_with_out.write( overwrite=True, filename=os.path.join(config.output_dir, ad_out_corrected_with_out.filename), ) for ext_out, ext_ref in zip(ad_out_corrected_with_out, ad_out_corrected_with_ref): np.testing.assert_allclose(ext_out.data, ext_ref.data)
def test_distortion_correction_is_applied_the_same_way(config): """ Applies the same distortion correction model to both output and reference arcs and compares the results. """ # Process the output file --- basename, ext = os.path.splitext(config.filename) basename, _ = basename.split("_")[0], basename.split("_")[1:] arc_basename = "{:s}_{:s}{:s}".format(basename, "distortionDetermined", ext) arc_name = os.path.join(config.output_dir, arc_basename) if not os.path.exists(arc_name): pytest.skip("Arc file not found: {}".format(arc_name)) ad_out = config.ad p = GMOSLongslit([]) ad_out = p.distortionCorrect(adinputs=[ad_out], arc=arc_name)[0] filename = ad_out.filename ad_out = p.determineDistortion(adinputs=[ad_out])[0] for ext in ad_out: assert "distortion_corrected" in ext.wcs.available_frames ad_out.write(filename=filename, overwrite=True) # assert False # # Reads the reference file --- # reference = os.path.join(config.ref_dir, ad_out.filename) # # if not os.path.exists(reference): # pytest.fail('Reference file not found: {}'.format(reference)) # # ad_ref = astrodata.open(reference) os.rename(filename, os.path.join(config.output_dir, filename))
def test_distortionCorrect_works_when_using_FullROI_on_CentralROI( path_to_outputs): """ Test if a model obtained in a Full Frame ROI can be applied to a Central Spectra ROI. """ class Config: def __init__(self, name): sub_path, basename = os.path.split(name) basename, ext = os.path.splitext(basename) basename = basename + "_distortionDetermined.fits" self._filename = basename self.output_dir = os.path.join(path_to_outputs, sub_path) self.full_name = os.path.join(self.output_dir, self._filename) @property def filename(self): return self._filename @filename.setter def filename(self, name): self._filename = name self.full_name = os.path.join(self.output_dir, self._filename) # B600:0.500 HAM, ROI="Central Spectrum" --- cs = Config("N20171016S0010.fits") cs.ad = astrodata.open(os.path.join(cs.output_dir, cs.filename)) # B600:0.500 HAM, ROI="Full Frame" --- ff = Config("N20171016S0127.fits") # Apply full frame roi to central-spect roi p = GMOSLongslit([]) cs.ad = p.distortionCorrect(adinputs=[cs.ad], arc=ff.full_name)[0] cs.filename = cs.filename.replace(".fits", "_fromFullFrame.fits") cs.ad.write(filename=cs.full_name, overwrite=True)
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils associated_calibrations = { "S20190808S0048.fits": { "arcs": ["S20190808S0167.fits"] }, "S20190808S0049.fits": { "arcs": ["S20190808S0168.fits"] }, # "S20190808S0052.fits": {"arcs": ["S20190808S0165.fits"]}, # Can't find aperture "S20190808S0053.fits": { "arcs": ["S20190808S0169.fits"] }, } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, cals in associated_calibrations.items(): print('Downloading files...') sci_path = download_from_archive(filename) arc_path = [download_from_archive(f) for f in cals['arcs']] sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) arc_reduce = Reduce() arc_reduce.files.extend(arc_path) arc_reduce.runr() arc = arc_reduce.output_filenames.pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")
def create_inputs(): """ Create inputs for `test_plot_spectra_for_qa_single_frame`. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import glob import os from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit from gempy.utils import logutils from recipe_system.reduction.coreReduce import Reduce from recipe_system.utils.reduce_utils import normalize_ucals cwd = os.getcwd() path = f"./dragons_test_inputs/geminidr/core/{__file__.split('.')[0]}/" os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("inputs/", exist_ok=True) for raw_list, bias_list, quartz_list, arc_list in single_aperture_data: if all([ os.path.exists(f"inputs/{s.split('.')[0]}_extracted.fits") for s in raw_list ]): print("Skipping already created input.") continue raw_paths = [download_from_archive(f) for f in raw_list] bias_paths = [download_from_archive(f) for f in bias_list] quartz_paths = [download_from_archive(f) for f in quartz_list] arc_paths = [download_from_archive(f) for f in arc_list] cals = [] raw_ads = [astrodata.open(p) for p in raw_paths] data_label = raw_ads[0].data_label() print('Current working directory:\n {:s}'.format(os.getcwd())) if len(bias_paths): logutils.config(file_name='log_bias_{}.txt'.format(data_label)) r = Reduce() r.files.extend(bias_paths) r.runr() master_bias = r.output_filenames.pop() cals.append(f"processed_bias:{master_bias}") del r else: master_bias = None if len(quartz_paths): logutils.config(file_name='log_quartz_{}.txt'.format(data_label)) r = Reduce() r.files.extend(quartz_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_quartz = r.output_filenames.pop() cals.append(f"processed_flat:{master_quartz}") del r else: master_quartz = None logutils.config(file_name='log_arc_{}.txt'.format(data_label)) r = Reduce() r.files.extend(arc_paths) r.ucals = normalize_ucals(r.files, cals) r.runr() master_arc = r.output_filenames.pop() logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit(raw_ads) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.biasCorrect(do_bias=master_bias is not None, bias=master_bias) p.ADUToElectrons() p.addVAR(poisson_noise=True) p.flatCorrect(do_flat=master_quartz is not None, flat=master_quartz) p.QECorrect(arc=master_arc) p.distortionCorrect(arc=master_arc) p.findSourceApertures(max_apertures=3) p.skyCorrectFromSlit() p.traceApertures() p.extract1DSpectra() p.linearizeSpectra() [os.remove(s) for s in glob.glob("*_arc.fits")] [os.remove(s) for s in glob.glob("*_bias.fits")] [os.remove(s) for s in glob.glob("*_flat.fits")] [os.remove(s) for s in glob.glob("*_mosaic.fits")] os.chdir("inputs/") print("\n\n Writing processed files for tests into:\n" " {:s}\n\n".format(os.getcwd())) _ = p.writeOutputs() os.chdir("../") os.chdir(cwd)
def create_inputs_recipe(): """ Creates input data for tests using pre-processed standard star and its calibration files. The raw files will be downloaded and saved inside the path stored in the `$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside a new folder called "dragons_test_inputs". The sub-directory structure should reflect the one returned by the `path_to_inputs` fixture. """ import os from astrodata.testing import download_from_archive from recipe_system.reduction.coreReduce import Reduce from gempy.utils import logutils from astrodata.testing import get_associated_calibrations associated_calibrations = { "S20190808S0048.fits": 'S20190808S0167.fits', "S20190808S0049.fits": 'S20190808S0168.fits', "S20190808S0053.fits": 'S20190808S0169.fits', "N20180106S0025.fits": 'N20180115S0264.fits', "N20180106S0026.fits": 'N20180115S0264.fits', "N20180106S0028.fits": 'N20180115S0264.fits', "N20180106S0029.fits": 'N20180115S0264.fits', } root_path = os.path.join("./dragons_test_inputs/") module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0]) path = os.path.join(root_path, module_path) os.makedirs(path, exist_ok=True) os.chdir(path) os.makedirs("./inputs", exist_ok=True) print('Current working directory:\n {:s}'.format(os.getcwd())) for fname, arc_fname in associated_calibrations.items(): sci_path = download_from_archive(fname) arc_path = download_from_archive(arc_fname) sci_ad = astrodata.open(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) if os.path.exists( arc_fname.replace('.fits', '_distortionDetermined.fits')): arc = astrodata.open( arc_fname.replace('.fits', '_distortionDetermined.fits')) else: p = GMOSLongslit([astrodata.open(arc_path)]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.makeIRAFCompatible() p.determineWavelengthSolution() p.determineDistortion() arc = p.writeOutputs().pop() print('Reducing pre-processed data:') logutils.config(file_name='log_{}.txt'.format(data_label)) p = GMOSLongslit([sci_ad]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) p.overscanCorrect() p.ADUToElectrons() p.addVAR(poisson_noise=True) p.mosaicDetectors() p.distortionCorrect(arc=arc) p.findSourceApertures(max_apertures=1) p.skyCorrectFromSlit() os.chdir("inputs/") _ = p.writeOutputs().pop() os.chdir("../")