def process_object(filename, arc, suffix="linearized"):
"""
Helper recipe to reduce the object file.
Returns
-------
AstroData
Processed arc.
"""
from astrodata.testing import download_from_archive
from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit
processed_filename, ext = os.path.splitext(filename)
processed_filename += "_{:s}{:s}".format(suffix, ext)
if os.path.exists(processed_filename):
ad = astrodata.open(processed_filename)
else:
if os.path.exists(filename):
ad = astrodata.open(filename)
else:
ad = astrodata.open(
download_from_archive(filename,
path='',
env_var='DRAGONS_TEST'))
p = GMOSLongslit([ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
# p.biasCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
# p.flatCorrect()
# p.applyQECorrection()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra() # TODO: needed?
p.calculateSensitivity()
ad = p.streams['main'][0]
ad.write(overwrite=True)
return ad
def test_distortion_correct(config):
"""
Corrects distortion on both output and reference files using the
distortion model stored in themselves. Previous tests assures that
these data are similar and that distortion correct is applied the same
way. Now, this one tests the model itself.
"""
# Recover name of the distortion corrected arc files ---
basename = os.path.basename(config.filename)
filename, extension = os.path.splitext(basename)
filename = filename.split("_")[0] + "_distortionDetermined" + extension
output = os.path.join(config.output_dir, filename)
reference = os.path.join(config.ref_dir, filename)
if not os.path.exists(output):
pytest.fail("Processed arc file not found: {}".format(output))
if not os.path.exists(reference):
pytest.fail(
"Processed reference file not found: {}".format(reference))
p = GMOSLongslit([])
ad_out = astrodata.open(output)
ad_out_corrected_with_out = p.distortionCorrect([ad_out],
arc=output)[0]
ad_out_corrected_with_ref = p.distortionCorrect([ad_out],
arc=reference)[0]
ad_out_corrected_with_out.write(
overwrite=True,
filename=os.path.join(config.output_dir,
ad_out_corrected_with_out.filename),
)
for ext_out, ext_ref in zip(ad_out_corrected_with_out,
ad_out_corrected_with_ref):
np.testing.assert_allclose(ext_out.data, ext_ref.data)
def test_distortion_correction_is_applied_the_same_way(config):
"""
Applies the same distortion correction model to both output and reference
arcs and compares the results.
"""
# Process the output file ---
basename, ext = os.path.splitext(config.filename)
basename, _ = basename.split("_")[0], basename.split("_")[1:]
arc_basename = "{:s}_{:s}{:s}".format(basename, "distortionDetermined",
ext)
arc_name = os.path.join(config.output_dir, arc_basename)
if not os.path.exists(arc_name):
pytest.skip("Arc file not found: {}".format(arc_name))
ad_out = config.ad
p = GMOSLongslit([])
ad_out = p.distortionCorrect(adinputs=[ad_out], arc=arc_name)[0]
filename = ad_out.filename
ad_out = p.determineDistortion(adinputs=[ad_out])[0]
for ext in ad_out:
assert "distortion_corrected" in ext.wcs.available_frames
ad_out.write(filename=filename, overwrite=True)
# assert False
# # Reads the reference file ---
# reference = os.path.join(config.ref_dir, ad_out.filename)
#
# if not os.path.exists(reference):
# pytest.fail('Reference file not found: {}'.format(reference))
#
# ad_ref = astrodata.open(reference)
os.rename(filename, os.path.join(config.output_dir, filename))
def test_distortionCorrect_works_when_using_FullROI_on_CentralROI(
path_to_outputs):
"""
Test if a model obtained in a Full Frame ROI can be applied to a Central
Spectra ROI.
"""
class Config:
def __init__(self, name):
sub_path, basename = os.path.split(name)
basename, ext = os.path.splitext(basename)
basename = basename + "_distortionDetermined.fits"
self._filename = basename
self.output_dir = os.path.join(path_to_outputs, sub_path)
self.full_name = os.path.join(self.output_dir, self._filename)
@property
def filename(self):
return self._filename
@filename.setter
def filename(self, name):
self._filename = name
self.full_name = os.path.join(self.output_dir, self._filename)
# B600:0.500 HAM, ROI="Central Spectrum" ---
cs = Config("N20171016S0010.fits")
cs.ad = astrodata.open(os.path.join(cs.output_dir, cs.filename))
# B600:0.500 HAM, ROI="Full Frame" ---
ff = Config("N20171016S0127.fits")
# Apply full frame roi to central-spect roi
p = GMOSLongslit([])
cs.ad = p.distortionCorrect(adinputs=[cs.ad], arc=ff.full_name)[0]
cs.filename = cs.filename.replace(".fits", "_fromFullFrame.fits")
cs.ad.write(filename=cs.full_name, overwrite=True)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from recipe_system.reduction.coreReduce import Reduce
from gempy.utils import logutils
associated_calibrations = {
"S20190808S0048.fits": {
"arcs": ["S20190808S0167.fits"]
},
"S20190808S0049.fits": {
"arcs": ["S20190808S0168.fits"]
},
# "S20190808S0052.fits": {"arcs": ["S20190808S0165.fits"]}, # Can't find aperture
"S20190808S0053.fits": {
"arcs": ["S20190808S0169.fits"]
},
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("./inputs", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
arc_path = [download_from_archive(f) for f in cals['arcs']]
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend(arc_path)
arc_reduce.runr()
arc = arc_reduce.output_filenames.pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
os.chdir("inputs/")
_ = p.writeOutputs().pop()
os.chdir("../")
def create_inputs():
"""
Create inputs for `test_plot_spectra_for_qa_single_frame`.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import glob
import os
from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit
from gempy.utils import logutils
from recipe_system.reduction.coreReduce import Reduce
from recipe_system.utils.reduce_utils import normalize_ucals
cwd = os.getcwd()
path = f"./dragons_test_inputs/geminidr/core/{__file__.split('.')[0]}/"
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
for raw_list, bias_list, quartz_list, arc_list in single_aperture_data:
if all([
os.path.exists(f"inputs/{s.split('.')[0]}_extracted.fits")
for s in raw_list
]):
print("Skipping already created input.")
continue
raw_paths = [download_from_archive(f) for f in raw_list]
bias_paths = [download_from_archive(f) for f in bias_list]
quartz_paths = [download_from_archive(f) for f in quartz_list]
arc_paths = [download_from_archive(f) for f in arc_list]
cals = []
raw_ads = [astrodata.open(p) for p in raw_paths]
data_label = raw_ads[0].data_label()
print('Current working directory:\n {:s}'.format(os.getcwd()))
if len(bias_paths):
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(bias_paths)
r.runr()
master_bias = r.output_filenames.pop()
cals.append(f"processed_bias:{master_bias}")
del r
else:
master_bias = None
if len(quartz_paths):
logutils.config(file_name='log_quartz_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(quartz_paths)
r.ucals = normalize_ucals(r.files, cals)
r.runr()
master_quartz = r.output_filenames.pop()
cals.append(f"processed_flat:{master_quartz}")
del r
else:
master_quartz = None
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(arc_paths)
r.ucals = normalize_ucals(r.files, cals)
r.runr()
master_arc = r.output_filenames.pop()
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit(raw_ads)
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(do_bias=master_bias is not None, bias=master_bias)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.flatCorrect(do_flat=master_quartz is not None, flat=master_quartz)
p.QECorrect(arc=master_arc)
p.distortionCorrect(arc=master_arc)
p.findSourceApertures(max_apertures=3)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra()
[os.remove(s) for s in glob.glob("*_arc.fits")]
[os.remove(s) for s in glob.glob("*_bias.fits")]
[os.remove(s) for s in glob.glob("*_flat.fits")]
[os.remove(s) for s in glob.glob("*_mosaic.fits")]
os.chdir("inputs/")
print("\n\n Writing processed files for tests into:\n"
" {:s}\n\n".format(os.getcwd()))
_ = p.writeOutputs()
os.chdir("../")
os.chdir(cwd)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from recipe_system.reduction.coreReduce import Reduce
from gempy.utils import logutils
from astrodata.testing import get_associated_calibrations
associated_calibrations = {
"S20190808S0048.fits": 'S20190808S0167.fits',
"S20190808S0049.fits": 'S20190808S0168.fits',
"S20190808S0053.fits": 'S20190808S0169.fits',
"N20180106S0025.fits": 'N20180115S0264.fits',
"N20180106S0026.fits": 'N20180115S0264.fits',
"N20180106S0028.fits": 'N20180115S0264.fits',
"N20180106S0029.fits": 'N20180115S0264.fits',
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("./inputs", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for fname, arc_fname in associated_calibrations.items():
sci_path = download_from_archive(fname)
arc_path = download_from_archive(arc_fname)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
if os.path.exists(
arc_fname.replace('.fits', '_distortionDetermined.fits')):
arc = astrodata.open(
arc_fname.replace('.fits', '_distortionDetermined.fits'))
else:
p = GMOSLongslit([astrodata.open(arc_path)])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.makeIRAFCompatible()
p.determineWavelengthSolution()
p.determineDistortion()
arc = p.writeOutputs().pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
os.chdir("inputs/")
_ = p.writeOutputs().pop()
os.chdir("../")