def create_quartz_inputs():
"""
Creates input data for tests using pre-processed twilight flat data and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
associated_calibrations = {
"N20200715S0059.fits": {
"quartz": ["N20200715S0059.fits"],
},
"S20130601S0121.fits": {
"quartz": ["S20130601S0121.fits"],
},
"S20190204S0081.fits": {
"quartz": ["S20190204S0081.fits"],
},
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/longslit/test_slit_illum_correct/inputs"
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
cwd = os.getcwd()
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Download raw files')
quartz_path = [download_from_archive(f) for f in cals['quartz']]
quartz_ad = astrodata.open(quartz_path[0])
data_label = quartz_ad.data_label()
print('Reducing quartz lamp:')
logutils.config(file_name='log_quartz_{}.txt'.format(data_label))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
p = GMOSLongslit([astrodata.open(f) for f in quartz_path])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
# p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.stackFrames()
# Write non-mosaicked data
p.writeOutputs(suffix="_quartz", strip=True)
os.chdir(cwd)
def process_object(filename, arc, suffix="linearized"):
"""
Helper recipe to reduce the object file.
Returns
-------
AstroData
Processed arc.
"""
from astrodata.testing import download_from_archive
from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit
processed_filename, ext = os.path.splitext(filename)
processed_filename += "_{:s}{:s}".format(suffix, ext)
if os.path.exists(processed_filename):
ad = astrodata.open(processed_filename)
else:
if os.path.exists(filename):
ad = astrodata.open(filename)
else:
ad = astrodata.open(
download_from_archive(filename,
path='',
env_var='DRAGONS_TEST'))
p = GMOSLongslit([ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
# p.biasCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
# p.flatCorrect()
# p.applyQECorrection()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra() # TODO: needed?
p.calculateSensitivity()
ad = p.streams['main'][0]
ad.write(overwrite=True)
return ad
def reduce(filename):
_p = GMOSLongslit([astrodata.open(filename)])
_p.viewer = geminidr.dormantViewer(_p, None)
_p.prepare()
_p.addDQ(static_bpm=None)
_p.addVAR(read_noise=True)
_p.overscanCorrect()
_p.ADUToElectrons()
_p.addVAR(poisson_noise=True)
_p.mosaicDetectors()
_p.makeIRAFCompatible()
_p.determineWavelengthSolution(suffix="_arc")
return _p
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from geminidr.gmos.tests.spect import CREATED_INPUTS_PATH_FOR_TESTS
module_name, _ = os.path.splitext(os.path.basename(__file__))
path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename in datasets:
print('Downloading files...')
basename = filename.split("_")[0] + ".fits"
sci_path = download_from_archive(basename)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.makeIRAFCompatible()
os.chdir("inputs/")
processed_ad = p.writeOutputs().pop()
os.chdir("..")
print("Wrote pre-processed file to:\n"
" {:s}".format(processed_ad.filename))
def process_arc(filename, suffix="distortionDetermined"):
"""
Helper recipe to reduce the arc file.
Returns
-------
AstroData
Processed arc.
"""
from astrodata.testing import download_from_archive
from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit
processed_filename, ext = os.path.splitext(filename)
processed_filename += "_{:s}{:s}".format(suffix, ext)
if os.path.exists(processed_filename):
ad = astrodata.open(processed_filename)
else:
if os.path.exists(filename):
ad = astrodata.open(filename)
else:
ad = astrodata.open(
download_from_archive(filename,
path='',
env_var='DRAGONS_TEST'))
p = GMOSLongslit([ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
# p.biasCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.makeIRAFCompatible()
p.determineWavelengthSolution()
p.determineDistortion()
ad = p.streams['main'][0]
ad.write(overwrite=True)
return ad
def create_twilight_inputs():
"""
Creates input data for tests using pre-processed twilight flat data and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
associated_calibrations = {
"S20190204S0006.fits": {
"bias": [
"S20190203S0110.fits", "S20190203S0109.fits",
"S20190203S0108.fits", "S20190203S0107.fits",
"S20190203S0106.fits"
],
"twilight": ["S20190204S0006.fits"],
},
"N20190103S0462.fits": {
"bias": [
"N20190102S0531.fits", "N20190102S0530.fits",
"N20190102S0529.fits", "N20190102S0528.fits",
"N20190102S0527.fits"
],
"twilight": ["N20190103S0462.fits", "N20190103S0463.fits"],
},
"N20190327S0056.fits": {
"bias": [
"N20190327S0098.fits", "N20190327S0099.fits",
"N20190327S0100.fits", "N20190327S0101.fits",
"N20190327S0102.fits"
],
"twilight": ["N20190327S0056.fits"],
},
"S20130602S0005.fits": {
"bias": [
"S20130601S0161.fits",
"S20130601S0160.fits",
"S20130601S0159.fits",
"S20130601S0158.fits",
"S20130601S0157.fits",
],
"twilight": ["S20130602S0005.fits"],
},
"N20190602S0306.fits": {
"bias": [
"N20190601S0648.fits",
"N20190601S0647.fits",
"N20190601S0646.fits",
"N20190601S0645.fits",
"N20190601S0644.fits",
],
"twilight": ["N20190602S0306.fits"],
}
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/longslit/test_slit_illumination_correct/inputs"
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
cwd = os.getcwd()
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Download raw files')
bias_path = [download_from_archive(f) for f in cals['bias']]
twilight_path = [download_from_archive(f) for f in cals['twilight']]
twilight_ad = astrodata.open(twilight_path[0])
data_label = twilight_ad.data_label()
print('Reducing BIAS for {:s}'.format(data_label))
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_path)
bias_reduce.runr()
bias_master = bias_reduce.output_filenames.pop()
del bias_reduce
print('Reducing twilight flat:')
logutils.config(file_name='log_twilight_{}.txt'.format(data_label))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
p = GMOSLongslit([astrodata.open(f) for f in twilight_path])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.stackFrames()
# Write non-mosaicked data
twilight = p.writeOutputs(suffix="_twilight", strip=True)[0]
# Write mosaicked data
p = GMOSLongslit([twilight])
p.makeSlitIllum()
p.writeOutputs()
os.chdir(cwd)
return
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from recipe_system.reduction.coreReduce import Reduce
from gempy.utils import logutils
associated_calibrations = {
"S20190808S0048.fits": {
"arcs": ["S20190808S0167.fits"]
},
"S20190808S0049.fits": {
"arcs": ["S20190808S0168.fits"]
},
# "S20190808S0052.fits": {"arcs": ["S20190808S0165.fits"]}, # Can't find aperture
"S20190808S0053.fits": {
"arcs": ["S20190808S0169.fits"]
},
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("./inputs", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
arc_path = [download_from_archive(f) for f in cals['arcs']]
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend(arc_path)
arc_reduce.runr()
arc = arc_reduce.output_filenames.pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
os.chdir("inputs/")
_ = p.writeOutputs().pop()
os.chdir("../")
def create_inputs():
"""
Create inputs for `test_plot_spectra_for_qa_single_frame`.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import glob
import os
from geminidr.gmos.primitives_gmos_longslit import GMOSLongslit
from gempy.utils import logutils
from recipe_system.reduction.coreReduce import Reduce
from recipe_system.utils.reduce_utils import normalize_ucals
cwd = os.getcwd()
path = f"./dragons_test_inputs/geminidr/core/{__file__.split('.')[0]}/"
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
for raw_list, bias_list, quartz_list, arc_list in single_aperture_data:
if all([
os.path.exists(f"inputs/{s.split('.')[0]}_extracted.fits")
for s in raw_list
]):
print("Skipping already created input.")
continue
raw_paths = [download_from_archive(f) for f in raw_list]
bias_paths = [download_from_archive(f) for f in bias_list]
quartz_paths = [download_from_archive(f) for f in quartz_list]
arc_paths = [download_from_archive(f) for f in arc_list]
cals = []
raw_ads = [astrodata.open(p) for p in raw_paths]
data_label = raw_ads[0].data_label()
print('Current working directory:\n {:s}'.format(os.getcwd()))
if len(bias_paths):
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(bias_paths)
r.runr()
master_bias = r.output_filenames.pop()
cals.append(f"processed_bias:{master_bias}")
del r
else:
master_bias = None
if len(quartz_paths):
logutils.config(file_name='log_quartz_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(quartz_paths)
r.ucals = normalize_ucals(r.files, cals)
r.runr()
master_quartz = r.output_filenames.pop()
cals.append(f"processed_flat:{master_quartz}")
del r
else:
master_quartz = None
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
r = Reduce()
r.files.extend(arc_paths)
r.ucals = normalize_ucals(r.files, cals)
r.runr()
master_arc = r.output_filenames.pop()
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit(raw_ads)
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(do_bias=master_bias is not None, bias=master_bias)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.flatCorrect(do_flat=master_quartz is not None, flat=master_quartz)
p.QECorrect(arc=master_arc)
p.distortionCorrect(arc=master_arc)
p.findSourceApertures(max_apertures=3)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra()
[os.remove(s) for s in glob.glob("*_arc.fits")]
[os.remove(s) for s in glob.glob("*_bias.fits")]
[os.remove(s) for s in glob.glob("*_flat.fits")]
[os.remove(s) for s in glob.glob("*_mosaic.fits")]
os.chdir("inputs/")
print("\n\n Writing processed files for tests into:\n"
" {:s}\n\n".format(os.getcwd()))
_ = p.writeOutputs()
os.chdir("../")
os.chdir(cwd)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from recipe_system.reduction.coreReduce import Reduce
from gempy.utils import logutils
from astrodata.testing import get_associated_calibrations
associated_calibrations = {
"S20190808S0048.fits": 'S20190808S0167.fits',
"S20190808S0049.fits": 'S20190808S0168.fits',
"S20190808S0053.fits": 'S20190808S0169.fits',
"N20180106S0025.fits": 'N20180115S0264.fits',
"N20180106S0026.fits": 'N20180115S0264.fits',
"N20180106S0028.fits": 'N20180115S0264.fits',
"N20180106S0029.fits": 'N20180115S0264.fits',
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("./inputs", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for fname, arc_fname in associated_calibrations.items():
sci_path = download_from_archive(fname)
arc_path = download_from_archive(arc_fname)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
if os.path.exists(
arc_fname.replace('.fits', '_distortionDetermined.fits')):
arc = astrodata.open(
arc_fname.replace('.fits', '_distortionDetermined.fits'))
else:
p = GMOSLongslit([astrodata.open(arc_path)])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.makeIRAFCompatible()
p.determineWavelengthSolution()
p.determineDistortion()
arc = p.writeOutputs().pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
os.chdir("inputs/")
_ = p.writeOutputs().pop()
os.chdir("../")