def test_regression_on_extract_1d_spectra(ad, ref_ad_factory,
change_working_dir):
"""
Regression test for the :func:`~geminidr.gmos.GMOSSpect.extract1DSpectra`
primitive.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(
file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.extract1DSpectra(method="standard", width=None, grow=10)
extracted_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(extracted_ad.filename)
for ext, ref_ext in zip(extracted_ad, ref_ad):
assert ext.data.ndim == 1
np.testing.assert_allclose(ext.data, ref_ext.data, atol=1e-3)
def test_regression_for_determine_distortion_using_models_coefficients(
ad, change_working_dir, ref_ad_factory, request):
"""
Runs the `determineDistortion` primitive on a preprocessed data and compare
its model with the one in the reference file.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
request : pytest.fixture
PyTest built-in containing command line options.
"""
with change_working_dir():
logutils.config(file_name='log_model_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(distortion_determined_ad.filename)
for ext, ext_ref in zip(distortion_determined_ad, ref_ad):
c = np.ma.masked_invalid(ext.FITCOORD["coefficients"])
c_ref = np.ma.masked_invalid(ext_ref.FITCOORD["coefficients"])
np.testing.assert_allclose(c, c_ref, atol=2)
if request.config.getoption("--do-plots"):
do_plots(distortion_determined_ad, ref_ad)
def test_regression_on_flux_calibration(ad, ref_ad_factory,
change_working_dir):
"""
Regression test for the :func:`~geminidr.gmos.GMOSSpect.fluxCalibrate`
primitive.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(
file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.fluxCalibrate(standard=ad)
flux_calibrated_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(flux_calibrated_ad.filename)
for flux_cal_ext, ref_ext in zip(flux_calibrated_ad, ref_ad):
np.testing.assert_allclose(flux_cal_ext.data, ref_ext.data, atol=1e-4)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
fnames = ["S20190808S0048.fits"]
path = pathlib.Path('dragons_test_inputs')
path = path / "geminidr" / "gmos" / "spect" / "test_cosmics" / "inputs"
path.mkdir(exist_ok=True, parents=True)
os.chdir(path)
print('Current working directory:\n {!s}'.format(path.cwd()))
for fname in fnames:
sci_ad = astrodata.open(download_from_archive(fname))
data_label = sci_ad.data_label()
print('===== Reducing pre-processed data =====')
logutils.config(file_name=f'log_{data_label}.txt')
p = GMOSSpect([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.writeOutputs()
p.mosaicDetectors()
p.writeOutputs()
def create_master_bias_for_tests():
"""
Creates input bias data for tests.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
root_path = os.path.join("./dragons_test_inputs/")
module_path = f"geminidr/gmos/recipes/ql/{__file__.split('.')[0]}/"
path = os.path.join(root_path, module_path, "inputs/")
os.makedirs(path, exist_ok=True)
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
associated_biases = {
"S20180707S0043.fits": [
"S20180707S0187.fits", "S20180707S0188.fits",
"S20180707S0189.fits", "S20180707S0190.fits", "S20180707S0191.fits"
],
"S20190502S0096.fits": [
"S20190502S0221.fits", "S20190502S0222.fits",
"S20190502S0223.fits", "S20190502S0224.fits", "S20190502S0225.fits"
],
"S20200122S0020.fits": [
"S20200121S0170.fits", "S20200121S0171.fits",
"S20200121S0172.fits", "S20200121S0173.fits", "S20200121S0174.fits"
],
"N20200101S0055.fits": [
"N20200101S0240.fits", "N20200101S0241.fits",
"N20200101S0242.fits", "N20200101S0243.fits", "N20200101S0244.fits"
],
"S20180410S0120.fits": [
"S20180410S0132.fits", "S20180410S0133.fits",
"S20180410S0134.fits", "S20180410S0135.fits", "S20180410S0136.fits"
],
"S20190410S0053.fits": [
"S20190410S0297.fits", "S20190410S0298.fits",
"S20190410S0299.fits", "S20190410S0300.fits", "S20190410S0301.fits"
],
}
for filename, bias_files in associated_biases.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
bias_paths = [download_from_archive(f) for f in bias_files]
print('Reducing BIAS for {:s}'.format(data_label))
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_paths)
bias_reduce.runr()
shutil.rmtree("calibrations/")
def test_slitdark_in_calservice(self, get_or_create_tmpdir, do_slit_dark):
"""
Check that:
- A bias slit calibrator exists in the local calibrations dir;
- It can be retrieved using a getProcessedSlitBias call.
"""
# Ensure the slit dark reduction has been done
_, _, _ = do_slit_dark
_, cal_service = get_or_create_tmpdir
# import pdb; pdb.set_trace()
assert len(glob.glob(os.path.join(
os.getcwd(), 'calibrations', 'processed_dark', '*dark*slit*.fits'
))) == 1, "Couldn't find the stored slit bias in the calibrations " \
"system OR found multiples\n " \
"(calibration ls: {})\n" \
"(caldb contents: {})".format(
glob.glob(os.path.join(os.getcwd(), 'calibrations',
'processed_dark', '*')),
[_ for _ in cal_service.list_files()],
)
# Do the master bias generation
reduce = Reduce()
reduce.drpkg = 'ghostdr'
# Use one of the 'dark slit' files to try and retrieve the slit bias
reduce.files = glob.glob(
os.path.join(os.getcwd(), 'flat95*MEF_2x2_slit.fits'))
reduce.mode = [
'test',
]
reduce.recipename = 'recipeRetrieveSlitDarkTest'
# reduce.mode = ['sq', ]
reduce.logfile = os.path.join(os.getcwd(),
'reduce_slitdark_retrieve.log')
reduce.logmode = 'quiet'
reduce.suffix = '_testSlitDarkRetrieve'
# FIXME cal_service will hopefully find the calibration itself later
# reduce.ucals = normalize_ucals(reduce.files, [
# 'processed_dark:{}'.format(
# glob.glob(os.path.join(
# 'calibrations',
# 'processed_dark',
# '*slit*dark*.fits'))[0]),
# ])
logutils.config(file_name=reduce.logfile, mode=reduce.logmode)
try:
reduce.runr()
except IOError as e:
assert 0, 'Calibration system could not locate the slit bias ' \
'frame ({})'.format(e.message)
finally:
# Teardown code
for _ in glob.glob(
os.path.join(os.getcwd(),
'*{}.fits'.format(reduce.suffix)), ):
os.remove(_)
def test_qe_correct_is_locally_continuous(ad, arc_ad, change_working_dir):
if ad.filename == 'S20180919S0139_flatCorrected.fits':
pytest.xfail('FIXME: this test fails following changes on the QE '
'curves. Needs more investigation.')
with change_working_dir():
logutils.config(
file_name='log_test_continuity{}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.QECorrect(arc=arc_ad)
# Need these extra steps to extract and analyse the data
p.distortionCorrect(arc=arc_ad)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra()
processed_ad = p.writeOutputs().pop()
for ext in processed_ad:
assert not np.any(np.isnan(ext.data))
assert not np.any(np.isinf(ext.data))
basename = processed_ad.filename.replace('_linearized', '')
kwargs = gap_local_kw[basename] if basename in gap_local_kw.keys() else {}
gap = MeasureGapSizeLocallyWithSpline(processed_ad, **kwargs)
assert abs(gap.measure_gaps(0) < 0.05)
assert abs(gap.measure_gaps(1) < 0.05)
def test_fitcoord_table_and_gwcs_match(ad, change_working_dir):
"""
Runs determineDistortion and checks that the model in the gWCS is the same
as the model in the FITCOORD table. The FITCOORD table is never used by
DRAGONS.
Parameters
----------
ad: pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
"""
with change_working_dir():
logutils.config(file_name='log_match_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
model = distortion_determined_ad[0].wcs.get_transform(
"pixels", "distortion_corrected")
fitcoord = distortion_determined_ad[0].FITCOORD
fitcoord_model = am.table_to_model(fitcoord[0])
fitcoord_inv = am.table_to_model(fitcoord[1])
np.testing.assert_allclose(model[1].parameters, fitcoord_model.parameters)
np.testing.assert_allclose(model.inverse[1].parameters,
fitcoord_inv.parameters)
def test_regression_for_determine_distortion_using_wcs(ad, change_working_dir,
ref_ad_factory):
"""
Runs the `determineDistortion` primitive on a preprocessed data and compare
its model with the one in the reference file. The distortion model needs to
be reconstructed because different coefficients might return same results.
Parameters
----------
ad : pytest.fixture (AstroData)
Fixture that reads the filename and loads as an AstroData object.
change_working_dir : pytest.fixture
Fixture that changes the working directory
(see :mod:`astrodata.testing`).
reference_ad : pytest.fixture
Fixture that contains a function used to load the reference AstroData
object (see :mod:`recipe_system.testing`).
"""
with change_working_dir():
logutils.config(
file_name='log_fitcoord_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineDistortion(**fixed_parameters_for_determine_distortion)
distortion_determined_ad = p.writeOutputs().pop()
ref_ad = ref_ad_factory(distortion_determined_ad.filename)
model = distortion_determined_ad[0].wcs.get_transform(
"pixels", "distortion_corrected")[1]
ref_model = ref_ad[0].wcs.get_transform("pixels",
"distortion_corrected")[1]
X, Y = np.mgrid[:ad[0].shape[0], :ad[0].shape[1]]
np.testing.assert_allclose(model(X, Y), ref_model(X, Y), atol=1)
def setup_class(cls):
"""Run once at the beginning."""
if os.path.exists(logfilename):
os.remove(logfilename)
log = logutils.get_logger(__name__)
log.root.handlers = []
logutils.config(mode='standard', file_name=logfilename)
def setup_class(cls):
"""Run once at the beginning."""
if os.path.exists(logfilename):
os.remove(logfilename)
log = logutils.get_logger(__name__)
log.root.handlers = []
logutils.config(mode='standard', console_lvl='stdinfo',
file_name=logfilename)
def test_reduce_image_GN_EEV_2x2_g(path_to_inputs):
logutils.config(file_name='gmos_test_reduce_image_GN_EEV_2x2_g.log')
calib_files = []
raw_subdir = 'GMOS/GN-2002A-Q-89'
all_files = sorted(glob.glob(
os.path.join(path_to_inputs, raw_subdir, '*.fits')))
assert len(all_files) > 1
list_of_bias = dataselect.select_data(
all_files,
['BIAS'],
[]
)
list_of_flats = dataselect.select_data(
all_files,
['IMAGE', 'FLAT'],
[],
dataselect.expr_parser('filter_name=="g"')
)
# These old data don't have an OBSCLASS keyword:
list_of_science_files = dataselect.select_data(
all_files, [],
['CAL'],
dataselect.expr_parser(
'object=="PerseusField4" and filter_name=="g"'
)
)
reduce_bias = Reduce()
assert len(reduce_bias.files) == 0
reduce_bias.files.extend(list_of_bias)
assert len(reduce_bias.files) == len(list_of_bias)
reduce_bias.runr()
calib_files.append(
'processed_bias:{}'.format(reduce_bias.output_filenames[0])
)
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_flats)
reduce_flats.ucals = normalize_ucals(reduce_flats.files, calib_files)
reduce_flats.runr()
calib_files.append(
'processed_flat:{}'.format(reduce_flats.output_filenames[0])
)
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.ucals = normalize_ucals(reduce_target.files, calib_files)
reduce_target.runr()
def create_quartz_inputs():
"""
Creates input data for tests using pre-processed twilight flat data and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
associated_calibrations = {
"N20200715S0059.fits": {
"quartz": ["N20200715S0059.fits"],
},
"S20130601S0121.fits": {
"quartz": ["S20130601S0121.fits"],
},
"S20190204S0081.fits": {
"quartz": ["S20190204S0081.fits"],
},
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/longslit/test_slit_illum_correct/inputs"
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
cwd = os.getcwd()
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Download raw files')
quartz_path = [download_from_archive(f) for f in cals['quartz']]
quartz_ad = astrodata.open(quartz_path[0])
data_label = quartz_ad.data_label()
print('Reducing quartz lamp:')
logutils.config(file_name='log_quartz_{}.txt'.format(data_label))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
p = GMOSLongslit([astrodata.open(f) for f in quartz_path])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
# p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.stackFrames()
# Write non-mosaicked data
p.writeOutputs(suffix="_quartz", strip=True)
os.chdir(cwd)
def reduce(file_list,
label,
calib_files,
recipe_name=None,
save_to=None,
user_pars=None):
"""
Helper function used to prevent replication of code.
Parameters
----------
file_list : list
List of files that will be reduced.
label : str
Labed used on log files name.
calib_files : list
List of calibration files properly formatted for DRAGONS Reduce().
recipe_name : str, optional
Name of the recipe used to reduce the data.
save_to : str, optional
Stores the calibration files locally in a list.
user_pars : list, optional
List of user parameters
Returns
-------
str : Output reduced file.
list : An updated list of calibration files.
"""
objgraph = pytest.importorskip("objgraph")
logutils.get_logger().info("\n\n\n")
logutils.config(file_name=f"test_image_{label}.log")
r = Reduce()
r.files = file_list
r.ucals = normalize_ucals(r.files, calib_files)
r.uparms = user_pars
if recipe_name:
r.recipename = recipe_name
r.runr()
output_file = r.output_filenames[0]
if save_to:
calib_files.append("{}:{}".format(
save_to,
os.path.join("calibrations", save_to, r.output_filenames[0])))
[os.remove(f) for f in r.output_filenames]
# check that we are not leaking objects
assert len(objgraph.by_type('NDAstroData')) == 0
return output_file, calib_files
def _reduce_bias(datalabel, bias_fnames):
with output_path():
logutils.config(file_name='log_bias_{}.txt'.format(datalabel))
reduce = Reduce()
reduce.files.extend(bias_fnames)
reduce.runr()
master_bias = reduce.output_filenames.pop()
return master_bias
def _reduce_arc(dlabel, arc_fnames):
with output_path():
# Use config to prevent duplicated outputs when running Reduce via API
logutils.config(file_name='log_arc_{}.txt'.format(dlabel))
reduce = Reduce()
reduce.files.extend(arc_fnames)
reduce.runr()
master_arc = reduce.output_filenames.pop()
return master_arc
def _reduce_arc(dlabel, arc_fnames):
with change_working_dir():
print("Reducing ARCs in folder:\n {}".format(os.getcwd()))
# Use config to prevent duplicated outputs when running Reduce via API
logutils.config(file_name='log_arc_{}.txt'.format(dlabel))
reduce = Reduce()
reduce.files.extend(arc_fnames)
reduce.runr()
master_arc = reduce.output_filenames.pop()
return master_arc
def _reduce_bias(datalabel, bias_fnames):
with change_working_dir():
print("Reducing BIAS in folder:\n {}".format(os.getcwd()))
logutils.config(file_name='log_bias_{}.txt'.format(datalabel))
reduce = Reduce()
reduce.files.extend(bias_fnames)
reduce.runr()
master_bias = reduce.output_filenames.pop()
return master_bias
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename in datasets:
print('Downloading files...')
basename = filename.split("_")[0] + ".fits"
sci_path = download_from_archive(basename)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None, user_bpm=None, add_illum_mask=False)
p.addVAR(read_noise=True, poisson_noise=False)
p.overscanCorrect(function="spline",
high_reject=3.,
low_reject=3.,
nbiascontam=0,
niterate=2,
order=None)
p.ADUToElectrons()
p.addVAR(poisson_noise=True, read_noise=False)
p.mosaicDetectors()
p.makeIRAFCompatible()
p.determineDistortion(**fixed_test_parameters_for_determine_distortion)
os.chdir("inputs")
processed_ad = p.writeOutputs().pop()
os.chdir("../../")
print('Wrote pre-processed file to:\n'
' {:s}'.format(processed_ad.filename))
def test_can_run_reduce_bias(config):
"""
Make sure that the reduce_BIAS works for spectroscopic data.
"""
logutils.config(mode='quiet',
file_name=os.path.join(config.log_dir,
'reduce_GMOS_LS_bias.log'))
reduce = Reduce()
reduce.files.extend(config.biases)
reduce.upload = 'calibs'
reduce.runr()
def log():
if os.path.exists(logfilename):
os.remove(logfilename)
log = logutils.get_logger(__name__)
log.root.handlers = []
logutils.config(mode='standard', file_name=logfilename)
yield log
os.remove(logfilename)
def test_can_run_reduce_science(config):
"""
Make sure that the recipes_ARC_LS_SPECT works for spectroscopic data.
"""
logutils.config(mode='quiet',
file_name=os.path.join(config.log_dir,
'reduce_GMOS_LS_science.log'))
reduce = Reduce()
reduce.files.extend(config.science)
reduce.upload = 'calibs'
reduce.runr()
def do_overscan_subtract(self, get_or_create_tmpdir, request):
"""
Run overscan correction on the main data.
.. note::
Fixture.
"""
# Copy the raw data file into here
rawfilename = 'bias*{}*.fits'.format(request.param)
tmpsubdir, cal_service = get_or_create_tmpdir
# Make sure we're working inside the temp dir
# rawfiles = glob.glob(os.path.join(
# os.path.dirname(os.path.abspath(__file__)),
# 'testdata',
# rawfilename))
# shutil.copy(
# rawfiles[0],
# os.path.join(tmpsubdir.dirname, tmpsubdir.basename))
rawfile = glob.glob(
os.path.join(tmpsubdir.dirname, tmpsubdir.basename,
rawfilename))[0]
# Do the overscan subtraction
reduce = Reduce()
reduce.drpkg = 'ghostdr'
reduce.files = [
rawfile,
]
reduce.mode = [
'test',
]
reduce.recipename = 'recipeBiasRemoveOverscan'
reduce.logfile = os.path.join(tmpsubdir.dirname, tmpsubdir.basename,
'reduce_overscancorrect.log')
reduce.logmode = 'quiet'
reduce.suffix = '_testOverscanCorrect'
logutils.config(file_name=reduce.logfile, mode=reduce.logmode)
reduce.runr()
corrfilename = os.path.join(tmpsubdir.dirname, tmpsubdir.basename,
'*' + reduce.suffix + '.fits')
corrfilename = glob.glob(corrfilename)[0]
corrfile = os.path.join(tmpsubdir.dirname, tmpsubdir.basename,
corrfilename)
# Return filenames of raw, subtracted files
yield rawfile, corrfile
# Execute teardown code
for _ in glob.glob(
os.path.join(os.getcwd(), '*{}.fits'.format(reduce.suffix))):
os.remove(_)
def setup_log(path_to_outputs):
"""
Fixture that setups DRAGONS' logging system to avoid duplicated outputs.
Parameters
----------
path_to_outputs : fixture
Custom fixture defined in `astrodata.testing` containing the path to
the output folder.
"""
log_file = "{}.log".format(os.path.splitext(os.path.basename(__file__))[0])
log_file = os.path.join(path_to_outputs, log_file)
logutils.config(mode="standard", file_name=log_file)
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr,
caplog, change_working_dir,
ref_ad_factory):
"""
Make sure that the wavelength solution gives same results on different
runs.
"""
caplog.set_level(logging.INFO, logger="geminidr")
with change_working_dir():
logutils.config(
file_name='log_regress_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineWavelengthSolution(
order=order,
min_snr=min_snr,
fwidth=fwidth,
**determine_wavelength_solution_parameters)
wcalibrated_ad = p.writeOutputs().pop()
for record in caplog.records:
if record.levelname == "WARNING":
assert "No acceptable wavelength solution found" not in record.message
ref_ad = ref_ad_factory(wcalibrated_ad.filename)
table = wcalibrated_ad[0].WAVECAL
table_ref = ref_ad[0].WAVECAL
model = astromodels.dict_to_chebyshev(
dict(zip(table["name"], table["coefficients"])))
ref_model = astromodels.dict_to_chebyshev(
dict(zip(table_ref["name"], table_ref["coefficients"])))
x = np.arange(wcalibrated_ad[0].shape[1])
wavelength = model(x)
ref_wavelength = ref_model(x)
pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px
slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace(
'arcsec', ''))
slit_size_in_px = slit_size_in_arcsec / pixel_scale
dispersion = abs(
wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px
tolerance = 0.5 * (slit_size_in_px * dispersion)
np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance)
def test_reduce_image_GN_HAM_2x2_z(path_to_inputs):
objgraph = pytest.importorskip("objgraph")
logutils.config(file_name='gmos_test_reduce_image_GN_HAM_2x2_z.log')
calib_files = []
raw_subdir = 'GMOS/GN-2017B-LP-15'
all_files = sorted(glob.glob(
os.path.join(path_to_inputs, raw_subdir, '*.fits')))
assert len(all_files) > 1
list_of_bias = dataselect.select_data(all_files, ['BIAS'], [])
expr = dataselect.expr_parser('filter_name=="z"')
list_of_z_flats = dataselect.select_data(all_files, ['TWILIGHT'], [], expr)
expr = dataselect.expr_parser(
'observation_class=="science" and filter_name=="z"'
)
list_of_science = dataselect.select_data(all_files, [], ['CAL'], expr)
def reduce(filelist, saveto=None, label='', calib_files=None,
recipename=None):
red = Reduce()
assert len(red.files) == 0
red.files.extend(filelist)
assert len(red.files) == len(filelist)
if calib_files:
red.ucals = normalize_ucals(red.files, calib_files)
if recipename:
red.recipename = recipename
red.runr()
if saveto:
calib_files.append(f'{saveto}:{red.output_filenames[0]}')
# check that we are not leaking objects
assert len(objgraph.by_type('NDAstroData')) == 0
reduce(list_of_bias, saveto='processed_bias', label='bias',
calib_files=calib_files)
reduce(list_of_z_flats, saveto='processed_flat', label='flat',
calib_files=calib_files)
# If makeFringe is included in the science recipe, this can be omitted:
reduce(list_of_science, saveto='processed_fringe', label='fringe',
calib_files=calib_files, recipename='makeProcessedFringe')
reduce(list_of_science, label='science', calib_files=calib_files)
def test_slitbias_in_calservice(self, get_or_create_tmpdir):
"""
Check that:
- A bias slit calibrator exists in the local calibrations dir;
- It can be retrieved using a getProcessedSlitBias call.
"""
assert len(glob.glob(os.path.join(
os.getcwd(), 'calibrations', 'processed_bias', '*bias*slit*.fits'
))) == 1, "Couldn't find the stored slit bias in the calibrations " \
"system OR found multiples"
# Do the master bias generation
reduce = Reduce()
reduce.drpkg = 'ghostdr'
# Use one of the 'dark slit' files to try and retrieve the slit bias
reduce.files = [
os.path.join(os.getcwd(), 'dark95_1_MEF_2x2_slit.fits'),
]
reduce.mode = [
'test',
]
reduce.recipename = 'recipeRetrieveSlitBiasTest'
# reduce.mode = ['sq', ]
# reduce.recipename = 'makeProcessedBias'
reduce.logfile = os.path.join(os.getcwd(),
'reduce_slitbias_retrieve.log')
# FIXME cal_service will hopefully find the calibration itself later
reduce.ucals = normalize_ucals(reduce.files, [
'processed_bias:{}'.format(
glob.glob(
os.path.join('calibrations', 'processed_bias',
'*slit*bias*.fits'))[0]),
])
reduce.logmode = 'quiet'
reduce.suffix = '_testSlitBiasRetrieve'
logutils.config(file_name=reduce.logfile, mode=reduce.logmode)
try:
reduce.runr()
except IOError as e:
assert 0, 'Calibration system could not locate the slit bias ' \
'frame ({})'.format(e.message)
finally:
# Teardown code
for _ in glob.glob(
os.path.join(os.getcwd(),
'*{}.fits'.format(reduce.suffix)), ):
os.remove(_)
def test_regression_on_calculate_sensitivity(ad, change_working_dir, ref_ad_factory):
with change_working_dir():
logutils.config(file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_spect.GMOSSpect([ad])
p.calculateSensitivity(bandpass=5, order=6)
calc_sens_ad = p.writeOutputs().pop()
assert hasattr(calc_sens_ad[0], 'SENSFUNC')
ref_ad = ref_ad_factory(ad.filename)
for calc_sens_ext, ref_ext in zip(ad, ref_ad):
np.testing.assert_allclose(
calc_sens_ext.data, ref_ext.data, atol=1e-4)
def processed_slit_illum(change_working_dir, path_to_inputs, request):
"""
Returns the processed slit illumination function that will be analysed.
Parameters
----------
change_working_dir : pytest.fixture
Fixture that changes the working directory (see :mod:`astrodata.testing`).
path_to_inputs : pytest.fixture
Fixture defined in :mod:`astrodata.testing` with the path to the
pre-processed input file.
request : pytest.fixture
PyTest built-in fixture containing information about parent test.
Returns
-------
AstroData
Input spectrum processed up to right before the `applyQECorrection`.
"""
twi_filename = request.param
twi_path = download_from_archive(twi_filename)
twi_ad = astrodata.open(twi_path)
print(twi_ad.tags)
master_bias = os.path.join(
path_to_inputs, associated_calibrations[twi_filename])
assert os.path.exists(master_bias)
calibration_files = ['processed_bias:{}'.format(master_bias)]
with change_working_dir():
print("Reducing SLITILLUM in folder:\n {}".format(os.getcwd()))
logutils.config(
file_name='log_flat_{}.txt'.format(twi_ad.data_label()))
reduce = Reduce()
reduce.files.extend([twi_path])
reduce.mode = 'sq'
reduce.recipename = 'makeProcessedSlitIllum'
reduce.ucals = normalize_ucals(reduce.files, calibration_files)
reduce.runr()
_processed_twi_filename = reduce.output_filenames.pop()
_processed_twi = astrodata.open(_processed_twi_filename)
return _processed_twi
def test_regression_determine_wavelength_solution(ad, fwidth, order, min_snr,
caplog, change_working_dir,
ref_ad_factory, request):
"""
Make sure that the wavelength solution gives same results on different
runs.
"""
caplog.set_level(logging.INFO, logger="geminidr")
with change_working_dir():
logutils.config(
file_name='log_regress_{:s}.txt'.format(ad.data_label()))
p = GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.determineWavelengthSolution(
order=order,
min_snr=min_snr,
fwidth=fwidth,
**determine_wavelength_solution_parameters)
wcalibrated_ad = p.writeOutputs().pop()
for record in caplog.records:
if record.levelname == "WARNING":
assert "No acceptable wavelength solution found" not in record.message
ref_ad = ref_ad_factory(wcalibrated_ad.filename)
model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform,
"WAVE")
ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE")
x = np.arange(wcalibrated_ad[0].shape[1])
wavelength = model(x)
ref_wavelength = ref_model(x)
pixel_scale = wcalibrated_ad[0].pixel_scale() # arcsec / px
slit_size_in_arcsec = float(wcalibrated_ad[0].focal_plane_mask().replace(
'arcsec', ''))
slit_size_in_px = slit_size_in_arcsec / pixel_scale
dispersion = abs(
wcalibrated_ad[0].dispersion(asNanometers=True)) # nm / px
tolerance = 0.5 * (slit_size_in_px * dispersion)
np.testing.assert_allclose(wavelength, ref_wavelength, rtol=tolerance)
if request.config.getoption("--do-plots"):
do_plots(wcalibrated_ad)
def _reduce_flat(datalabel, flat_fname, master_bias):
with output_path():
logutils.config(file_name='log_flat_{}.txt'.format(datalabel))
calibration_files = ['processed_bias:{}'.format(master_bias)]
reduce = Reduce()
reduce.files.extend([flat_fname])
reduce.mode = 'ql'
reduce.ucals = normalize_ucals(reduce.files, calibration_files)
reduce.runr()
master_flat = reduce.output_filenames.pop()
master_flat_ad = astrodata.open(master_flat)
return master_flat_ad
def test_reduce_image(test_path, caldb):
logutils.config(file_name='f2_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(
os.path.join(test_path, 'F2/test_reduce/', '*.fits'))
assert len(all_files) > 1
darks_3s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==3'))
darks_20s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==20'))
darks_120s = dataselect.select_data(
all_files, ['F2', 'DARK', 'RAW'], [],
dataselect.expr_parser('exposure_time==120'))
flats = dataselect.select_data(
all_files, ['F2', 'FLAT', 'RAW'], [],
dataselect.expr_parser('filter_name=="Y"'))
science = dataselect.select_data(
all_files, ['F2', 'RAW'], ['CAL'],
dataselect.expr_parser('filter_name=="Y"'))
for darks in [darks_3s, darks_20s, darks_120s]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(flats)
reduce_bpm.files.extend(darks_3s)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(science)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)
def test_reduce_image(test_path, caldb):
logutils.config(file_name='gsaoi_test_reduce_image.log')
caldb.init(wipe=True)
all_files = glob.glob(
os.path.join(test_path, 'GSAOI/test_reduce/', '*.fits'))
assert len(all_files) > 1
list_of_darks = dataselect.select_data(
all_files, ['DARK'], [])
list_of_kshort_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="Kshort"'))
list_of_h_flats = dataselect.select_data(
all_files, ['FLAT'], [],
dataselect.expr_parser('filter_name=="H"'))
list_of_std_LHS_2026 = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('object=="LHS 2026"'))
list_of_std_cskd8 = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('object=="cskd-8"'))
list_of_science_files = dataselect.select_data(
all_files, [], [],
dataselect.expr_parser('observation_class=="science" and exposure_time==60.'))
for darks in [list_of_darks]:
reduce_darks = Reduce()
assert len(reduce_darks.files) == 0
reduce_darks.files.extend(darks)
assert len(reduce_darks.files) == len(darks)
reduce_darks.runr()
caldb.add_cal(reduce_darks.output_filenames[0])
reduce_bpm = Reduce()
reduce_bpm.files.extend(list_of_h_flats)
reduce_bpm.files.extend(list_of_darks)
reduce_bpm.recipename = 'makeProcessedBPM'
reduce_bpm.runr()
bpm_filename = reduce_bpm.output_filenames[0]
reduce_flats = Reduce()
reduce_flats.files.extend(list_of_kshort_flats)
reduce_flats.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_flats.runr()
caldb.add_cal(reduce_flats.output_filenames[0])
reduce_target = Reduce()
reduce_target.files.extend(list_of_science_files)
reduce_target.uparms = [('addDQ:user_bpm', bpm_filename)]
reduce_target.runr()
for f in caldb.list_files():
print(f)
#!/usr/bin/env python
import glob
import pytest
import os
from gempy.adlibrary import dataselect
from recipe_system.reduction.coreReduce import Reduce
from gempy.utils import logutils
logutils.config(file_name='dummy.log')
@pytest.fixture
def test_path():
try:
path = os.environ['TEST_PATH']
except KeyError:
pytest.skip("Could not find environment variable: $TEST_PATH")
if not os.path.exists(path):
pytest.skip("Could not find path stored in $TEST_PATH: {}".format(path))
return path
def test_expr_parser_can_parse_for_filter_name():
"""
Does exactly what expr_parser does when the requested descriptor is
