def test_adjust_wcs(files, path_to_inputs):
"""
We take preprocessed files with a single aperture and run them through
adjustWCSToReference. After this, the celestial coordinates of the aperture
should be similar in all images.
In order to make this a real test, we edit the gWCS objects of the inputs
so they're wrong (except for the first one, which is the reference)
"""
adinputs = [astrodata.open(os.path.join(path_to_inputs, f)) for f in files]
# Hack the WCS of all but the first input so they're wrong
for ad in adinputs[1:]:
ad[0].wcs.pipeline[0][1]['crpix2'].offset = 600
p = primitives_gmos_longslit.GMOSLongslit(adinputs)
p.adjustWCSToReference()
# Return the (RA, dec) as a SkyCoord at the location of each aperture
# Then confirm that the sky coordinates are all similar
skycoords = [ad[0].wcs(0, ad[0].APERTURE['c0'][0], with_units=True)[1]
for ad in adinputs]
c0 = skycoords[0]
for c in skycoords[1:]:
assert c0.separation(c).arcsecond < 0.05
# Todo: Implement recipe to create input files
def test_qe_correct_is_locally_continuous(ad, arc_ad, change_working_dir):
if ad.filename == 'S20180919S0139_flatCorrected.fits':
pytest.xfail('FIXME: this test fails following changes on the QE '
'curves. Needs more investigation.')
with change_working_dir():
logutils.config(
file_name='log_test_continuity{}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.QECorrect(arc=arc_ad)
# Need these extra steps to extract and analyse the data
p.distortionCorrect(arc=arc_ad)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra()
processed_ad = p.writeOutputs().pop()
for ext in processed_ad:
assert not np.any(np.isnan(ext.data))
assert not np.any(np.isinf(ext.data))
basename = processed_ad.filename.replace('_linearized', '')
kwargs = gap_local_kw[basename] if basename in gap_local_kw.keys() else {}
gap = MeasureGapSizeLocallyWithSpline(processed_ad, **kwargs)
assert abs(gap.measure_gaps(0) < 0.05)
assert abs(gap.measure_gaps(1) < 0.05)
def processed_ad(request, qe_corrected_ad, get_master_arc, output_path):
"""
Process the QE corrected data so we can measure the jump size at each gap.
Parameters
----------
request
qe_corrected_ad : AstroData
QE corrected astrodata (as it says).
get_master_arc : pytest.fixture
Fixture that reads the master flat either from the permanent input folder
or from the temporary cache folder.
Returns
-------
AstroData
Processed 1D bias + flat + qe + distortion + sky corrected, extracted,
and linearized spectrum.
"""
pre_process = request.config.getoption("--force-preprocess-data")
master_arc = get_master_arc(qe_corrected_ad, pre_process)
with output_path():
p = primitives_gmos_longslit.GMOSLongslit([qe_corrected_ad])
p.distortionCorrect(arc=master_arc)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.linearizeSpectra()
processed_ad = p.writeOutputs().pop()
return processed_ad
def qe_corrected_ad(request, get_input_ad, get_master_arc, output_path):
"""
Returns the processed spectrum right after running `applyQECorrection`.
Parameters
----------
request : pytest.fixture
Fixture that contains information this fixture's parent.
get_input_ad : pytest.fixture
Fixture that reads the input data or cache/process it in a temporary
folder.
get_master_arc : pytest.fixture
Fixture that reads the master flat either from the permanent input folder
or from the temporary cache folder.
output_path : contextmanager
Enable easy change to temporary folder when reducing data.
Returns
-------
AstroData
QE Corrected astrodata.
"""
filename = request.param
pre_process = request.config.getoption("--force-preprocess-data")
input_ad = get_input_ad(filename, pre_process)
master_arc = get_master_arc(input_ad, pre_process)
with output_path():
p = primitives_gmos_longslit.GMOSLongslit([input_ad])
p.applyQECorrection(arc=master_arc)
qe_corrected_ad = p.writeOutputs().pop()
return qe_corrected_ad
def test_full_frame_distortion_works_on_smaller_region(ad, change_working_dir):
"""
Takes a full-frame arc and self-distortion-corrects it. It then fakes
sub-regions of this and corrects those using the full-frame distortion to
confirm that the result is the same as the appropriate region of the
distortion-corrected full-frame image. There's no need to do this more
than once for a given binning, so we loop within the function, keeping
track of binnings we've already processed.
"""
NSUB = 4 # we're going to take combos of horizontal quadrants
completed_binnings = []
xbin, ybin = ad.detector_x_bin(), ad.detector_y_bin()
if ad.detector_roi_setting() != "Full Fame" or (xbin, ybin) in completed_binnings:
return
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.viewer.viewer_name = None
ad_out = p.distortionCorrect([deepcopy(ad)], arc=ad, order=1)[0]
for start in range(NSUB):
for end in range(start + 1, NSUB + 1):
ad_copy = deepcopy(ad)
y1b = start * ad[0].shape[0] // NSUB
y2b = end * ad[0].shape[0] // NSUB
y1, y2 = y1b * ybin, y2b * ybin # unbinned pixels
# Fake the section header keywords and set the SCI and DQ
# to the appropriate sub-region
for ext in ad_copy:
arrsec = ext.array_section()
detsec = ext.detector_section()
ext.hdr['CCDSEC'] = '[{}:{},{}:{}]'.format(arrsec.x1 + 1,
arrsec.x2, y1 + 1, y2)
ext.hdr['DETSEC'] = '[{}:{},{}:{}]'.format(detsec.x1 + 1,
detsec.x2, y1 + 1, y2)
ext.data = ext.data[y1b:y2b]
ext.mask = ext.mask[y1b:y2b]
ext.hdr['DATASEC'] = '[1:{},1:{}]'.format(ext.shape[1], y2b - y1b)
ad2 = p.distortionCorrect(
[ad_copy], arc=p.streams['mosaic'][0], order=1)[0]
# It's GMOS LS so the offset between this AD and the full-frame
# will be the same as the DETSEC offset, but the width may be
# smaller so we need to shuffle the smaller image within the
# larger one to look for a match
ny, nx = ad2[0].shape
xsizediff = ad_out[0].shape[1] - nx
for xoffset in range(xsizediff + 1):
np.testing.assert_allclose(
np.ma.masked_array(ad2[0].data, mask=ad2[0].mask),
ad_out[0].data[y1b:y1b + ny, xoffset:xoffset + nx],
atol=0.01,
err_msg="Problem with {} {}:{}".format(ad.filename, start, end))
completed_binnings.append((xbin, ybin))
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename in datasets:
print('Downloading files...')
basename = filename.split("_")[0] + ".fits"
sci_path = download_from_archive(basename)
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None, user_bpm=None, add_illum_mask=False)
p.addVAR(read_noise=True, poisson_noise=False)
p.overscanCorrect(function="spline",
high_reject=3.,
low_reject=3.,
nbiascontam=0,
niterate=2,
order=None)
p.ADUToElectrons()
p.addVAR(poisson_noise=True, read_noise=False)
p.mosaicDetectors()
p.makeIRAFCompatible()
p.determineDistortion(**fixed_test_parameters_for_determine_distortion)
os.chdir("inputs")
processed_ad = p.writeOutputs().pop()
os.chdir("../../")
print('Wrote pre-processed file to:\n'
' {:s}'.format(processed_ad.filename))
def test_create_slit_illumination_with_multi_extension_data(
ad, change_working_dir, request):
"""
Test that can run `makeSlitIllum` in multi-extension data.
"""
plot = request.config.getoption("--do-plots")
with change_working_dir():
cwd = os.getcwd()
print("Running tests inside folder:\n {}".format(cwd))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.makeSlitIllum(bins=25, border=10, debug_plot=plot)
slit_illum_ad = p.writeOutputs()[0]
for ext, slit_ext in zip(ad, slit_illum_ad):
assert ext.shape == slit_ext.shape
# Create output data
data_o = (np.ma.masked_array(ext.data, mask=ext.mask) /
np.ma.masked_array(slit_ext.data, mask=slit_ext.mask))
# Bin columns
fitter = fitting.LinearLSQFitter()
model = models.Polynomial1D(degree=2)
nbins = 10
rows = np.arange(data_o.shape[0])
for i in range(nbins):
col_start = i * data_o.shape[1] // nbins
col_end = (i + 1) * data_o.shape[1] // nbins
cols = np.ma.mean(data_o[:, col_start:col_end], axis=1)
fitted_model = fitter(model, rows, cols)
# Check column is linear
np.testing.assert_allclose(fitted_model.c2.value, 0, atol=0.01)
# Check if slope is (almost) horizontal (< 2.0 deg)
assert np.abs(
np.rad2deg(
np.arctan(fitted_model.c1.value /
(rows.size // 2)))) < 1.5
if plot:
os.makedirs(PLOT_PATH, exist_ok=True)
print("Renaming plots to ",
os.path.join(PLOT_PATH, ad.filename.replace(".fits", ".png")))
os.rename(
os.path.join(cwd, slit_illum_ad.filename.replace(".fits", ".png")),
os.path.join(PLOT_PATH, ad.filename.replace(".fits", ".png")))
def test_split_mosaic_into_extensions_metadata(filename):
"""
Tests that the metadata is correctly propagated to the split object.
"""
ad = astrodata.open(download_from_archive(filename))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.prepare()
p.overscanCorrect()
mosaicked_ad = p.mosaicDetectors().pop()
ad2 = primitives_gmos_longslit._split_mosaic_into_extensions(
ad, mosaicked_ad, border_size=10)
for (ext, ext2) in zip(ad, ad2):
assert (ext.shape == ext2.shape)
assert (ext.data_section() == ext2.data_section())
assert (ext.detector_section() == ext2.detector_section())
assert (ext.array_section() == ext2.array_section())
def _reduce_data(ad, master_arc, master_bias, master_flat):
with output_path():
# Use config to prevent outputs when running Reduce via API
logutils.config(file_name='log_{}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(bias=master_bias)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.flatCorrect(flat=master_flat)
# p.applyQECorrection(arc=master_arc)
processed_ad = p.writeOutputs().pop()
return processed_ad
def test_regression_on_qe_correct(ad, arc_ad, change_working_dir,
ref_ad_factory):
with change_working_dir():
logutils.config(
file_name='log_test_regression{}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.QECorrect(arc=arc_ad)
qe_corrected_ad = p.writeOutputs().pop()
assert 'QECORR' in qe_corrected_ad.phu.keys()
ref_ad = ref_ad_factory(qe_corrected_ad.filename)
for qe_corrected_ext, ref_ext in zip(qe_corrected_ad, ref_ad):
np.testing.assert_allclose(np.ma.masked_array(
qe_corrected_ext.data, mask=qe_corrected_ext.mask),
np.ma.masked_array(ref_ext.data,
mask=ref_ext.mask),
atol=0.05)
def test_regression_in_distortion_correct(ad, change_working_dir, ref_ad_factory):
"""
Runs the `distortionCorrect` primitive on a preprocessed data and compare
its model with the one in the reference file.
"""
with change_working_dir():
logutils.config(
file_name='log_regression_{:s}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([deepcopy(ad)])
p.viewer = geminidr.dormantViewer(p, None)
p.distortionCorrect(arc=deepcopy(ad), order=3, subsample=1)
dist_corrected_ad = p.writeOutputs()[0]
ref_ad = ref_ad_factory(dist_corrected_ad.filename)
for ext, ext_ref in zip(dist_corrected_ad, ref_ad):
data = np.ma.masked_invalid(ext.data)
ref_data = np.ma.masked_invalid(ext_ref.data)
np.testing.assert_allclose(data, ref_data, atol=1)
def test_regression_sky_correct_from_slit(filename, params, refname,
change_working_dir, path_to_inputs,
path_to_refs):
func = params.get('function', 'spline')
if not func.startswith('spline') and ASTROPY_LT_42:
pytest.skip('Astropy 4.2 is required to use the linear fitter '
'with weights')
path = os.path.join(path_to_inputs, filename)
ad = astrodata.open(path)
with change_working_dir():
logutils.config(file_name=f'log_regression_{ad.data_label()}.txt')
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.viewer = geminidr.dormantViewer(p, None)
p.skyCorrectFromSlit(**params)
sky_subtracted_ad = p.writeOutputs(outfilename=refname).pop()
ref_ad = astrodata.open(os.path.join(path_to_refs, refname))
for ext, ref_ext in zip(sky_subtracted_ad, ref_ad):
np.testing.assert_allclose(ext.data, ref_ext.data, atol=0.01)
def test_regression_on_qe_correct(ad, arc_ad, change_working_dir,
ref_ad_factory):
# The GMOS-N tests need to be run with `use_iraf=False` because the
# reference files for those use the DRAGONS spline models.
is_gmos_s = ad.instrument() == 'GMOS-S'
with change_working_dir():
logutils.config(
file_name='log_test_regression{}.txt'.format(ad.data_label()))
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.QECorrect(arc=arc_ad, use_iraf=is_gmos_s)
qe_corrected_ad = p.writeOutputs().pop()
assert 'QECORR' in qe_corrected_ad.phu.keys()
ref_ad = ref_ad_factory(qe_corrected_ad.filename)
for qe_corrected_ext, ref_ext in zip(qe_corrected_ad, ref_ad):
np.testing.assert_allclose(np.ma.masked_array(
qe_corrected_ext.data, mask=qe_corrected_ext.mask),
np.ma.masked_array(ref_ext.data,
mask=ref_ext.mask),
atol=0.05)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from recipe_system.utils.reduce_utils import normalize_ucals
from recipe_system.reduction.coreReduce import Reduce
associated_calibrations = {
"N20180109S0287.fits": {
"arcs": ["N20180109S0315.fits"],
"bias": [
"N20180103S0563.fits",
"N20180103S0564.fits",
"N20180103S0565.fits",
"N20180103S0566.fits",
"N20180103S0567.fits",
],
"flat": ["N20180109S0288.fits"],
}
}
root_path = os.path.join("./dragons_test_inputs/")
module_path = "geminidr/gmos/spect/{}/".format(__file__.split('.')[0])
path = os.path.join(root_path, module_path)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
bias_path = [download_from_archive(f) for f in cals['bias']]
flat_path = [download_from_archive(f) for f in cals['flat']]
arc_path = [download_from_archive(f) for f in cals['arcs']]
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing BIAS for {:s}'.format(data_label))
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_path)
bias_reduce.runr()
bias_master = bias_reduce.output_filenames.pop()
calibration_files = ['processed_bias:{}'.format(bias_master)]
del bias_reduce
print('Reducing FLAT for {:s}'.format(data_label))
logutils.config(file_name='log_flat_{}.txt'.format(data_label))
flat_reduce = Reduce()
flat_reduce.files.extend(flat_path)
flat_reduce.ucals = normalize_ucals(flat_reduce.files,
calibration_files)
flat_reduce.runr()
flat_master = flat_reduce.output_filenames.pop()
calibration_files.append('processed_flat:{}'.format(flat_master))
del flat_reduce
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend(arc_path)
arc_reduce.ucals = normalize_ucals(arc_reduce.files, calibration_files)
arc_reduce.runr()
arc_master = arc_reduce.output_filenames.pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.flatCorrect(flat=flat_master)
p.QECorrect(arc=arc_master)
p.distortionCorrect(arc=arc_master)
p.findSourceApertures(max_apertures=1)
p.skyCorrectFromSlit()
p.traceApertures()
p.extract1DSpectra()
p.calculateSensitivity()
os.chdir("inputs/")
_ = p.writeOutputs().pop()
os.chdir("../../")
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from gempy.utils import logutils
from recipe_system.reduction.coreReduce import Reduce
from recipe_system.utils.reduce_utils import normalize_ucals
from geminidr.gmos.tests.spect import CREATED_INPUTS_PATH_FOR_TESTS
associated_calibrations = {
"N20180109S0287.fits": {
'bias': [
"N20180103S0563.fits", "N20180103S0564.fits",
"N20180103S0565.fits", "N20180103S0566.fits",
"N20180103S0567.fits"
],
'flat': ["N20180109S0288.fits"],
'arcs': ["N20180109S0315.fits"]
}
}
module_name, _ = os.path.splitext(os.path.basename(__file__))
path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name)
os.makedirs(path, exist_ok=True)
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
os.makedirs("inputs/", exist_ok=True)
for filename, cals in associated_calibrations.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
bias_path = [download_from_archive(f) for f in cals['bias']]
flat_path = [download_from_archive(f) for f in cals['flat']]
arc_path = [download_from_archive(f) for f in cals['arcs']]
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing BIAS for {:s}'.format(data_label))
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_path)
bias_reduce.runr()
bias_master = bias_reduce.output_filenames.pop()
calibration_files = ['processed_bias:{}'.format(bias_master)]
del bias_reduce
print('Reducing FLAT for {:s}'.format(data_label))
logutils.config(file_name='log_flat_{}.txt'.format(data_label))
flat_reduce = Reduce()
flat_reduce.files.extend(flat_path)
flat_reduce.ucals = normalize_ucals(flat_reduce.files,
calibration_files)
flat_reduce.runr()
flat_master = flat_reduce.output_filenames.pop()
calibration_files.append('processed_flat:{}'.format(flat_master))
del flat_reduce
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend(arc_path)
arc_reduce.ucals = normalize_ucals(arc_reduce.files, calibration_files)
arc_reduce.runr()
arc_master = arc_reduce.output_filenames.pop()
del arc_reduce
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None, user_bpm=None, add_illum_mask=False)
p.addVAR(read_noise=True, poisson_noise=False)
p.overscanCorrect(function="spline",
high_reject=3.,
low_reject=3.,
nbiascontam=0,
niterate=2,
order=None)
p.biasCorrect(bias=bias_master, do_bias=True)
p.ADUToElectrons()
p.addVAR(poisson_noise=True, read_noise=False)
p.flatCorrect(flat=flat_master)
p.QECorrect(arc=arc_master)
p.distortionCorrect(arc=arc_master, order=3, subsample=1)
p.findSourceApertures(max_apertures=1,
threshold=0.01,
min_sky_region=20)
p.skyCorrectFromSlit(order=5, grow=0)
p.traceApertures(order=2,
nsum=10,
step=10,
max_missed=5,
max_shift=0.05)
p.extract1DSpectra(grow=10, method="standard", width=None)
os.chdir("inputs/")
processed_ad = p.writeOutputs().pop()
print('Wrote pre-processed file to:\n'
' {:s}'.format(processed_ad.filename))
os.chdir("../")
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed twilight flat data and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
from geminidr.gmos.tests.longslit import INPUTS_ROOT_PATH
associated_calibrations = {
"S20190204S0006.fits": {
"bias": ["S20190203S0110.fits",
"S20190203S0109.fits",
"S20190203S0108.fits",
"S20190203S0107.fits",
"S20190203S0106.fits"],
"twilight": ["S20190204S0006.fits"],
},
"N20190103S0462.fits": {
"bias": ["N20190102S0531.fits",
"N20190102S0530.fits",
"N20190102S0529.fits",
"N20190102S0528.fits",
"N20190102S0527.fits"],
"twilight": ["N20190103S0462.fits",
"N20190103S0463.fits"],
},
"N20190327S0056.fits": {
"bias": ["N20190327S0098.fits",
"N20190327S0099.fits",
"N20190327S0100.fits",
"N20190327S0101.fits",
"N20190327S0102.fits"],
"twilight": ["N20190327S0056.fits"],
}
}
module_name, _ = os.path.splitext(os.path.basename(__file__))
path = os.path.join(INPUTS_ROOT_PATH, module_name)
os.makedirs(path, exist_ok=True)
os.chdir(path)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, cals in associated_calibrations.items():
print('Downloading files...')
twilight_path = [download_from_archive(f) for f in cals['twilight']]
bias_path = [download_from_archive(f) for f in cals['bias']]
twilight_ad = astrodata.open(twilight_path[0])
data_label = twilight_ad.data_label()
print('Reducing BIAS for {:s}'.format(data_label))
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_path)
bias_reduce.runr()
bias_master = bias_reduce.output_filenames.pop()
del bias_reduce
print('Reducing twilight flat:')
logutils.config(file_name='log_twilight_{}.txt'.format(data_label))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
p = primitives_gmos_longslit.GMOSLongslit(
[astrodata.open(f) for f in twilight_path])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.stackFrames()
# Write non-mosaicked data
twilight = p.writeOutputs(suffix="_twilight", strip=True)[0]
# Write mosaicked data
p = primitives_gmos_longslit.GMOSLongslit([twilight])
p.mosaicDetectors()
p.writeOutputs(suffix="_mosaickedTwilight", strip=True)
def test_create_slit_illumination_with_mosaicked_data(ad, change_working_dir, request):
"""
Test that can run `makeSlitIllum` in mosaicked data. This primitive
creates a 2D image that is used to normalize the input data.
After normalization, the intensity along a column should be more or less
constant.
There are several ways of doing this but, given the noise levels, we bin
the data, fit a polynomium, and check that the fitted polynomium has its 1st
and 2nd coefficients almost zero.
"""
plot = request.config.getoption("--do-plots")
np.random.seed(0)
with change_working_dir():
cwd = os.getcwd()
print("Running tests inside folder:\n {}".format(cwd))
assert hasattr(ad[0], "wcs")
p = primitives_gmos_longslit.GMOSLongslit([ad])
p.makeSlitIllum(bins=25, border=10, debug_plot=plot)
slit_illum_ad = p.writeOutputs(
suffix="_mosaickedSlitIllum", strip=True)[0]
for ext, slit_ext in zip(ad, slit_illum_ad):
assert ext.shape == slit_ext.shape
# Create output data
data_o = (np.ma.masked_array(ext.data, mask=ext.mask) /
np.ma.masked_array(slit_ext.data, mask=slit_ext.mask))
# Bin columns
fitter = fitting.LinearLSQFitter()
model = models.Polynomial1D(degree=2)
nbins = 50
rows = np.arange(data_o.shape[0])
for i in range(nbins):
col_start = i * data_o.shape[1] // nbins
col_end = (i + 1) * data_o.shape[1] // nbins
cols = np.ma.mean(data_o[:, col_start:col_end], axis=1)
fitted_model = fitter(model, rows, cols)
# Check column is linear
np.testing.assert_allclose(fitted_model.c2.value, 0, atol=0.01)
# Check if slope is (almost) horizontal (< 1.0 deg)
assert np.abs(
np.rad2deg(
np.arctan(
fitted_model.c1.value / (rows.size // 2)))) < 1.0
if plot:
os.makedirs(PLOT_PATH, exist_ok=True)
print("Renaming plots to ",
os.path.join(PLOT_PATH, ad.filename.replace(".fits", ".png")))
os.rename(
os.path.join(cwd, slit_illum_ad.filename.replace(".fits", ".png")),
os.path.join(PLOT_PATH, ad.filename.replace(".fits", ".png")))
def create_inputs_recipe(use_branch_name=False):
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
import os
from astrodata.testing import download_from_archive
from gempy.utils import logutils
from geminidr.gmos.tests.spect import CREATED_INPUTS_PATH_FOR_TESTS
from recipe_system.reduction.coreReduce import Reduce
from recipe_system.utils.reduce_utils import normalize_ucals
module_name, _ = os.path.splitext(os.path.basename(__file__))
path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name)
os.makedirs(path, exist_ok=True)
os.chdir(path)
input_path = os.path.join(path, "inputs/")
os.makedirs(input_path, exist_ok=True)
for filename, cals in associated_calibrations.items():
print(filename)
print(cals)
sci_path = download_from_archive(filename)
cals = associated_calibrations[filename]
bias_paths = [download_from_archive(f) for f in cals['bias']]
flat_paths = [download_from_archive(f) for f in cals['flat']]
arc_paths = [download_from_archive(f) for f in cals['arcs']]
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
logutils.config(file_name='log_bias_{}.txt'.format(data_label))
bias_reduce = Reduce()
bias_reduce.files.extend(bias_paths)
bias_reduce.runr()
bias_master = bias_reduce.output_filenames.pop()
calibration_files = ['processed_bias:{}'.format(bias_master)]
del bias_reduce
logutils.config(file_name='log_flat_{}.txt'.format(data_label))
flat_reduce = Reduce()
flat_reduce.files.extend(flat_paths)
flat_reduce.ucals = normalize_ucals(calibration_files)
flat_reduce.runr()
flat_master = flat_reduce.output_filenames.pop()
calibration_files.append('processed_flat:{}'.format(flat_master))
del flat_reduce
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend(arc_paths)
arc_reduce.ucals = normalize_ucals(calibration_files)
os.chdir(input_path)
arc_reduce.runr()
_ = arc_reduce.output_filenames.pop()
os.chdir(path)
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ(static_bpm=None)
p.addVAR(read_noise=True)
p.overscanCorrect()
p.biasCorrect(bias=bias_master)
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.flatCorrect(flat=flat_master)
os.chdir(input_path)
_ = p.writeOutputs().pop()
os.chdir(path)
def create_inputs_recipe():
"""
Creates input data for tests using pre-processed standard star and its
calibration files.
The raw files will be downloaded and saved inside the path stored in the
`$DRAGONS_TEST/raw_inputs` directory. Processed files will be stored inside
a new folder called "dragons_test_inputs". The sub-directory structure
should reflect the one returned by the `path_to_inputs` fixture.
"""
input_data = {
"N20180508S0021.fits": {
"arc": "N20180615S0409.fits",
},
"S20190204S0079.fits": {
"arc": "S20190206S0030.fits",
},
"S20181024S0035.fits": {
"arc": "S20181025S0012.fits",
},
}
module_name, _ = os.path.splitext(os.path.basename(__file__))
path = os.path.join(CREATED_INPUTS_PATH_FOR_TESTS, module_name)
os.makedirs(path, exist_ok=True)
os.chdir(path)
os.makedirs("inputs/", exist_ok=True)
print('Current working directory:\n {:s}'.format(os.getcwd()))
for filename, pars in input_data.items():
print('Downloading files...')
sci_path = download_from_archive(filename)
arc_path = download_from_archive(pars['arc'])
sci_ad = astrodata.open(sci_path)
data_label = sci_ad.data_label()
print('Reducing ARC for {:s}'.format(data_label))
logutils.config(file_name='log_arc_{}.txt'.format(data_label))
arc_reduce = Reduce()
arc_reduce.files.extend([arc_path])
arc_reduce.runr()
arc = arc_reduce.output_filenames.pop()
print('Reducing pre-processed data:')
logutils.config(file_name='log_{}.txt'.format(data_label))
p = primitives_gmos_longslit.GMOSLongslit([sci_ad])
p.prepare()
p.addDQ()
p.addVAR(read_noise=True)
p.overscanCorrect()
p.ADUToElectrons()
p.addVAR(poisson_noise=True)
p.mosaicDetectors()
p.distortionCorrect(arc=arc)
p.findSourceApertures(max_apertures=1)
p.traceApertures(order=2,
nsum=20,
step=10,
max_shift=0.09,
max_missed=5)
os.chdir("inputs/")
processed_ad = p.writeOutputs().pop()
os.chdir("../")
print(f'Wrote pre-processed file to:\n {processed_ad.filename}')
def test_full_frame_distortion_works_on_smaller_region(fname, path_to_inputs):
"""
Takes a full-frame arc and self-distortion-corrects it. It then fakes
subregions of this and corrects those using the full-frame distortion to
confirm that the result is the same as the appropriate region of the
distortion-corrected full-frame image. There's no need to do this more
than once for a given binning, so we loop within the function, keeping
track of binnings we've already processed.
"""
subpath, basename = os.path.split(fname)
basename, extension = os.path.splitext(basename)
basename = basename.split('_')[0] + extension
raw_fname = testing.download_from_archive(basename, path=subpath)
_ad = astrodata.open(os.path.join(path_to_inputs, subpath, raw_fname))
NSUB = 4 # we're going to take combos of horizontal quadrants
completed_binnings = []
xbin, ybin = _ad.detector_x_bin(), _ad.detector_y_bin()
if _ad.detector_roi_setting() != "Full Fame" or (xbin, ybin) in completed_binnings:
return
p = primitives_gmos_longslit.GMOSLongslit([_ad])
p.viewer.viewer_name = None
p.prepare()
p.addDQ(static_bpm=None)
p.overscanCorrect()
p.ADUToElectrons()
p.mosaicDetectors(outstream='mosaic')
# Speed things up a bit with a larger step
p.determineDistortion(stream='mosaic', step=48 // ybin)
ad_out = p.distortionCorrect(
[deepcopy(_ad)], arc=p.streams['mosaic'][0], order=1)[0]
for start in range(NSUB):
for end in range(start + 1, NSUB + 1):
ad_copy = deepcopy(_ad)
y1b = start * _ad[0].shape[0] // NSUB
y2b = end * _ad[0].shape[0] // NSUB
y1, y2 = y1b * ybin, y2b * ybin # unbinned pixels
# Fake the section header keywords and set the SCI and DQ
# to the appropriate sub-region
for ext in ad_copy:
arrsec = ext.array_section()
detsec = ext.detector_section()
ext.hdr['CCDSEC'] = '[{}:{},{}:{}]'.format(arrsec.x1 + 1,
arrsec.x2, y1 + 1, y2)
ext.hdr['DETSEC'] = '[{}:{},{}:{}]'.format(detsec.x1 + 1,
detsec.x2, y1 + 1, y2)
ext.data = ext.data[y1b:y2b]
ext.mask = ext.mask[y1b:y2b]
ext.hdr['DATASEC'] = '[1:{},1:{}]'.format(ext.shape[1], y2b - y1b)
ad2 = p.distortionCorrect([ad_copy], arc=p.streams['mosaic'][0],
order=1)[0]
# It's GMOS LS so the offset between this AD and the full-frame
# will be the same as the DETSEC offset, but the width may be
# smaller so we need to shuffle the smaller image within the
# larger one to look for a match
ny, nx = ad2[0].shape
xsizediff = ad_out[0].shape[1] - nx
for xoffset in range(xsizediff + 1):
np.testing.assert_allclose(
np.ma.masked_array(ad2[0].data, mask=ad2[0].mask),
ad_out[0].data[y1b:y1b + ny, xoffset:xoffset + nx],
atol=0.01,
err_msg="Problem with {} {}:{}".format(ad.filename, start, end))
completed_binnings.append((xbin, ybin))
