def test_absolute_path():
current_absolute_path = os.path.abspath(
"test/test_data/sprat_LHS6328.list")
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(filelist=current_absolute_path)
img.load_data()
img.reduce()
def test_input_with_one_line_and_reduction():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328_one_line.list")
)
img.load_data()
img.reduce()
def test_saturated_data():
# Science frame
lhs6328_frame = image_reduction.ImageReduction(
log_level="DEBUG", log_file_folder=os.path.join(HERE, "test_output")
)
lhs6328_frame.add_filelist(
os.path.join(
HERE, "test_data", "sprat_LHS6328_fake_saturated_data.list"
)
)
lhs6328_frame.load_data()
lhs6328_frame.reduce()
lhs6328_frame.save_masks(
os.path.join(
HERE, "test_output", "fake_saturated_reduced_image_mask.fits"
),
overwrite=True,
)
assert (lhs6328_frame.bad_mask == lhs6328_frame.saturation_mask).all()
lhs6328_twodspec = spectral_reduction.TwoDSpec(
lhs6328_frame,
spatial_mask=spatial_mask,
spec_mask=spec_mask,
cosmicray=True,
readnoise=5.7,
log_level="DEBUG",
log_file_folder=os.path.join(HERE, "test_output"),
)
lhs6328_twodspec.ap_trace(nspec=1)
lhs6328_twodspec.ap_extract(model="lowess", lowess_frac=0.8)
def test_input_with_only_one_column():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(
HERE, "test_data", "sprat_LHS6328_expect_fail.list"
)
)
def test_input_with_wrong_bias_combine_type():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
img.set_properties(combinetype_bias="FAIL")
img.load_data()
img.reduce()
def test_input_with_multiple_frames_to_combine_and_reduction():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(filelist=os.path.join(HERE, "test_data",
"sprat_LHS6328_repeated_data.list"))
img.set_properties(combinetype_flat="average")
img.load_data()
img.reduce()
def test_input_with_fits_data_object():
filelist = np.loadtxt(
os.path.join(HERE, "test_data", "sprat_LHS6328.list"),
delimiter=",",
dtype="object",
)
for i, filepath in enumerate(filelist[:, 1]):
filelist[:, 1][i] = os.path.join(os.path.join(HERE, "test_data"),
filepath.strip())
img = image_reduction.ImageReduction(log_file_name=None)
imtype = filelist[:, 0].astype("object")
impath = filelist[:, 1].astype("object")
dark_list = impath[imtype == "dark"]
arc_list = impath[imtype == "arc"]
light_list = impath[imtype == "light"]
for i in range(light_list.size):
light = fits.open(light_list[i])[0]
img.add_light(light.data, light.header, float(light.header["EXPTIME"]))
for i in range(arc_list.size):
arc = fits.open(arc_list[i])[0]
img.add_arc(arc.data, arc.header)
for i in range(dark_list.size):
dark = fits.open(dark_list[i])[0]
img.add_dark(dark.data, dark.header, float(dark.header["EXPTIME"]))
img.reduce()
def test_space_separated_input_2():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.txt"),
delimiter=" ",
)
img.load_data()
img.reduce()
def test_tsv_input_2():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.tsv"),
ftype="tsv",
delimiter="\t",
)
img.load_data()
img.reduce()
def test_cosmicray_cleaning():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
img.set_properties(cosmicray=True,
psfmodel="gaussx",
combinetype_bias="average")
img.load_data()
img.reduce()
def test_input_with_data_cube_extra_bracket():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(filelist=os.path.join(
HERE,
"test_data",
"sprat_LHS6328_fake_data_cube_extra_bracket.list",
))
img.load_data()
img.reduce()
def test_input_with_numpy_array_and_reduction():
filelist = np.loadtxt(
os.path.join(HERE, "test_data", "sprat_LHS6328.list"),
delimiter=",",
dtype="object",
)
for i, filepath in enumerate(filelist[:, 1]):
filelist[:, 1][i] = os.path.join(HERE, "test_data", filepath.strip())
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(filelist=filelist)
img.load_data()
img.reduce()
def test_input_with_numpy_array_and_clean_bad_pixels_expect_fail():
filelist = np.loadtxt(
os.path.join(HERE, "test_data", "sprat_LHS6328.list"),
delimiter=",",
dtype="object",
)
for i, filepath in enumerate(filelist[:, 1]):
filelist[:, 1][i] = os.path.join(os.path.join(HERE, "test_data"),
filepath.strip())
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(filelist=filelist)
img.load_data()
img.reduce()
img.create_bad_mask()
img.heal_bad_pixels(bad_mask=np.ones(100))
def test_reduction_and_save(mock_show):
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
img.load_data()
img.reduce()
img.save_fits(os.path.join(HERE, "test_output", "reduced_image"),
overwrite=True)
img.inspect(
display=False,
filename=os.path.join(HERE, "test_output", "reduced_image"),
save_fig=True,
fig_type="iframe+png+svg+pdf+jpg",
)
img.inspect(display=True)
img.list_files()
def test_set_everything_to_bla():
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.list")
)
img.set_properties(
saxis="bla",
combinetype_light="bla",
sigma_clipping_light="bla",
clip_low_light="bla",
clip_high_light="bla",
exptime_light="bla",
exptime_light_keyword="bla",
combinetype_arc="bla",
sigma_clipping_arc="bla",
clip_low_arc="bla",
clip_high_arc="bla",
combinetype_dark="bla",
sigma_clipping_dark="bla",
clip_low_dark="bla",
clip_high_dark="bla",
exptime_dark="bla",
exptime_dark_keyword="bla",
combinetype_bias="bla",
sigma_clipping_bias="bla",
clip_low_bias="bla",
clip_high_bias="bla",
combinetype_flat="bla",
sigma_clipping_flat="bla",
clip_low_flat="bla",
clip_high_flat="bla",
exptime_flat="bla",
exptime_flat_keyword="bla",
cosmicray="bla",
gain="bla",
readnoise="bla",
fsmode="bla",
psfmodel="bla",
heal_pixels="bla",
cutoff="bla",
grow="bla",
iterations="bla",
diagonal="bla",
)
img.load_data()
img.reduce()
def test_logger():
imred_debug = image_reduction.ImageReduction(
log_level="DEBUG",
logger_name="imred_debug",
log_file_name="imred_debug.log",
log_file_folder=os.path.join(HERE, "test_output"),
)
imred_info = image_reduction.ImageReduction(
log_level="INFO",
logger_name="imred_info",
log_file_name="imred_info.log",
log_file_folder=os.path.join(HERE, "test_output"),
)
imred_warning = image_reduction.ImageReduction(
log_level="WARNING",
logger_name="imred_warning",
log_file_name="imred_warning.log",
log_file_folder=os.path.join(HERE, "test_output"),
)
imred_error = image_reduction.ImageReduction(
log_level="ERROR",
logger_name="imred_error",
log_file_name="imred_error.log",
log_file_folder=os.path.join(HERE, "test_output"),
)
imred_critical = image_reduction.ImageReduction(
log_level="CRITICAL",
logger_name="imred_critical",
log_file_name="imred_critical.log",
log_file_folder=os.path.join(HERE, "test_output"),
)
imred_debug.logger.debug("debug: debug mode")
imred_debug.logger.info("debug: info mode")
imred_debug.logger.warning("debug: warning mode")
imred_debug.logger.error("debug: error mode")
imred_debug.logger.critical("debug: critical mode")
imred_info.logger.debug("info: debug mode")
imred_info.logger.info("info: info mode")
imred_info.logger.warning("info: warning mode")
imred_info.logger.error("info: error mode")
imred_info.logger.critical("info: critical mode")
imred_warning.logger.debug("warning: debug mode")
imred_warning.logger.info("warning: info mode")
imred_warning.logger.warning("warning: warning mode")
imred_warning.logger.error("warning: error mode")
imred_warning.logger.critical("warning: critical mode")
imred_error.logger.debug("error: debug mode")
imred_error.logger.info("error: info mode")
imred_error.logger.warning("error: warning mode")
imred_error.logger.error("error: error mode")
imred_error.logger.critical("error: critical mode")
imred_critical.logger.debug("critical: debug mode")
imred_critical.logger.info("critical: info mode")
imred_critical.logger.warning("critical: warning mode")
imred_critical.logger.error("critical: error mode")
imred_critical.logger.critical("critical: critical mode")
debug_debug_length = file_len(
os.path.join(HERE, "test_output", "imred_debug.log"))
debug_info_length = file_len(
os.path.join(HERE, "test_output", "imred_info.log"))
debug_warning_length = file_len(
os.path.join(HERE, "test_output", "imred_warning.log"))
debug_error_length = file_len(
os.path.join(HERE, "test_output", "imred_error.log"))
debug_critical_length = file_len(
os.path.join(HERE, "test_output", "imred_critical.log"))
assert (debug_debug_length == 5
), "Expecting 5 lines in the log file, " + "{} is logged.".format(
debug_debug_length)
assert (debug_info_length == 4
), "Expecting 4 lines in the log file, " + "{} is logged.".format(
debug_info_length)
assert (debug_warning_length == 3
), "Expecting 3 lines in the log file, " + "{} is logged.".format(
debug_warning_length)
assert (debug_error_length == 2
), "Expecting 2 lines in the log file, " + "{} is logged.".format(
debug_error_length)
assert (debug_critical_length == 1
), "Expecting 1 lines in the log file, " + "{} is logged.".format(
debug_critical_length)
def test_str_verbose_at_initialisation():
img = image_reduction.ImageReduction(verbose="haha", log_file_name=None)
7584.68,
7642.02,
7740.31,
7802.65,
7887.40,
7967.34,
8057.258,
]
element = ["Xe"] * len(atlas)
spatial_mask = np.arange(35, 200)
spec_mask = np.arange(50, 1024)
# Science frame
lhs6328_frame = image_reduction.ImageReduction(log_level="DEBUG",
log_file_folder=os.path.join(
HERE, "test_output"))
lhs6328_frame.add_filelist(
os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
lhs6328_frame.load_data()
lhs6328_frame.reduce()
lhs6328_twodspec = spectral_reduction.TwoDSpec(
lhs6328_frame,
spatial_mask=spatial_mask,
spec_mask=spec_mask,
cosmicray=True,
sigclip=1.0,
readnoise=5.7,
log_level="DEBUG",
log_file_folder=os.path.join(HERE, "test_output"),
import os
from unittest.mock import patch
from astropy import units
from astropy.io import fits
from astropy.nddata import CCDData
import numpy as np
import pytest
from aspired import image_reduction
from aspired import spectral_reduction
from aspired import util
HERE = os.path.dirname(os.path.realpath(__file__))
img = image_reduction.ImageReduction(log_file_name=None)
img.add_filelist(
filelist=os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
img.load_data()
img.reduce()
img_with_fits = copy.copy(img)
img_with_fits._create_image_fits()
img_in_hdulist = fits.HDUList(img_with_fits.image_fits)
img_in_CCDData = CCDData(
img.light_reduced,
header=fits.Header(img.light_header[0]),
unit=units.count,
)
lhs6328_twodspec.save_fits(
output="count",
filename=os.path.join(HERE, "test_output",
"user_supplied_trace_for_extraction"),
overwrite=True,
)
lhs6328_twodspec.remove_trace()
spatial_mask = np.arange(30, 200)
spec_mask = np.arange(50, 1024)
# Science frame
lhs6328_frame = image_reduction.ImageReduction(log_level="INFO",
log_file_name=None)
lhs6328_frame.add_filelist(
os.path.join(HERE, "test_data", "sprat_LHS6328.list"))
lhs6328_frame.load_data()
lhs6328_frame.reduce()
lhs6328_twodspec = spectral_reduction.TwoDSpec(
lhs6328_frame,
spatial_mask=spatial_mask,
spec_mask=spec_mask,
cosmicray=True,
readnoise=5.7,
log_level="DEBUG",
log_file_name=None,
)
def test_full_run(mock_show):
# Extract two spectra
# Line list
atlas = [
4193.5,
4385.77,
4500.98,
4524.68,
4582.75,
4624.28,
4671.23,
4697.02,
4734.15,
4807.02,
4921.48,
5028.28,
5618.88,
5823.89,
5893.29,
5934.17,
6182.42,
6318.06,
6472.841,
6595.56,
6668.92,
6728.01,
6827.32,
6976.18,
7119.60,
7257.9,
7393.8,
7584.68,
7642.02,
7740.31,
7802.65,
7887.40,
7967.34,
8057.258,
]
element = ["Xe"] * len(atlas)
spatial_mask = np.arange(35, 200)
spec_mask = np.arange(50, 1024)
# Science frame
lhs6328_frame = image_reduction.ImageReduction(
log_level="ERROR", log_file_folder=os.path.join(HERE, "test_output"))
lhs6328_frame.add_filelist("test/test_data/sprat_LHS6328.list")
lhs6328_frame.load_data()
lhs6328_frame.reduce()
lhs6328_frame.save_fits("test/test_output/test_full_run_standard_image",
overwrite=True)
lhs6328_twodspec = spectral_reduction.TwoDSpec(
lhs6328_frame,
spatial_mask=spatial_mask,
spec_mask=spec_mask,
cosmicray=True,
psfmodel="gaussxy",
readnoise=5.7,
log_level="ERROR",
log_file_folder=os.path.join(HERE, "test_output"),
)
lhs6328_twodspec.ap_trace(nspec=2, display=False)
# Optimal extraction to get the LSF for force extraction below
lhs6328_twodspec.ap_extract(apwidth=15,
skywidth=10,
skydeg=1,
optimal=True,
display=False)
# Force extraction
lhs6328_twodspec.ap_extract(
apwidth=15,
skywidth=10,
skydeg=1,
optimal=True,
forced=True,
variances=lhs6328_twodspec.spectrum_list[1].var,
display=False,
)
# Aperture extraction
lhs6328_twodspec.ap_extract(
apwidth=15,
skywidth=5,
skydeg=1,
optimal=False,
display=False,
save_fig=True,
filename=os.path.join(HERE, "test_output", "test_full_run_extract"),
)
lhs6328_twodspec.inspect_line_spread_function(spec_id=0,
display=False,
return_jsonstring=True)
lhs6328_twodspec.inspect_line_spread_function(display=False,
return_jsonstring=True)
# Optimal extraction
lhs6328_twodspec.ap_extract(
apwidth=15,
skywidth=5,
skydeg=1,
optimal=True,
forced=True,
variances=1000000.0,
display=False,
save_fig=True,
filename=os.path.join(HERE, "test_output", "test_full_run_extract"),
)
lhs6328_twodspec.inspect_line_spread_function(
save_fig=True,
display=True,
open_iframe=False,
filename=os.path.join(HERE, "test_output", "test_full_run_lsf"),
)
lhs6328_twodspec.save_fits(
filename=os.path.join(HERE, "test_output", "test_full_run_twospec"),
overwrite=True,
)
# Standard frame
standard_frame = image_reduction.ImageReduction(
log_level="ERROR", log_file_folder=os.path.join(HERE, "test_output"))
standard_frame.add_filelist(
os.path.join(HERE, "test_data", "sprat_Hiltner102.list"))
standard_frame.load_data()
standard_frame.reduce()
standard_frame.save_fits(
os.path.join(HERE, "test_output", "test_full_run_standard_image"),
overwrite=True,
)
hilt102_twodspec = spectral_reduction.TwoDSpec(
standard_frame,
cosmicray=True,
psfmodel="gaussyx",
spatial_mask=spatial_mask,
spec_mask=spec_mask,
readnoise=5.7,
log_level="ERROR",
log_file_folder=os.path.join(HERE, "test_output"),
)
hilt102_twodspec.ap_trace(nspec=1, resample_factor=10, display=False)
hilt102_twodspec.ap_extract(apwidth=15,
skysep=3,
skywidth=5,
skydeg=1,
optimal=True,
display=False)
hilt102_twodspec.inspect_line_spread_function(
display=True,
save_fig=True,
fig_type="jpg+png+pdf+svg+iframe",
filename=os.path.join(HERE, "test_output", "test_full_run_lsf"),
return_jsonstring=True,
)
# Extract the 1D arc by aperture sum of the traces provided
lhs6328_twodspec.extract_arc_spec(
display=True,
save_fig=True,
fig_type="jpg+png+pdf+svg+iframe",
filename=os.path.join(HERE, "test_output", "test_full_run_arc_spec"),
return_jsonstring=True,
)
hilt102_twodspec.extract_arc_spec(display=True)
hilt102_twodspec.create_fits(output="trace+count")
# Handle 1D Science spectrum
lhs6328_onedspec = spectral_reduction.OneDSpec(
log_level="ERROR", log_file_folder=os.path.join(HERE, "test_output"))
lhs6328_onedspec.from_twodspec(lhs6328_twodspec, stype="science")
lhs6328_onedspec.from_twodspec(hilt102_twodspec, stype="standard")
# Create the trace and count as FITS BEFORE flux and wavelength calibration
# This is an uncommon operation, but it should work.
lhs6328_onedspec.create_fits(output="trace+count",
stype="science+standard")
# Save the trace and count as FITS BEFORE flux and wavelength calibration
# This is an uncommon operation, but it should work.
lhs6328_onedspec.save_fits(
output="trace+count",
filename=os.path.join(HERE, "test_output", "test_full_run"),
stype="science+standard",
overwrite=True,
)
# Find the peaks of the arc
lhs6328_onedspec.find_arc_lines(display=True, stype="science+standard")
lhs6328_onedspec.find_arc_lines(display=False, stype="science+standard")
lhs6328_onedspec.inspect_arc_lines(spec_id=0,
display=True,
stype="science+standard")
lhs6328_onedspec.inspect_arc_lines(spec_id=0,
display=False,
stype="science+standard")
# Configure the wavelength calibrator
lhs6328_onedspec.initialise_calibrator(stype="science+standard")
lhs6328_onedspec.set_hough_properties(
num_slopes=1000,
xbins=200,
ybins=200,
min_wavelength=3500,
max_wavelength=8500,
stype="science+standard",
)
lhs6328_onedspec.set_ransac_properties(filter_close=True,
stype="science+standard")
lhs6328_onedspec.add_user_atlas(elements=element,
wavelengths=atlas,
stype="science+standard")
lhs6328_onedspec.do_hough_transform()
# Solve for the pixel-to-wavelength solution
lhs6328_onedspec.fit(max_tries=200,
stype="science+standard",
display=False)
lhs6328_onedspec.robust_refit()
# list the matched pixel-peaks
lhs6328_onedspec.get_pix_wave_pairs(spec_id=0)
lhs6328_onedspec.get_pix_wave_pairs()
# get all the calibrators
cal = lhs6328_onedspec.get_calibrator()
# Apply the wavelength calibration and display it
lhs6328_onedspec.apply_wavelength_calibration(stype="science+standard")
# Get the standard from the library
lhs6328_onedspec.load_standard(target="hiltner102")
lhs6328_onedspec.get_continuum()
lhs6328_onedspec.get_sensitivity(sens_deg=11,
method="polynomial",
mask_fit_size=1)
lhs6328_onedspec.get_sensitivity(k=3,
method="interpolate",
mask_fit_size=1)
lhs6328_onedspec.apply_flux_calibration(stype="science+standard")
lhs6328_onedspec.inspect_reduced_spectrum()
lhs6328_onedspec.get_telluric_profile()
lhs6328_onedspec.inspect_telluric_profile(display=False)
lhs6328_onedspec.apply_telluric_correction()
lhs6328_onedspec.inspect_reduced_spectrum()
# Apply atmospheric extinction correction
lhs6328_onedspec.set_atmospheric_extinction(location="orm")
lhs6328_onedspec.apply_atmospheric_extinction_correction()
lhs6328_onedspec.inspect_reduced_spectrum()
# Create FITS
lhs6328_onedspec.create_fits(output="trace+count")
# Modify FITS header for the trace
lhs6328_onedspec.modify_trace_header(0, "set", "COMMENT", "Hello World!")
print(lhs6328_onedspec.science_spectrum_list[0].trace_hdulist[0].
header["COMMENT"])
# Create more FITS
lhs6328_onedspec.create_fits(
output="trace+count+arc_spec+wavecal",
stype="science+standard",
recreate=False,
)
# Check the modified headers are not overwritten
print(lhs6328_onedspec.science_spectrum_list[0].trace_hdulist[0].
header["COMMENT"])
# Save as FITS (and create the ones that were not created earlier)
lhs6328_onedspec.save_fits(
output="trace+count+arc_spec+wavecal+wavelength+count_resampled+flux+"
"flux_resampled",
filename=os.path.join(HERE, "test_output", "test_full_run"),
stype="science+standard",
recreate=False,
overwrite=True,
)
# Check the modified headers are still not overwritten
print(lhs6328_onedspec.science_spectrum_list[0].trace_hdulist[0].
header["COMMENT"])
# Create more FITS
lhs6328_onedspec.create_fits(output="trace",
stype="science+standard",
recreate=True)
# Now the FITS header should be back to default
print(lhs6328_onedspec.science_spectrum_list[0].trace_hdulist[0].header)
# save as CSV
lhs6328_onedspec.save_csv(
output="trace+count+arc_spec+wavecal+wavelength+count_resampled+flux+"
"flux_resampled",
filename=os.path.join(HERE, "test_output", "test_full_run"),
stype="science+standard",
overwrite=True,
)
# save figure
lhs6328_onedspec.inspect_reduced_spectrum(
filename=os.path.join(HERE, "test_output", "test_full_run"),
display=False,
save_fig=True,
fig_type="iframe+png",
)
# Plot the hough search space
lhs6328_onedspec.plot_search_space(
filename=os.path.join(HERE, "test_output",
"test_full_run_plot_hough_space"),
display=False,
save_fig=True,
fig_type="iframe+png",
)