def display_slitlet_arrangement(fileobj,
grism=None,
spfilter=None,
bbox=None,
adjust=None,
geometry=None,
debugplot=0):
"""Display slitlet arrangment from CSUP keywords in FITS header.
Parameters
----------
fileobj : file object
FITS or TXT file object.
grism : str
Grism.
grism : str
Filter.
bbox : tuple of 4 floats
If not None, values for xmin, xmax, ymin and ymax.
adjust : bool
Adjust X range according to minimum and maximum csu_bar_left
and csu_bar_right (note that this option is overriden by 'bbox')
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
csu_bar_left : list of floats
Location (mm) of the left bar for each slitlet.
csu_bar_right : list of floats
Location (mm) of the right bar for each slitlet, using the
same origin employed for csu_bar_left (which is not the
value stored in the FITS keywords.
csu_bar_slit_center : list of floats
Middle point (mm) in between the two bars defining a slitlet.
csu_bar_slit_width : list of floats
Slitlet width (mm), computed as the distance between the two
bars defining the slitlet.
"""
if fileobj.name[-4:] == ".txt":
if grism is None:
raise ValueError("Undefined grism!")
if spfilter is None:
raise ValueError("Undefined filter!")
# define CsuConfiguration object
csu_config = CsuConfiguration()
csu_config._csu_bar_left = []
csu_config._csu_bar_right = []
csu_config._csu_bar_slit_center = []
csu_config._csu_bar_slit_width = []
# since the input filename has been opened with argparse in binary
# mode, it is necessary to close it and open it in text mode
fileobj.close()
# read TXT file
with open(fileobj.name, mode='rt') as f:
file_content = f.read().splitlines()
next_id_bar = 1
for line in file_content:
if len(line) > 0:
if line[0] not in ['#']:
line_contents = line.split()
id_bar = int(line_contents[0])
position = float(line_contents[1])
if id_bar == next_id_bar:
if id_bar <= EMIR_NBARS:
csu_config._csu_bar_left.append(position)
next_id_bar = id_bar + EMIR_NBARS
else:
csu_config._csu_bar_right.append(341.5 - position)
next_id_bar = id_bar - EMIR_NBARS + 1
else:
raise ValueError("Unexpected id_bar:" + str(id_bar))
# compute slit width and center
for i in range(EMIR_NBARS):
csu_config._csu_bar_slit_center.append(
(csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i]) /
2)
csu_config._csu_bar_slit_width.append(
csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i])
else:
# read input FITS file
hdulist = fits.open(fileobj.name)
image_header = hdulist[0].header
hdulist.close()
# additional info from header
grism = image_header['grism']
spfilter = image_header['filter']
# define slitlet arrangement
csu_config = CsuConfiguration.define_from_fits(fileobj)
# determine calibration
if grism in ["J", "OPEN"] and spfilter == "J":
wv_parameters = set_wv_parameters("J", "J")
elif grism in ["H", "OPEN"] and spfilter == "H":
wv_parameters = set_wv_parameters("H", "H")
elif grism in ["K", "OPEN"] and spfilter == "Ksp":
wv_parameters = set_wv_parameters("Ksp", "K")
elif grism in ["LR", "OPEN"] and spfilter == "YJ":
wv_parameters = set_wv_parameters("YJ", "LR")
elif grism in ["LR", "OPEN"] and spfilter == "HK":
wv_parameters = set_wv_parameters("HK", "LR")
else:
raise ValueError("Invalid grism + filter configuration")
crval1 = wv_parameters['poly_crval1_linear']
cdelt1 = wv_parameters['poly_cdelt1_linear']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# display arrangement
if abs(debugplot) >= 10:
print("slit left right center width min.wave max.wave")
print("==== ======= ======= ======= ===== ======== ========")
for i in range(EMIR_NBARS):
ibar = i + 1
csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar))
csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar))
csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1
if wvmin_useful is not None:
csu_crval1 = np.amax([csu_crval1, wvmin_useful])
if wvmax_useful is not None:
csu_crvaln = np.amin([csu_crvaln, wvmax_useful])
print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} "
"{5:8.2f} {6:8.2f}".format(
ibar, csu_config.csu_bar_left(ibar),
csu_config.csu_bar_right(ibar),
csu_config.csu_bar_slit_center(ibar),
csu_config.csu_bar_slit_width(ibar), csu_crval1,
csu_crvaln))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format(
np.mean(csu_config._csu_bar_left),
np.mean(csu_config._csu_bar_right),
np.mean(csu_config._csu_bar_slit_center),
np.mean(csu_config._csu_bar_slit_width)))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format(
np.mean(csu_config._csu_bar_left[::2]),
np.mean(csu_config._csu_bar_right[::2]),
np.mean(csu_config._csu_bar_slit_center[::2]),
np.mean(csu_config._csu_bar_slit_width[::2])))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format(
np.mean(csu_config._csu_bar_left[1::2]),
np.mean(csu_config._csu_bar_right[1::2]),
np.mean(csu_config._csu_bar_slit_center[1::2]),
np.mean(csu_config._csu_bar_slit_width[1::2])))
# display slit arrangement
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
if bbox is None:
if adjust:
xmin = min(csu_config._csu_bar_left)
xmax = max(csu_config._csu_bar_right)
dx = xmax - xmin
if dx == 0:
dx = 1
xmin -= dx / 20
xmax += dx / 20
ax.set_xlim(xmin, xmax)
else:
ax.set_xlim(0., 341.5)
ax.set_ylim(0, 56)
else:
ax.set_xlim(bbox[0], bbox[1])
ax.set_ylim(bbox[2], bbox[3])
ax.set_xlabel('csu_bar_position (mm)')
ax.set_ylabel('slit number')
for i in range(EMIR_NBARS):
ibar = i + 1
ax.add_patch(
patches.Rectangle((csu_config.csu_bar_left(ibar), ibar - 0.5),
csu_config.csu_bar_slit_width(ibar), 1.0))
ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar],
'-',
color='gray')
ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar],
'-',
color='gray')
plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" +
spfilter)
pause_debugplot(debugplot, pltshow=True)
# return results
return csu_config._csu_bar_left, csu_config._csu_bar_right, \
csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def refine_rectwv_coeff(input_image,
rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'.format(
refine_wavecalib_mode))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data('emirdrp.instrument.configs',
'Oliva_etal_2013.dat')
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1 * lok2]
catlines_reference_flux = catlines_all_flux[lok1 * lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
logger.info('list of useful slitlets: {}'.format(list_useful_slitlets))
logger.info(
'list of not useful slitlets: {}'.format(list_not_useful_slitlets))
tempwidths = np.array([
csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets
])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave,
sp_median,
'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum',
show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave,
catlines_reference_flux,
'C4-',
markerfmt=' ',
basefmt='C4-',
label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference,
sp_reference,
'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset * cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot)
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i] * cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1 - offset_array[i] * cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(
stat_summary['robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot,
yplot,
linestyle='',
marker='o',
color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False,
**{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset,
linestyle='--',
color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser(
description='description: overplot boundary model over FITS image')
# positional arguments
parser.add_argument("fitsfile",
help="FITS file name to be displayed",
type=argparse.FileType('rb'))
parser.add_argument("--rect_wpoly_MOSlibrary",
required=True,
help="Input JSON file with library of rectification "
"and wavelength calibration coefficients",
type=argparse.FileType('rt'))
# optional arguments
parser.add_argument("--global_integer_offset_x_pix",
help="Global integer offset in the X direction "
"(default=0)",
default=0,
type=int)
parser.add_argument("--global_integer_offset_y_pix",
help="Global integer offset in the Y direction "
"(default=0)",
default=0,
type=int)
parser.add_argument("--arc_lines",
help="Overplot arc lines",
action="store_true")
parser.add_argument("--oh_lines",
help="Overplot OH lines",
action="store_true")
parser.add_argument("--ds9_frontiers",
help="Output ds9 region file with slitlet frontiers",
type=lambda x: arg_file_is_new(parser, x))
parser.add_argument("--ds9_boundaries",
help="Output ds9 region file with slitlet boundaries",
type=lambda x: arg_file_is_new(parser, x))
parser.add_argument("--ds9_lines",
help="Output ds9 region file with arc/oh lines",
type=lambda x: arg_file_is_new(parser, x))
parser.add_argument("--debugplot",
help="Integer indicating plotting/debugging" +
" (default=12)",
type=int,
default=12,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args()
if args.echo:
print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n')
# ---
# avoid incompatible options
if args.arc_lines and args.oh_lines:
raise ValueError("--arc_lines and --oh_lines cannot be used "
"simultaneously")
# --ds9_lines requires --arc_lines or --oh_lines
if args.ds9_lines:
if not (args.arc_lines or args.oh_lines):
raise ValueError("--ds9_lines requires the use of either "
"--arc_lines or --oh_lines")
# read input FITS file
hdulist = fits.open(args.fitsfile)
image_header = hdulist[0].header
image2d = hdulist[0].data
hdulist.close()
naxis1 = image_header['naxis1']
naxis2 = image_header['naxis2']
if image2d.shape != (naxis2, naxis1):
raise ValueError("Unexpected error with NAXIS1, NAXIS2")
if image2d.shape != (EMIR_NAXIS2, EMIR_NAXIS1):
raise ValueError("Unexpected values for NAXIS1, NAXIS2")
# remove path from fitsfile
sfitsfile = os.path.basename(args.fitsfile.name)
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# read GRISM, FILTER and ROTANG from FITS header
grism = image_header['grism']
spfilter = image_header['filter']
rotang = image_header['rotang']
# ---
# generate MasterRectWave object
master_rectwv = MasterRectWave._datatype_load(
args.rect_wpoly_MOSlibrary.name)
# check that grism and filter are the expected ones
grism_ = master_rectwv.tags['grism']
if grism_ != grism:
raise ValueError('Unexpected grism: ' + str(grism_))
spfilter_ = master_rectwv.tags['filter']
if spfilter_ != spfilter:
raise ValueError('Unexpected filter ' + str(spfilter_))
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in master_rectwv.missing_slitlets:
list_valid_islitlets.remove(idel)
# read CsuConfiguration object from FITS file
csu_config = CsuConfiguration.define_from_fits(args.fitsfile)
# list with csu_bar_slit_center for valid slitlets
list_csu_bar_slit_center = []
for islitlet in list_valid_islitlets:
list_csu_bar_slit_center.append(
csu_config.csu_bar_slit_center(islitlet))
# define parmodel and params
fitted_bound_param_json = {
'contents': master_rectwv.meta_info['refined_boundary_model']
}
parmodel = fitted_bound_param_json['contents']['parmodel']
fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}})
params = bound_params_from_dict(fitted_bound_param_json)
if parmodel != "multislit":
raise ValueError('parmodel = "multislit" not found')
# ---
# define lines to be overplotted
if args.arc_lines or args.oh_lines:
rectwv_coeff = rectwv_coeff_from_mos_library(hdulist, master_rectwv)
rectwv_coeff.global_integer_offset_x_pix = \
args.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
args.global_integer_offset_y_pix
# rectwv_coeff.writeto('xxx.json')
if args.arc_lines:
if grism == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs',
catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
elif args.oh_lines:
dumdata = pkgutil.get_data('emirdrp.instrument.configs',
'Oliva_etal_2013.dat')
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:,
0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:,
2]))
else:
raise ValueError("This should not happen!")
else:
rectwv_coeff = None
catlines_all_wave = None
catlines_all_flux = None
# ---
# generate output ds9 region file with slitlet boundaries
if args.ds9_boundaries is not None:
save_boundaries_from_params_ds9(
params=params,
parmodel=parmodel,
list_islitlet=list_valid_islitlets,
list_csu_bar_slit_center=list_csu_bar_slit_center,
uuid=master_rectwv.uuid,
grism=grism,
spfilter=spfilter,
ds9_filename=args.ds9_boundaries.name,
global_offset_x_pix=-args.global_integer_offset_x_pix,
global_offset_y_pix=-args.global_integer_offset_y_pix)
# generate output ds9 region file with slitlet frontiers
if args.ds9_frontiers is not None:
save_frontiers_from_params_ds9(
params=params,
parmodel=parmodel,
list_islitlet=list_valid_islitlets,
list_csu_bar_slit_center=list_csu_bar_slit_center,
uuid=master_rectwv.uuid,
grism=grism,
spfilter=spfilter,
ds9_filename=args.ds9_frontiers.name,
global_offset_x_pix=-args.global_integer_offset_x_pix,
global_offset_y_pix=-args.global_integer_offset_y_pix)
# ---
# display full image
if abs(args.debugplot) % 10 != 0:
ax = ximshow(image2d=image2d,
title=sfitsfile + "\ngrism=" + grism + ", filter=" +
spfilter + ", rotang=" + str(round(rotang, 2)),
image_bbox=(1, naxis1, 1, naxis2),
show=False)
# overplot boundaries
overplot_boundaries_from_params(
ax=ax,
params=params,
parmodel=parmodel,
list_islitlet=list_valid_islitlets,
list_csu_bar_slit_center=list_csu_bar_slit_center,
global_offset_x_pix=-args.global_integer_offset_x_pix,
global_offset_y_pix=-args.global_integer_offset_y_pix)
# overplot frontiers
overplot_frontiers_from_params(
ax=ax,
params=params,
parmodel=parmodel,
list_islitlet=list_valid_islitlets,
list_csu_bar_slit_center=list_csu_bar_slit_center,
micolors=('b', 'b'),
linetype='-',
labels=False, # already displayed with the boundaries
global_offset_x_pix=-args.global_integer_offset_x_pix,
global_offset_y_pix=-args.global_integer_offset_y_pix)
else:
ax = None
# overplot lines
if catlines_all_wave is not None:
if args.ds9_lines is None:
ds9_file = None
else:
ds9_file = open(args.ds9_lines.name, 'w')
ds9_file.write('# Region file format: DS9 version 4.1\n')
ds9_file.write('global color=#00ffff dashlist=0 0 width=2 '
'font="helvetica 10 normal roman" select=1 '
'highlite=1 dash=0 fixed=0 edit=1 '
'move=1 delete=1 include=1 source=1\n')
ds9_file.write('physical\n#\n')
ds9_file.write('#\n# uuid..: {0}\n'.format(master_rectwv.uuid))
ds9_file.write('# filter: {0}\n'.format(spfilter))
ds9_file.write('# grism.: {0}\n'.format(grism))
ds9_file.write('#\n# global_offset_x_pix: {0}\n'.format(
args.global_integer_offset_x_pix))
ds9_file.write('# global_offset_y_pix: {0}\n#\n'.format(
args.global_integer_offset_y_pix))
if parmodel == "longslit":
for dumpar in EXPECTED_PARAMETER_LIST:
parvalue = params[dumpar].value
ds9_file.write('# {0}: {1}\n'.format(dumpar, parvalue))
else:
for dumpar in EXPECTED_PARAMETER_LIST_EXTENDED:
parvalue = params[dumpar].value
ds9_file.write('# {0}: {1}\n'.format(dumpar, parvalue))
overplot_lines(ax, catlines_all_wave, list_valid_islitlets,
rectwv_coeff, args.global_integer_offset_x_pix,
args.global_integer_offset_y_pix, ds9_file,
args.debugplot)
if ds9_file is not None:
ds9_file.close()
if ax is not None:
# show plot
pause_debugplot(12, pltshow=True)
def display_slitlet_arrangement(fileobj,
grism=None,
spfilter=None,
bbox=None,
adjust=None,
geometry=None,
debugplot=0):
"""Display slitlet arrangment from CSUP keywords in FITS header.
Parameters
----------
fileobj : file object
FITS or TXT file object.
grism : str
Grism.
grism : str
Filter.
bbox : tuple of 4 floats
If not None, values for xmin, xmax, ymin and ymax.
adjust : bool
Adjust X range according to minimum and maximum csu_bar_left
and csu_bar_right (note that this option is overriden by 'bbox')
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
csu_bar_left : list of floats
Location (mm) of the left bar for each slitlet.
csu_bar_right : list of floats
Location (mm) of the right bar for each slitlet, using the
same origin employed for csu_bar_left (which is not the
value stored in the FITS keywords.
csu_bar_slit_center : list of floats
Middle point (mm) in between the two bars defining a slitlet.
csu_bar_slit_width : list of floats
Slitlet width (mm), computed as the distance between the two
bars defining the slitlet.
"""
if fileobj.name[-4:] == ".txt":
if grism is None:
raise ValueError("Undefined grism!")
if spfilter is None:
raise ValueError("Undefined filter!")
# define CsuConfiguration object
csu_config = CsuConfiguration()
csu_config._csu_bar_left = []
csu_config._csu_bar_right = []
csu_config._csu_bar_slit_center = []
csu_config._csu_bar_slit_width = []
# since the input filename has been opened with argparse in binary
# mode, it is necessary to close it and open it in text mode
fileobj.close()
# read TXT file
with open(fileobj.name, mode='rt') as f:
file_content = f.read().splitlines()
next_id_bar = 1
for line in file_content:
if len(line) > 0:
if line[0] not in ['#']:
line_contents = line.split()
id_bar = int(line_contents[0])
position = float(line_contents[1])
if id_bar == next_id_bar:
if id_bar <= EMIR_NBARS:
csu_config._csu_bar_left.append(position)
next_id_bar = id_bar + EMIR_NBARS
else:
csu_config._csu_bar_right.append(341.5 - position)
next_id_bar = id_bar - EMIR_NBARS + 1
else:
raise ValueError("Unexpected id_bar:" + str(id_bar))
# compute slit width and center
for i in range(EMIR_NBARS):
csu_config._csu_bar_slit_center.append(
(csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i])/2
)
csu_config._csu_bar_slit_width.append(
csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i]
)
else:
# read input FITS file
hdulist = fits.open(fileobj.name)
image_header = hdulist[0].header
hdulist.close()
# additional info from header
grism = image_header['grism']
spfilter = image_header['filter']
# define slitlet arrangement
csu_config = CsuConfiguration.define_from_fits(fileobj)
# determine calibration
if grism in ["J", "OPEN"] and spfilter == "J":
wv_parameters = set_wv_parameters("J", "J")
elif grism in ["H", "OPEN"] and spfilter == "H":
wv_parameters = set_wv_parameters("H", "H")
elif grism in ["K", "OPEN"] and spfilter == "Ksp":
wv_parameters = set_wv_parameters("Ksp", "K")
elif grism in ["LR", "OPEN"] and spfilter == "YJ":
wv_parameters = set_wv_parameters("YJ", "LR")
elif grism in ["LR", "OPEN"] and spfilter == "HK":
wv_parameters = set_wv_parameters("HK", "LR")
else:
raise ValueError("Invalid grism + filter configuration")
crval1 = wv_parameters['poly_crval1_linear']
cdelt1 = wv_parameters['poly_cdelt1_linear']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# display arrangement
if abs(debugplot) >= 10:
print("slit left right center width min.wave max.wave")
print("==== ======= ======= ======= ===== ======== ========")
for i in range(EMIR_NBARS):
ibar = i + 1
csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar))
csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar))
csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1
if wvmin_useful is not None:
csu_crval1 = np.amax([csu_crval1, wvmin_useful])
if wvmax_useful is not None:
csu_crvaln = np.amin([csu_crvaln, wvmax_useful])
print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} "
"{5:8.2f} {6:8.2f}".format(
ibar, csu_config.csu_bar_left(ibar),
csu_config.csu_bar_right(ibar),
csu_config.csu_bar_slit_center(ibar),
csu_config.csu_bar_slit_width(ibar),
csu_crval1, csu_crvaln)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format(
np.mean(csu_config._csu_bar_left),
np.mean(csu_config._csu_bar_right),
np.mean(csu_config._csu_bar_slit_center),
np.mean(csu_config._csu_bar_slit_width)
)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format(
np.mean(csu_config._csu_bar_left[::2]),
np.mean(csu_config._csu_bar_right[::2]),
np.mean(csu_config._csu_bar_slit_center[::2]),
np.mean(csu_config._csu_bar_slit_width[::2])
)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format(
np.mean(csu_config._csu_bar_left[1::2]),
np.mean(csu_config._csu_bar_right[1::2]),
np.mean(csu_config._csu_bar_slit_center[1::2]),
np.mean(csu_config._csu_bar_slit_width[1::2])
)
)
# display slit arrangement
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
if bbox is None:
if adjust:
xmin = min(csu_config._csu_bar_left)
xmax = max(csu_config._csu_bar_right)
dx = xmax - xmin
if dx == 0:
dx = 1
xmin -= dx/20
xmax += dx/20
ax.set_xlim(xmin, xmax)
else:
ax.set_xlim(0., 341.5)
ax.set_ylim(0, 56)
else:
ax.set_xlim(bbox[0], bbox[1])
ax.set_ylim(bbox[2], bbox[3])
ax.set_xlabel('csu_bar_position (mm)')
ax.set_ylabel('slit number')
for i in range(EMIR_NBARS):
ibar = i + 1
ax.add_patch(patches.Rectangle(
(csu_config.csu_bar_left(ibar), ibar-0.5),
csu_config.csu_bar_slit_width(ibar), 1.0))
ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar],
'-', color='gray')
ax.plot([csu_config.csu_bar_right(ibar), 341.5],
[ibar, ibar], '-', color='gray')
plt.title("File: " + fileobj.name + "\ngrism=" + grism +
", filter=" + spfilter)
pause_debugplot(debugplot, pltshow=True)
# return results
return csu_config._csu_bar_left, csu_config._csu_bar_right, \
csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser(
description='description: compute pixel-to-pixel flatfield'
)
# required arguments
parser.add_argument("fitsfile",
help="Input FITS file (flat ON-OFF)",
type=argparse.FileType('rb'))
parser.add_argument("--rectwv_coeff", required=True,
help="Input JSON file with rectification and "
"wavelength calibration coefficients",
type=argparse.FileType('rt'))
parser.add_argument("--minimum_slitlet_width_mm", required=True,
help="Minimum slitlet width in mm",
type=float)
parser.add_argument("--maximum_slitlet_width_mm", required=True,
help="Maximum slitlet width in mm",
type=float)
parser.add_argument("--minimum_fraction", required=True,
help="Minimum allowed flatfielding value",
type=float, default=0.01)
parser.add_argument("--minimum_value_in_output",
help="Minimum value allowed in output file: pixels "
"below this value are set to 1.0 (default=0.01)",
type=float, default=0.01)
parser.add_argument("--maximum_value_in_output",
help="Maximum value allowed in output file: pixels "
"above this value are set to 1.0 (default=10.0)",
type=float, default=10.0)
parser.add_argument("--nwindow_median",
help="Window size to smooth median spectrum in the "
"spectral direction",
type=int)
parser.add_argument("--outfile", required=True,
help="Output FITS file",
type=lambda x: arg_file_is_new(parser, x, mode='wb'))
# optional arguments
parser.add_argument("--delta_global_integer_offset_x_pix",
help="Delta global integer offset in the X direction "
"(default=0)",
default=0, type=int)
parser.add_argument("--delta_global_integer_offset_y_pix",
help="Delta global integer offset in the Y direction "
"(default=0)",
default=0, type=int)
parser.add_argument("--resampling",
help="Resampling method: 1 -> nearest neighbor, "
"2 -> linear interpolation (default)",
default=2, type=int,
choices=(1, 2))
parser.add_argument("--ignore_DTUconf",
help="Ignore DTU configurations differences between "
"model and input image",
action="store_true")
parser.add_argument("--debugplot",
help="Integer indicating plotting & debugging options"
" (default=0)",
default=0, type=int,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args(args)
if args.echo:
print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n')
# This code is obsolete
raise ValueError('This code is obsolete: use recipe in '
'emirdrp/recipes/spec/flatpix2pix.py')
# read calibration structure from JSON file
rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name)
# modify (when requested) global offsets
rectwv_coeff.global_integer_offset_x_pix += \
args.delta_global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix += \
args.delta_global_integer_offset_y_pix
# read FITS image and its corresponding header
hdulist = fits.open(args.fitsfile)
header = hdulist[0].header
image2d = hdulist[0].data
hdulist.close()
# apply global offsets
image2d = apply_integer_offsets(
image2d=image2d,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix
)
# protections
naxis2, naxis1 = image2d.shape
if naxis1 != header['naxis1'] or naxis2 != header['naxis2']:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2')
if abs(args.debugplot) >= 10:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
# check that the input FITS file grism and filter match
filter_name = header['filter']
if filter_name != rectwv_coeff.tags['filter']:
raise ValueError("Filter name does not match!")
grism_name = header['grism']
if grism_name != rectwv_coeff.tags['grism']:
raise ValueError("Filter name does not match!")
if abs(args.debugplot) >= 10:
print('>>> grism.......:', grism_name)
print('>>> filter......:', filter_name)
# check that the DTU configurations are compatible
dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile)
dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary(
rectwv_coeff.meta_info['dtu_configuration'])
if dtu_conf_fitsfile != dtu_conf_jsonfile:
print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile)
print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile)
if args.ignore_DTUconf:
print('WARNING: DTU configuration differences found!')
else:
raise ValueError('DTU configurations do not match')
else:
if abs(args.debugplot) >= 10:
print('>>> DTU Configuration match!')
print(dtu_conf_fitsfile)
# load CSU configuration
csu_conf_fitsfile = CsuConfiguration.define_from_fits(args.fitsfile)
if abs(args.debugplot) >= 10:
print(csu_conf_fitsfile)
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
print('-> Removing slitlet (not defined):', idel)
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
if (slitwidth < args.minimum_slitlet_width_mm) or \
(slitwidth > args.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
print('-> Removing slitlet (invalid width):', islitlet)
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
print('>>> valid slitlet numbers:\n', list_valid_islitlets)
# ---
# compute and store median spectrum (and masked region) for each
# individual slitlet
image2d_sp_median = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
image2d_sp_mask = np.zeros((EMIR_NBARS, EMIR_NAXIS1), dtype=bool)
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=False)
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=args.debugplot)
if abs(args.debugplot) >= 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(
image_2k2k=image2d,
subtitle='original image'
)
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=args.resampling,
subtitle='original rectified'
)
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
sp_mask = np.zeros(naxis1_slitlet2d, dtype=bool)
# for grism LR set to zero data beyond useful wavelength range
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
x_pix = np.arange(1, naxis1_slitlet2d + 1)
wl_pix = polyval(x_pix, slt.wpoly)
lremove = wl_pix < wv_parameters['wvmin_useful']
sp_mask[lremove] = True
slitlet2d_rect[:, lremove] = 0.0
lremove = wl_pix > wv_parameters['wvmax_useful']
slitlet2d_rect[:, lremove] = 0.0
sp_mask[lremove] = True
# get useful slitlet region (use boundaries instead of frontiers;
# note that the nscan_minmax_frontiers() works well independently
# of using frontiers of boundaries as arguments)
nscan_min, nscan_max = nscan_minmax_frontiers(
slt.y0_reference_lower,
slt.y0_reference_upper,
resize=False
)
ii1 = nscan_min - slt.bb_ns1_orig
ii2 = nscan_max - slt.bb_ns1_orig + 1
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0)
# smooth median spectrum along the spectral direction
sp_median = ndimage.median_filter(
sp_collapsed,
args.nwindow_median,
mode='nearest'
)
"""
nremove = 5
spl = AdaptiveLSQUnivariateSpline(
x=xaxis1[nremove:-nremove],
y=sp_collapsed[nremove:-nremove],
t=11,
adaptive=True
)
xknots = spl.get_knots()
yknots = spl(xknots)
sp_median = spl(xaxis1)
# compute rms within each knot interval
nknots = len(xknots)
rms_array = np.zeros(nknots - 1, dtype=float)
for iknot in range(nknots - 1):
residuals = []
for xdum, ydum, yydum in \
zip(xaxis1, sp_collapsed, sp_median):
if xknots[iknot] <= xdum <= xknots[iknot + 1]:
residuals.append(abs(ydum - yydum))
if len(residuals) > 5:
rms_array[iknot] = np.std(residuals)
else:
rms_array[iknot] = 0
# determine in which knot interval falls each pixel
iknot_array = np.zeros(len(xaxis1), dtype=int)
for idum, xdum in enumerate(xaxis1):
for iknot in range(nknots - 1):
if xknots[iknot] <= xdum <= xknots[iknot + 1]:
iknot_array[idum] = iknot
# compute new fit removing deviant points (with fixed knots)
xnewfit = []
ynewfit = []
for idum in range(len(xaxis1)):
delta_sp = abs(sp_collapsed[idum] - sp_median[idum])
rms_tmp = rms_array[iknot_array[idum]]
if idum == 0 or idum == (len(xaxis1) - 1):
lok = True
elif rms_tmp > 0:
if delta_sp < 3.0 * rms_tmp:
lok = True
else:
lok = False
else:
lok = True
if lok:
xnewfit.append(xaxis1[idum])
ynewfit.append(sp_collapsed[idum])
nremove = 5
splnew = AdaptiveLSQUnivariateSpline(
x=xnewfit[nremove:-nremove],
y=ynewfit[nremove:-nremove],
t=xknots[1:-1],
adaptive=False
)
sp_median = splnew(xaxis1)
"""
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * args.minimum_fraction
lremove = np.where(sp_median < y_threshold)
sp_median[lremove] = 0.0
sp_mask[lremove] = True
image2d_sp_median[islitlet - 1, :] = sp_median
image2d_sp_mask[islitlet - 1, :] = sp_mask
if abs(args.debugplot) % 10 != 0:
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1, sp_collapsed,
title=title,
show=False, **{'label' : 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d], 2*[y_threshold],
label='threshold')
# ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05*ymax_spmedian, 1.05*ymax_spmedian)
pause_debugplot(args.debugplot,
pltshow=True, tight_layout=True)
else:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=True)
# ToDo: compute "average" spectrum for each pseudo-longslit, scaling
# with the median signal in each slitlet; derive a particular
# spectrum for each slitlet (scaling properly)
image2d_sp_median_masked = np.ma.masked_array(
image2d_sp_median,
mask=image2d_sp_mask
)
ycut_median = np.ma.median(image2d_sp_median_masked, axis=1).data
ycut_median_2d = np.repeat(ycut_median, EMIR_NAXIS1).reshape(
EMIR_NBARS, EMIR_NAXIS1)
image2d_sp_median_eq = image2d_sp_median_masked / ycut_median_2d
image2d_sp_median_eq = image2d_sp_median_eq.data
if True:
ximshow(image2d_sp_median, title='sp_median', debugplot=12)
ximplotxy(np.arange(1, EMIR_NBARS + 1), ycut_median, 'ro',
title='median value of each spectrum', debugplot=12)
ximshow(image2d_sp_median_eq, title='sp_median_eq', debugplot=12)
csu_conf_fitsfile.display_pseudo_longslits(
list_valid_slitlets=list_valid_islitlets)
dict_longslits = csu_conf_fitsfile.pseudo_longslits()
# compute median spectrum for each longslit and insert (properly
# scaled) that spectrum in each slitlet belonging to that longslit
image2d_sp_median_longslit = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
islitlet = 1
loop = True
while loop:
if islitlet in list_valid_islitlets:
imin = dict_longslits[islitlet].imin()
imax = dict_longslits[islitlet].imax()
print('--> imin, imax: ', imin, imax)
sp_median_longslit = np.median(
image2d_sp_median_eq[(imin - 1):imax, :], axis=0)
for i in range(imin, imax+1):
print('----> i: ', i)
image2d_sp_median_longslit[(i - 1), :] = \
sp_median_longslit * ycut_median[i - 1]
islitlet = imax
else:
print('--> ignoring: ', islitlet)
if islitlet == EMIR_NBARS:
loop = False
else:
islitlet += 1
if True:
ximshow(image2d_sp_median_longslit, debugplot=12)
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=False)
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=args.debugplot)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(
image_2k2k=image2d,
subtitle='original image'
)
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=args.resampling,
subtitle='original rectified'
)
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
sp_median = image2d_sp_median_longslit[islitlet - 1, :]
# generate rectified slitlet region filled with the median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1))
if abs(args.debugplot) > 10:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum'
)
# unrectified image
slitlet2d_unrect_spmedian = slt.rectify(
slitlet2d=slitlet2d_rect_spmedian,
resampling=args.resampling,
inverse=True,
subtitle='unrectified, filled with median spectrum'
)
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_spmedian[i, j]
if den == 0:
slitlet2d_norm[i, j] = 1.0
else:
slitlet2d_norm[i, j] = slitlet2d[i, j] / den
if abs(args.debugplot) > 10:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm,
subtitle='unrectified, pixel-to-pixel'
)
# check for pseudo-longslit with previous slitlet
if islitlet > 1:
if (islitlet - 1) in list_valid_islitlets:
c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet - 1)
w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet - 1)
c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet)
w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
if abs(w1-w2)/w1 < 0.25:
wmean = (w1 + w2) / 2.0
if abs(c1 - c2) < wmean/4.0:
same_slitlet_below = True
else:
same_slitlet_below = False
else:
same_slitlet_below = False
else:
same_slitlet_below = False
else:
same_slitlet_below = False
# check for pseudo-longslit with next slitlet
if islitlet < EMIR_NBARS:
if (islitlet + 1) in list_valid_islitlets:
c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet)
w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet + 1)
w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet + 1)
if abs(w1-w2)/w1 < 0.25:
wmean = (w1 + w2) / 2.0
if abs(c1 - c2) < wmean/4.0:
same_slitlet_above = True
else:
same_slitlet_above = False
else:
same_slitlet_above = False
else:
same_slitlet_above = False
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
else:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=True)
if args.debugplot == 0:
print('OK!')
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded ,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix
)
# set pixels below minimum value to 1.0
filtered = np.where(image2d_flatfielded < args.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(image2d_flatfielded > args.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# save output file
save_ndarray_to_fits(
array=image2d_flatfielded,
file_name=args.outfile,
main_header=header,
overwrite=True
)
print('>>> Saving file ' + args.outfile.name)
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser()
# positional arguments
parser.add_argument("fitsfile",
help="FITS file name to be displayed",
type=argparse.FileType('rb'))
parser.add_argument("--fitted_bound_param", required=True,
help="JSON file with fitted boundary coefficients "
"corresponding to the multislit model",
type=argparse.FileType('rt'))
parser.add_argument("--slitlets", required=True,
help="Slitlet selection: string between double "
"quotes providing tuples of the form "
"n1[,n2[,step]]",
type=str)
# optional arguments
parser.add_argument("--outfile",
help="Output FITS file name",
type=lambda x: arg_file_is_new(parser, x, mode='wb'))
parser.add_argument("--maskonly",
help="Generate mask for the indicated slitlets",
action="store_true")
parser.add_argument("--debugplot",
help="Integer indicating plotting/debugging" +
" (default=0)",
type=int, default=0,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args()
if args.echo:
print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n')
# read input FITS file
hdulist_image = fits.open(args.fitsfile.name)
image_header = hdulist_image[0].header
image2d = hdulist_image[0].data
naxis1 = image_header['naxis1']
naxis2 = image_header['naxis2']
if image2d.shape != (naxis2, naxis1):
raise ValueError("Unexpected error with NAXIS1, NAXIS2")
if image2d.shape != (EMIR_NAXIS2, EMIR_NAXIS1):
raise ValueError("NAXIS1, NAXIS2 unexpected for EMIR detector")
# remove path from fitsfile
if args.outfile is None:
sfitsfile = os.path.basename(args.fitsfile.name)
else:
sfitsfile = os.path.basename(args.outfile.name)
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# read GRISM, FILTER and ROTANG from FITS header
grism = image_header['grism']
spfilter = image_header['filter']
rotang = image_header['rotang']
# read fitted_bound_param JSON file
fittedpar_dict = json.loads(open(args.fitted_bound_param.name).read())
params = bound_params_from_dict(fittedpar_dict)
if abs(args.debugplot) in [21, 22]:
params.pretty_print()
parmodel = fittedpar_dict['meta_info']['parmodel']
if parmodel != 'multislit':
raise ValueError("Unexpected parameter model: ", parmodel)
# define slitlet range
islitlet_min = fittedpar_dict['tags']['islitlet_min']
islitlet_max = fittedpar_dict['tags']['islitlet_max']
list_islitlet = list_slitlets_from_string(
s=args.slitlets,
islitlet_min=islitlet_min,
islitlet_max=islitlet_max
)
# read CsuConfiguration object from FITS file
csu_config = CsuConfiguration.define_from_fits(args.fitsfile)
# define csu_bar_slit_center associated to each slitlet
list_csu_bar_slit_center = []
for islitlet in list_islitlet:
list_csu_bar_slit_center.append(
csu_config.csu_bar_slit_center(islitlet))
# initialize output data array
image2d_output = np.zeros((naxis2, naxis1))
# main loop
for islitlet, csu_bar_slit_center in \
zip(list_islitlet, list_csu_bar_slit_center):
image2d_tmp = select_unrectified_slitlet(
image2d=image2d,
islitlet=islitlet,
csu_bar_slit_center=csu_bar_slit_center,
params=params,
parmodel=parmodel,
maskonly=args.maskonly
)
image2d_output += image2d_tmp
# update the array of the output file
hdulist_image[0].data = image2d_output
# save output FITS file
hdulist_image.writeto(args.outfile)
# close original image
hdulist_image.close()
# display full image
if abs(args.debugplot) % 10 != 0:
ax = ximshow(image2d=image2d_output,
title=sfitsfile + "\n" + args.slitlets,
image_bbox=(1, naxis1, 1, naxis2), show=False)
# overplot boundaries
overplot_boundaries_from_params(
ax=ax,
params=params,
parmodel=parmodel,
list_islitlet=list_islitlet,
list_csu_bar_slit_center=list_csu_bar_slit_center
)
# overplot frontiers
overplot_frontiers_from_params(
ax=ax,
params=params,
parmodel=parmodel,
list_islitlet=list_islitlet,
list_csu_bar_slit_center=list_csu_bar_slit_center,
micolors=('b', 'b'), linetype='-',
labels=False # already displayed with the boundaries
)
# show plot
pause_debugplot(12, pltshow=True)
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
self.logger.info(
'Wavelength calibration refinement mode....: {}'.format(
rinput.refine_wavecalib_mode))
self.logger.info(
'Minimum slitlet width (mm)................: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info(
'Maximum slitlet width (mm)................: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info(
'Global integer offsets mode...............: {}'.format(
rinput.global_integer_offsets_mode))
self.logger.info(
'Global integer offset X direction (pixels): {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info(
'Global integer offset Y direction (pixels): {}'.format(
rinput.global_integer_offset_y_pix))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=sigmaclip)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image,
rinput.master_rectwv)
# wavelength calibration refinement
# 0 -> no refinement
# 1 -> apply global offset to all the slitlets (using ARC lines)
# 2 -> apply individual offset to each slitlet (using ARC lines)
# 11 -> apply global offset to all the slitlets (using OH lines)
# 12 -> apply individual offset to each slitlet (using OH lines)
if rinput.refine_wavecalib_mode != 0:
main_header = reduced_image[0].header
# determine useful slitlets
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=rinput.minimum_slitlet_width_mm,
maxwidth=rinput.maximum_slitlet_width_mm)
# remove missing slitlets
if len(rectwv_coeff.missing_slitlets) > 0:
for iremove in rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
self.logger.info(
'list of useful slitlets: {}'.format(list_useful_slitlets))
self.logger.info('list of unusable slitlets: {}'.format(
list_not_useful_slitlets))
# retrieve arc/OH lines
catlines_all_wave, catlines_all_flux = retrieve_catlines(
rinput.refine_wavecalib_mode, main_header['grism'])
# global integer offsets
if rinput.global_integer_offsets_mode == 'auto':
if (rinput.global_integer_offset_x_pix != 0) or \
(rinput.global_integer_offset_y_pix != 0):
raise ValueError('Global integer offsets must be zero when'
' mode=auto')
# ToDo: include additional airglow emission lines
self.logger.info('computing synthetic image')
# generate synthetic image
synthetic_raw_data = synthetic_lines_rawdata(
catlines_all_wave, catlines_all_flux, list_useful_slitlets,
rectwv_coeff)
synthetic_raw_header = main_header.copy()
synthetic_raw_header['DATE-OBS'] = \
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
chistory = 'Synthetic image'
synthetic_raw_header.add_history(chistory)
hdu = fits.PrimaryHDU(synthetic_raw_data.astype('float32'),
header=synthetic_raw_header)
synthetic_raw_image = fits.HDUList([hdu])
if self.intermediate_results:
self.save_intermediate_img(synthetic_raw_image,
'synthetic_raw_image.fits')
# cross-correlation to determine global integer offsets
# (rescaling data arrays to [0, 1] before using skimage
# function)
data1_rs, coef1_rs = rescale_array_to_z1z2(
reduced_image[0].data, (0, 1))
data2_rs, coef2_rs = rescale_array_to_z1z2(
synthetic_raw_data, (0, 1))
shifts, error, diffphase = register_translation(
data1_rs, data2_rs, 100)
self.logger.info(
'global_float_offset_x_pix..: {}'.format(-shifts[1]))
self.logger.info(
'global_float_offset_y_pix..: {}'.format(-shifts[0]))
rectwv_coeff.global_integer_offset_x_pix = \
-int(round(shifts[1]))
rectwv_coeff.global_integer_offset_y_pix = \
-int(round(shifts[0]))
self.logger.info('global_integer_offset_x_pix: {}'.format(
rectwv_coeff.global_integer_offset_x_pix))
self.logger.info('global_integer_offset_y_pix: {}'.format(
rectwv_coeff.global_integer_offset_y_pix))
if self.intermediate_results:
data_product = np.fft.fft2(data1_rs) * \
np.fft.fft2(data2_rs).conj()
cc_image = np.fft.fftshift(np.fft.ifft2(data_product))
power = np.log10(cc_image.real)
hdu_power = fits.PrimaryHDU(power)
hdul_power = fits.HDUList([hdu_power])
hdul_power.writeto('power.fits', overwrite=True)
else:
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
# apply initial rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
self.logger.info(
'Refining wavelength calibration (mode={})'.format(
rinput.refine_wavecalib_mode))
# refine RectWaveCoeff object
rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff(
reduced_mos,
rectwv_coeff,
catlines_all_wave,
catlines_all_flux,
rinput.refine_wavecalib_mode,
list_useful_slitlets,
save_intermediate_results=self.intermediate_results)
self.save_intermediate_img(expected_catalog_lines,
'expected_catalog_lines.fits')
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos,
rectwv_coeff=rectwv_coeff)
return result
def rectwv_coeff_from_arc_image(reduced_image,
bound_param,
lines_catalog,
args_nbrightlines=None,
args_ymargin_bb=2,
args_remove_sp_background=True,
args_times_sigma_threshold=10,
args_order_fmap=2,
args_sigma_gaussian_filtering=2,
args_margin_npix=50,
args_poldeg_initial=3,
args_poldeg_refined=5,
args_interactive=False,
args_threshold_wv=0,
args_ylogscale=False,
args_pdf=None,
args_geometry=(0, 0, 640, 480),
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from arc image
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
bound_param : RefinedBoundaryModelParam instance
Refined boundary model.
lines_catalog : Numpy array
2D numpy array with the contents of the master file with the
expected arc line wavelengths.
args_nbrightlines : int
TBD
args_ymargin_bb : int
TBD
args_remove_sp_background : bool
TBD
args_times_sigma_threshold : float
TBD
args_order_fmap : int
TBD
args_sigma_gaussian_filtering : float
TBD
args_margin_npix : int
TBD
args_poldeg_initial : int
TBD
args_poldeg_refined : int
TBD
args_interactive : bool
TBD
args_threshold_wv : float
TBD
args_ylogscale : bool
TBD
args_pdf : TBD
args_geometry : TBD
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration of the input arc image.
reduced_55sp : HDUList object
Image with 55 spectra corresponding to the median spectrum for
each slitlet, employed to derived the wavelength calibration
polynomial.
"""
logger = logging.getLogger(__name__)
# protections
if args_interactive and args_pdf is not None:
logger.error('--interactive and --pdf are incompatible options')
raise ValueError('--interactive and --pdf are incompatible options')
# header and data array
header = reduced_image[0].header
image2d = reduced_image[0].data
# check grism and filter
filter_name = header['filter']
logger.info('Filter: ' + filter_name)
if filter_name != bound_param.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.info('Grism: ' + grism_name)
if grism_name != bound_param.tags['grism']:
raise ValueError('Grism name does not match!')
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
logger.debug(csu_conf)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
logger.debug(dtu_conf)
# set boundary parameters
parmodel = bound_param.meta_info['parmodel']
params = bound_params_from_dict(bound_param.__getstate__())
if abs(debugplot) >= 10:
print('-' * 83)
print('* FITTED BOUND PARAMETERS')
params.pretty_print()
pause_debugplot(debugplot)
# determine parameters according to grism+filter combination
wv_parameters = set_wv_parameters(filter_name, grism_name)
islitlet_min = wv_parameters['islitlet_min']
islitlet_max = wv_parameters['islitlet_max']
if args_nbrightlines is None:
nbrightlines = wv_parameters['nbrightlines']
else:
nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')]
poly_crval1_linear = wv_parameters['poly_crval1_linear']
poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear']
wvmin_expected = wv_parameters['wvmin_expected']
wvmax_expected = wv_parameters['wvmax_expected']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# list of slitlets to be computed
logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' +
str(islitlet_max) + ']')
# read master arc line wavelengths (only brightest lines)
wv_master = read_wv_master_from_array(master_table=lines_catalog,
lines='brightest',
debugplot=debugplot)
# read master arc line wavelengths (whole data set)
wv_master_all = read_wv_master_from_array(master_table=lines_catalog,
lines='all',
debugplot=debugplot)
# check that the arc lines in the master file are properly sorted
# in ascending order
for i in range(len(wv_master_all) - 1):
if wv_master_all[i] >= wv_master_all[i + 1]:
logger.error('>>> wavelengths: ' + str(wv_master_all[i]) + ' ' +
str(wv_master_all[i + 1]))
raise ValueError('Arc lines are not sorted in master file')
# ---
image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
# compute rectification transformation and wavelength calibration
# polynomials
measured_slitlets = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet_min <= islitlet <= islitlet_max:
# define Slitlet2dArc object
slt = Slitlet2dArc(islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=params,
parmodel=parmodel,
debugplot=debugplot)
# extract 2D image corresponding to the selected slitlet, clipping
# the image beyond the unrectified slitlet (in order to isolate
# the arc lines of the current slitlet; otherwise there are
# problems with arc lines from neighbour slitlets)
image2d_tmp = select_unrectified_slitlet(
image2d=image2d,
islitlet=islitlet,
csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet),
params=params,
parmodel=parmodel,
maskonly=False)
slitlet2d = slt.extract_slitlet2d(image2d_tmp)
# subtract smooth background computed as follows:
# - median collapsed spectrum of the whole slitlet2d
# - independent median filtering of the previous spectrum in the
# two halves in the spectral direction
if args_remove_sp_background:
spmedian = np.median(slitlet2d, axis=0)
naxis1_tmp = spmedian.shape[0]
jmidpoint = naxis1_tmp // 2
sp1 = medfilt(spmedian[:jmidpoint], [201])
sp2 = medfilt(spmedian[jmidpoint:], [201])
spbackground = np.concatenate((sp1, sp2))
slitlet2d -= spbackground
# locate unknown arc lines
slt.locate_unknown_arc_lines(
slitlet2d=slitlet2d,
times_sigma_threshold=args_times_sigma_threshold)
# continue working with current slitlet only if arc lines have
# been detected
if slt.list_arc_lines is not None:
# compute intersections between spectrum trails and arc lines
slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d)
# compute rectification transformation
slt.estimate_tt_to_rectify(order=args_order_fmap,
slitlet2d=slitlet2d)
# rectify image
slitlet2d_rect = slt.rectify(slitlet2d,
resampling=2,
transformation=1)
# median spectrum and line peaks from rectified image
sp_median, fxpeaks = slt.median_spectrum_from_rectified_image(
slitlet2d_rect,
sigma_gaussian_filtering=args_sigma_gaussian_filtering,
nwinwidth_initial=5,
nwinwidth_refined=5,
times_sigma_threshold=5,
npix_avoid_border=6,
nbrightlines=nbrightlines)
image2d_55sp[islitlet - 1, :] = sp_median
# determine expected wavelength limits prior to the wavelength
# calibration
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
crval1_linear = poly_crval1_linear(csu_bar_slit_center)
cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center)
expected_wvmin = crval1_linear - \
args_margin_npix * cdelt1_linear
naxis1_linear = sp_median.shape[0]
crvaln_linear = crval1_linear + \
(naxis1_linear - 1) * cdelt1_linear
expected_wvmax = crvaln_linear + \
args_margin_npix * cdelt1_linear
# override previous estimates when necessary
if wvmin_expected is not None:
expected_wvmin = wvmin_expected
if wvmax_expected is not None:
expected_wvmax = wvmax_expected
# clip initial master arc line list with bright lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master
lok2 = wv_master <= expected_wvmax
lok = lok1 * lok2
wv_master_eff = wv_master[lok]
# perform initial wavelength calibration
solution_wv = wvcal_spectrum(
sp=sp_median,
fxpeaks=fxpeaks,
poly_degree_wfit=args_poldeg_initial,
wv_master=wv_master_eff,
wv_ini_search=expected_wvmin,
wv_end_search=expected_wvmax,
wvmin_useful=wvmin_useful,
wvmax_useful=wvmax_useful,
geometry=args_geometry,
debugplot=slt.debugplot)
# store initial wavelength calibration polynomial in current
# slitlet instance
slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff)
pause_debugplot(debugplot)
# clip initial master arc line list with all the lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master_all
lok2 = wv_master_all <= expected_wvmax
lok = lok1 * lok2
wv_master_all_eff = wv_master_all[lok]
# clip master arc line list to useful region
if wvmin_useful is not None:
lok = wvmin_useful <= wv_master_all_eff
wv_master_all_eff = wv_master_all_eff[lok]
if wvmax_useful is not None:
lok = wv_master_all_eff <= wvmax_useful
wv_master_all_eff = wv_master_all_eff[lok]
# refine wavelength calibration
if args_poldeg_refined > 0:
plottitle = '[slitlet#{}, refined]'.format(islitlet)
poly_refined, yres_summary = refine_arccalibration(
sp=sp_median,
poly_initial=slt.wpoly,
wv_master=wv_master_all_eff,
poldeg=args_poldeg_refined,
ntimes_match_wv=1,
interactive=args_interactive,
threshold=args_threshold_wv,
plottitle=plottitle,
ylogscale=args_ylogscale,
geometry=args_geometry,
pdf=args_pdf,
debugplot=slt.debugplot)
# store refined wavelength calibration polynomial in
# current slitlet instance
slt.wpoly = poly_refined
# compute approximate linear values for CRVAL1 and CDELT1
naxis1_linear = sp_median.shape[0]
crmin1_linear = slt.wpoly(1)
crmax1_linear = slt.wpoly(naxis1_linear)
slt.crval1_linear = crmin1_linear
slt.cdelt1_linear = \
(crmax1_linear - crmin1_linear) / (naxis1_linear - 1)
# check that the trimming of wv_master and wv_master_all has
# preserved the wavelength range [crmin1_linear, crmax1_linear]
if crmin1_linear < expected_wvmin:
logger.warning(">>> islitlet: " + str(islitlet))
logger.warning("expected_wvmin: " + str(expected_wvmin))
logger.warning("crmin1_linear.: " + str(crmin1_linear))
logger.warning("WARNING: Unexpected crmin1_linear < "
"expected_wvmin")
if crmax1_linear > expected_wvmax:
logger.warning(">>> islitlet: " + str(islitlet))
logger.warning("expected_wvmax: " + str(expected_wvmax))
logger.warning("crmax1_linear.: " + str(crmax1_linear))
logger.warning("WARNING: Unexpected crmax1_linear > "
"expected_wvmax")
cout += '.'
else:
cout += 'x'
if islitlet % 10 == 0:
if cout != 'x':
cout = str(islitlet // 10)
if debugplot != 0:
pause_debugplot(debugplot)
else:
# define Slitlet2dArc object
slt = Slitlet2dArc(islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=None,
parmodel=None,
debugplot=debugplot)
cout += 'i'
# store current slitlet in list of measured slitlets
measured_slitlets.append(slt)
logger.info(cout)
# ---
# generate FITS file structure with 55 spectra corresponding to the
# median spectrum for each slitlet
reduced_55sp = fits.PrimaryHDU(data=image2d_55sp)
reduced_55sp.header['crpix1'] = (0.0, 'reference pixel')
reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt1'] = (1.0, 'increment')
reduced_55sp.header['ctype1'] = 'PIXEL'
reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2')
reduced_55sp.header['crpix2'] = (0.0, 'reference pixel')
reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt2'] = (1.0, 'increment')
reduced_55sp.header['ctype2'] = 'PIXEL'
reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2')
# ---
# Generate structure to store intermediate results
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration'
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['bound_param_uuid'] = \
bound_param.uuid
outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined'
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['tags']['islitlet_min'] = islitlet_min
outdict['tags']['islitlet_max'] = islitlet_max
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
missing_slitlets = []
for slt in measured_slitlets:
islitlet = slt.islitlet
if islitlet_min <= islitlet <= islitlet_max:
# avoid error when creating a python list of coefficients from
# numpy polynomials when the polynomials do not exist (note that
# the JSON format doesn't handle numpy arrays and such arrays must
# be transformed into native python lists)
if slt.wpoly is None:
wpoly_coeff = None
else:
wpoly_coeff = slt.wpoly.coef.tolist()
if slt.wpoly_longslit_model is None:
wpoly_coeff_longslit_model = None
else:
wpoly_coeff_longslit_model = \
slt.wpoly_longslit_model.coef.tolist()
# avoid similar error when creating a python list of coefficients
# when the numpy array does not exist; note that this problem
# does not happen with tt?_aij_longslit_model and
# tt?_bij_longslit_model because the latter have already been
# created as native python lists
if slt.ttd_aij is None:
ttd_aij = None
else:
ttd_aij = slt.ttd_aij.tolist()
if slt.ttd_bij is None:
ttd_bij = None
else:
ttd_bij = slt.ttd_bij.tolist()
if slt.tti_aij is None:
tti_aij = None
else:
tti_aij = slt.tti_aij.tolist()
if slt.tti_bij is None:
tti_bij = None
else:
tti_bij = slt.tti_bij.tolist()
# creating temporary dictionary with the information corresponding
# to the current slitlett that will be saved in the JSON file
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_reference_lower': slt.y0_reference_lower,
'y0_reference_middle': slt.y0_reference_middle,
'y0_reference_upper': slt.y0_reference_upper,
'y0_reference_lower_expected': slt.y0_reference_lower_expected,
'y0_reference_middle_expected':
slt.y0_reference_middle_expected,
'y0_reference_upper_expected': slt.y0_reference_upper_expected,
'y0_frontier_lower': slt.y0_frontier_lower,
'y0_frontier_upper': slt.y0_frontier_upper,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected,
'corr_yrect_a': slt.corr_yrect_a,
'corr_yrect_b': slt.corr_yrect_b,
'min_row_rectified': slt.min_row_rectified,
'max_row_rectified': slt.max_row_rectified,
'ymargin_bb': slt.ymargin_bb,
'bb_nc1_orig': slt.bb_nc1_orig,
'bb_nc2_orig': slt.bb_nc2_orig,
'bb_ns1_orig': slt.bb_ns1_orig,
'bb_ns2_orig': slt.bb_ns2_orig,
'spectrail': {
'poly_coef_lower':
slt.list_spectrails[
slt.i_lower_spectrail].poly_funct.coef.tolist(),
'poly_coef_middle':
slt.list_spectrails[
slt.i_middle_spectrail].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_spectrails[
slt.i_upper_spectrail].poly_funct.coef.tolist(),
},
'frontier': {
'poly_coef_lower':
slt.list_frontiers[0].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_frontiers[1].poly_funct.coef.tolist(),
},
'ttd_order': slt.ttd_order,
'ttd_aij': ttd_aij,
'ttd_bij': ttd_bij,
'tti_aij': tti_aij,
'tti_bij': tti_bij,
'ttd_order_longslit_model': slt.ttd_order_longslit_model,
'ttd_aij_longslit_model': slt.ttd_aij_longslit_model,
'ttd_bij_longslit_model': slt.ttd_bij_longslit_model,
'tti_aij_longslit_model': slt.tti_aij_longslit_model,
'tti_bij_longslit_model': slt.tti_bij_longslit_model,
'wpoly_coeff': wpoly_coeff,
'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model,
'crval1_linear': slt.crval1_linear,
'cdelt1_linear': slt.cdelt1_linear
}
else:
missing_slitlets.append(islitlet)
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
outdict['contents'][slitlet_label] = tmp_dict
# ---
# OBSOLETE
'''
# save JSON file needed to compute the MOS model
with open(args.out_json.name, 'w') as fstream:
json.dump(outdict, fstream, indent=2, sort_keys=True)
print('>>> Saving file ' + args.out_json.name)
'''
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
'uuid' + bound_param.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
rectwv_coeff.missing_slitlets = missing_slitlets
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
raise ValueError("Unexpected error")
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_json.name)
# print('>>> Saving file ' + args.out_json.name)
# check_setstate_getstate(rectwv_coeff, args.out_json.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff, reduced_55sp
def refine_rectwv_coeff(input_image, rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'. format(
refine_wavecalib_mode
))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data(
'emirdrp.instrument.configs',
'Oliva_etal_2013.dat'
)
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm
)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1*lok2]
catlines_reference_flux = catlines_all_flux[lok1*lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm
)
# remove missing slitlets
if len(refined_rectwv_coeff.missing_slitlets) > 0:
for iremove in refined_rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets]
logger.info('list of useful slitlets: {}'.format(
list_useful_slitlets))
logger.info('list of not useful slitlets: {}'.format(
list_not_useful_slitlets))
tempwidths = np.array([csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1
)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave, sp_median, 'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum', show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-',
markerfmt=' ', basefmt='C4-', label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference, sp_reference, 'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot
)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset*cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
if np.any(lok):
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot
)
else:
offset_array[i] = 0.0
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i]*cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1-offset_array[i]*cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot
)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(stat_summary[
'robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot, yplot,
linestyle='', marker='o', color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False, **{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset, linestyle='--', color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image
def rectwv_coeff_from_arc_image(reduced_image,
bound_param,
lines_catalog,
args_nbrightlines=None,
args_ymargin_bb=2,
args_remove_sp_background=True,
args_times_sigma_threshold=10,
args_order_fmap=2,
args_sigma_gaussian_filtering=2,
args_margin_npix=50,
args_poldeg_initial=3,
args_poldeg_refined=5,
args_interactive=False,
args_threshold_wv=0,
args_ylogscale=False,
args_pdf=None,
args_geometry=(0,0,640,480),
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from arc image
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
bound_param : RefinedBoundaryModelParam instance
Refined boundary model.
lines_catalog : Numpy array
2D numpy array with the contents of the master file with the
expected arc line wavelengths.
args_nbrightlines : int
TBD
args_ymargin_bb : int
TBD
args_remove_sp_background : bool
TBD
args_times_sigma_threshold : float
TBD
args_order_fmap : int
TBD
args_sigma_gaussian_filtering : float
TBD
args_margin_npix : int
TBD
args_poldeg_initial : int
TBD
args_poldeg_refined : int
TBD
args_interactive : bool
TBD
args_threshold_wv : float
TBD
args_ylogscale : bool
TBD
args_pdf : TBD
args_geometry : TBD
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration of the input arc image.
reduced_55sp : HDUList object
Image with 55 spectra corresponding to the median spectrum for
each slitlet, employed to derived the wavelength calibration
polynomial.
"""
logger = logging.getLogger(__name__)
# protections
if args_interactive and args_pdf is not None:
logger.error('--interactive and --pdf are incompatible options')
raise ValueError('--interactive and --pdf are incompatible options')
# header and data array
header = reduced_image[0].header
image2d = reduced_image[0].data
# check grism and filter
filter_name = header['filter']
logger.info('Filter: ' + filter_name)
if filter_name != bound_param.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.info('Grism: ' + grism_name)
if grism_name != bound_param.tags['grism']:
raise ValueError('Grism name does not match!')
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
logger.debug(csu_conf)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
logger.debug(dtu_conf)
# set boundary parameters
parmodel = bound_param.meta_info['parmodel']
params = bound_params_from_dict(bound_param.__getstate__())
if abs(debugplot) >= 10:
print('-' * 83)
print('* FITTED BOUND PARAMETERS')
params.pretty_print()
pause_debugplot(debugplot)
# determine parameters according to grism+filter combination
wv_parameters = set_wv_parameters(filter_name, grism_name)
islitlet_min = wv_parameters['islitlet_min']
islitlet_max = wv_parameters['islitlet_max']
if args_nbrightlines is None:
nbrightlines = wv_parameters['nbrightlines']
else:
nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')]
poly_crval1_linear = wv_parameters['poly_crval1_linear']
poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear']
wvmin_expected = wv_parameters['wvmin_expected']
wvmax_expected = wv_parameters['wvmax_expected']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# list of slitlets to be computed
logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' +
str(islitlet_max) + ']')
# read master arc line wavelengths (only brightest lines)
wv_master = read_wv_master_from_array(
master_table=lines_catalog, lines='brightest', debugplot=debugplot
)
# read master arc line wavelengths (whole data set)
wv_master_all = read_wv_master_from_array(
master_table=lines_catalog, lines='all', debugplot=debugplot
)
# check that the arc lines in the master file are properly sorted
# in ascending order
for i in range(len(wv_master_all) - 1):
if wv_master_all[i] >= wv_master_all[i + 1]:
logger.error('>>> wavelengths: ' +
str(wv_master_all[i]) + ' ' +
str(wv_master_all[i+1]))
raise ValueError('Arc lines are not sorted in master file')
# ---
image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
# compute rectification transformation and wavelength calibration
# polynomials
measured_slitlets = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet_min <= islitlet <= islitlet_max:
# define Slitlet2dArc object
slt = Slitlet2dArc(
islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=params,
parmodel=parmodel,
debugplot=debugplot
)
# extract 2D image corresponding to the selected slitlet, clipping
# the image beyond the unrectified slitlet (in order to isolate
# the arc lines of the current slitlet; otherwise there are
# problems with arc lines from neighbour slitlets)
image2d_tmp = select_unrectified_slitlet(
image2d=image2d,
islitlet=islitlet,
csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet),
params=params,
parmodel=parmodel,
maskonly=False
)
slitlet2d = slt.extract_slitlet2d(image2d_tmp)
# subtract smooth background computed as follows:
# - median collapsed spectrum of the whole slitlet2d
# - independent median filtering of the previous spectrum in the
# two halves in the spectral direction
if args_remove_sp_background:
spmedian = np.median(slitlet2d, axis=0)
naxis1_tmp = spmedian.shape[0]
jmidpoint = naxis1_tmp // 2
sp1 = medfilt(spmedian[:jmidpoint], [201])
sp2 = medfilt(spmedian[jmidpoint:], [201])
spbackground = np.concatenate((sp1, sp2))
slitlet2d -= spbackground
# locate unknown arc lines
slt.locate_unknown_arc_lines(
slitlet2d=slitlet2d,
times_sigma_threshold=args_times_sigma_threshold)
# continue working with current slitlet only if arc lines have
# been detected
if slt.list_arc_lines is not None:
# compute intersections between spectrum trails and arc lines
slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d)
# compute rectification transformation
slt.estimate_tt_to_rectify(order=args_order_fmap,
slitlet2d=slitlet2d)
# rectify image
slitlet2d_rect = slt.rectify(slitlet2d,
resampling=2,
transformation=1)
# median spectrum and line peaks from rectified image
sp_median, fxpeaks = slt.median_spectrum_from_rectified_image(
slitlet2d_rect,
sigma_gaussian_filtering=args_sigma_gaussian_filtering,
nwinwidth_initial=5,
nwinwidth_refined=5,
times_sigma_threshold=5,
npix_avoid_border=6,
nbrightlines=nbrightlines
)
image2d_55sp[islitlet - 1, :] = sp_median
# determine expected wavelength limits prior to the wavelength
# calibration
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
crval1_linear = poly_crval1_linear(csu_bar_slit_center)
cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center)
expected_wvmin = crval1_linear - \
args_margin_npix * cdelt1_linear
naxis1_linear = sp_median.shape[0]
crvaln_linear = crval1_linear + \
(naxis1_linear - 1) * cdelt1_linear
expected_wvmax = crvaln_linear + \
args_margin_npix * cdelt1_linear
# override previous estimates when necessary
if wvmin_expected is not None:
expected_wvmin = wvmin_expected
if wvmax_expected is not None:
expected_wvmax = wvmax_expected
# clip initial master arc line list with bright lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master
lok2 = wv_master <= expected_wvmax
lok = lok1 * lok2
wv_master_eff = wv_master[lok]
# perform initial wavelength calibration
solution_wv = wvcal_spectrum(
sp=sp_median,
fxpeaks=fxpeaks,
poly_degree_wfit=args_poldeg_initial,
wv_master=wv_master_eff,
wv_ini_search=expected_wvmin,
wv_end_search=expected_wvmax,
wvmin_useful=wvmin_useful,
wvmax_useful=wvmax_useful,
geometry=args_geometry,
debugplot=slt.debugplot
)
# store initial wavelength calibration polynomial in current
# slitlet instance
slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff)
pause_debugplot(debugplot)
# clip initial master arc line list with all the lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master_all
lok2 = wv_master_all <= expected_wvmax
lok = lok1 * lok2
wv_master_all_eff = wv_master_all[lok]
# clip master arc line list to useful region
if wvmin_useful is not None:
lok = wvmin_useful <= wv_master_all_eff
wv_master_all_eff = wv_master_all_eff[lok]
if wvmax_useful is not None:
lok = wv_master_all_eff <= wvmax_useful
wv_master_all_eff = wv_master_all_eff[lok]
# refine wavelength calibration
if args_poldeg_refined > 0:
plottitle = '[slitlet#{}, refined]'.format(islitlet)
poly_refined, yres_summary = refine_arccalibration(
sp=sp_median,
poly_initial=slt.wpoly,
wv_master=wv_master_all_eff,
poldeg=args_poldeg_refined,
ntimes_match_wv=1,
interactive=args_interactive,
threshold=args_threshold_wv,
plottitle=plottitle,
ylogscale=args_ylogscale,
geometry=args_geometry,
pdf=args_pdf,
debugplot=slt.debugplot
)
# store refined wavelength calibration polynomial in
# current slitlet instance
slt.wpoly = poly_refined
# compute approximate linear values for CRVAL1 and CDELT1
naxis1_linear = sp_median.shape[0]
crmin1_linear = slt.wpoly(1)
crmax1_linear = slt.wpoly(naxis1_linear)
slt.crval1_linear = crmin1_linear
slt.cdelt1_linear = \
(crmax1_linear - crmin1_linear) / (naxis1_linear - 1)
# check that the trimming of wv_master and wv_master_all has
# preserved the wavelength range [crmin1_linear, crmax1_linear]
if crmin1_linear < expected_wvmin:
logger.warning(">>> islitlet: " +str(islitlet))
logger.warning("expected_wvmin: " + str(expected_wvmin))
logger.warning("crmin1_linear.: " + str(crmin1_linear))
logger.warning("WARNING: Unexpected crmin1_linear < "
"expected_wvmin")
if crmax1_linear > expected_wvmax:
logger.warning(">>> islitlet: " +str(islitlet))
logger.warning("expected_wvmax: " + str(expected_wvmax))
logger.warning("crmax1_linear.: " + str(crmax1_linear))
logger.warning("WARNING: Unexpected crmax1_linear > "
"expected_wvmax")
cout += '.'
else:
cout += 'x'
if islitlet % 10 == 0:
if cout != 'x':
cout = str(islitlet // 10)
if debugplot != 0:
pause_debugplot(debugplot)
else:
# define Slitlet2dArc object
slt = Slitlet2dArc(
islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=None,
parmodel=None,
debugplot=debugplot
)
cout += 'i'
# store current slitlet in list of measured slitlets
measured_slitlets.append(slt)
logger.info(cout)
# ---
# generate FITS file structure with 55 spectra corresponding to the
# median spectrum for each slitlet
reduced_55sp = fits.PrimaryHDU(data=image2d_55sp)
reduced_55sp.header['crpix1'] = (0.0, 'reference pixel')
reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt1'] = (1.0, 'increment')
reduced_55sp.header['ctype1'] = 'PIXEL'
reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2')
reduced_55sp.header['crpix2'] = (0.0, 'reference pixel')
reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt2'] = (1.0, 'increment')
reduced_55sp.header['ctype2'] = 'PIXEL'
reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2')
# ---
# Generate structure to store intermediate results
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration'
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['bound_param_uuid'] = \
bound_param.uuid
outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined'
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['tags']['islitlet_min'] = islitlet_min
outdict['tags']['islitlet_max'] = islitlet_max
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
missing_slitlets = []
for slt in measured_slitlets:
islitlet = slt.islitlet
if islitlet_min <= islitlet <= islitlet_max:
# avoid error when creating a python list of coefficients from
# numpy polynomials when the polynomials do not exist (note that
# the JSON format doesn't handle numpy arrays and such arrays must
# be transformed into native python lists)
if slt.wpoly is None:
wpoly_coeff = None
else:
wpoly_coeff = slt.wpoly.coef.tolist()
if slt.wpoly_longslit_model is None:
wpoly_coeff_longslit_model = None
else:
wpoly_coeff_longslit_model = \
slt.wpoly_longslit_model.coef.tolist()
# avoid similar error when creating a python list of coefficients
# when the numpy array does not exist; note that this problem
# does not happen with tt?_aij_longslit_model and
# tt?_bij_longslit_model because the latter have already been
# created as native python lists
if slt.ttd_aij is None:
ttd_aij = None
else:
ttd_aij = slt.ttd_aij.tolist()
if slt.ttd_bij is None:
ttd_bij = None
else:
ttd_bij = slt.ttd_bij.tolist()
if slt.tti_aij is None:
tti_aij = None
else:
tti_aij = slt.tti_aij.tolist()
if slt.tti_bij is None:
tti_bij = None
else:
tti_bij = slt.tti_bij.tolist()
# creating temporary dictionary with the information corresponding
# to the current slitlett that will be saved in the JSON file
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_reference_lower': slt.y0_reference_lower,
'y0_reference_middle': slt.y0_reference_middle,
'y0_reference_upper': slt.y0_reference_upper,
'y0_reference_lower_expected':
slt.y0_reference_lower_expected,
'y0_reference_middle_expected':
slt.y0_reference_middle_expected,
'y0_reference_upper_expected':
slt.y0_reference_upper_expected,
'y0_frontier_lower': slt.y0_frontier_lower,
'y0_frontier_upper': slt.y0_frontier_upper,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected,
'corr_yrect_a': slt.corr_yrect_a,
'corr_yrect_b': slt.corr_yrect_b,
'min_row_rectified': slt.min_row_rectified,
'max_row_rectified': slt.max_row_rectified,
'ymargin_bb': slt.ymargin_bb,
'bb_nc1_orig': slt.bb_nc1_orig,
'bb_nc2_orig': slt.bb_nc2_orig,
'bb_ns1_orig': slt.bb_ns1_orig,
'bb_ns2_orig': slt.bb_ns2_orig,
'spectrail': {
'poly_coef_lower':
slt.list_spectrails[
slt.i_lower_spectrail].poly_funct.coef.tolist(),
'poly_coef_middle':
slt.list_spectrails[
slt.i_middle_spectrail].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_spectrails[
slt.i_upper_spectrail].poly_funct.coef.tolist(),
},
'frontier': {
'poly_coef_lower':
slt.list_frontiers[0].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_frontiers[1].poly_funct.coef.tolist(),
},
'ttd_order': slt.ttd_order,
'ttd_aij': ttd_aij,
'ttd_bij': ttd_bij,
'tti_aij': tti_aij,
'tti_bij': tti_bij,
'ttd_order_longslit_model': slt.ttd_order_longslit_model,
'ttd_aij_longslit_model': slt.ttd_aij_longslit_model,
'ttd_bij_longslit_model': slt.ttd_bij_longslit_model,
'tti_aij_longslit_model': slt.tti_aij_longslit_model,
'tti_bij_longslit_model': slt.tti_bij_longslit_model,
'wpoly_coeff': wpoly_coeff,
'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model,
'crval1_linear': slt.crval1_linear,
'cdelt1_linear': slt.cdelt1_linear
}
else:
missing_slitlets.append(islitlet)
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
outdict['contents'][slitlet_label] = tmp_dict
# ---
# OBSOLETE
'''
# save JSON file needed to compute the MOS model
with open(args.out_json.name, 'w') as fstream:
json.dump(outdict, fstream, indent=2, sort_keys=True)
print('>>> Saving file ' + args.out_json.name)
'''
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
'uuid' + bound_param.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
rectwv_coeff.missing_slitlets = missing_slitlets
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
raise ValueError("Unexpected error")
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_json.name)
# print('>>> Saving file ' + args.out_json.name)
# check_setstate_getstate(rectwv_coeff, args.out_json.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff, reduced_55sp
def compute_slitlet_boundaries(
filename, grism, spfilter, list_slitlets,
size_x_medfilt, size_y_savgol,
times_sigma_threshold,
bounddict, debugplot=0):
"""Compute slitlet boundaries using continuum lamp images.
Parameters
----------
filename : string
Input continumm lamp image.
grism : string
Grism name. It must be one in EMIR_VALID_GRISMS.
spfilter : string
Filter name. It must be one in EMIR_VALID_FILTERS.
list_slitlets : list of integers
Number of slitlets to be updated.
size_x_medfilt : int
Window in the X (spectral) direction, in pixels, to apply
the 1d median filter in order to remove bad pixels.
size_y_savgol : int
Window in the Y (spatial) direction to be used when using the
1d Savitzky-Golay filter.
times_sigma_threshold : float
Times sigma to detect peaks in derivatives.
bounddict : dictionary of dictionaries
Structure to store the boundaries.
debugplot : int
Determines whether intermediate computations and/or plots are
displayed.
"""
# read 2D image
hdulist = fits.open(filename)
image_header = hdulist[0].header
image2d = hdulist[0].data
naxis2, naxis1 = image2d.shape
hdulist.close()
if debugplot >= 10:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
# ToDo: replace this by application of cosmetic defect mask!
for j in range(1024):
image2d[1024, j] = (image2d[1023, j] + image2d[1025, j]) / 2
image2d[1023, j + 1024] = (image2d[1022, j + 1024] +
image2d[1024, j + 1024]) / 2
# remove path from filename
sfilename = os.path.basename(filename)
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# read CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_fits(filename)
# read DTU configuration from FITS header
dtu_config = DtuConfiguration.define_from_fits(filename)
# read grism
grism_in_header = image_header['grism']
if grism != grism_in_header:
raise ValueError("GRISM keyword=" + grism_in_header +
" is not the expected value=" + grism)
# read filter
spfilter_in_header = image_header['filter']
if spfilter != spfilter_in_header:
raise ValueError("FILTER keyword=" + spfilter_in_header +
" is not the expected value=" + spfilter)
# read rotator position angle
rotang = image_header['rotang']
# read date-obs
date_obs = image_header['date-obs']
for islitlet in list_slitlets:
if debugplot < 10:
sys.stdout.write('.')
sys.stdout.flush()
sltlim = SlitletLimits(grism, spfilter, islitlet)
# extract slitlet2d
slitlet2d = extract_slitlet2d(image2d, sltlim)
if debugplot % 10 != 0:
ximshow(slitlet2d,
title=sfilename + " [original]"
"\nslitlet=" + str(islitlet) +
", grism=" + grism +
", filter=" + spfilter +
", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
debugplot=debugplot)
# apply 1d median filtering (along the spectral direction)
# to remove bad pixels
size_x = size_x_medfilt
size_y = 1
slitlet2d_smooth = ndimage.filters.median_filter(
slitlet2d, size=(size_y, size_x))
if debugplot % 10 != 0:
ximshow(slitlet2d_smooth,
title=sfilename + " [smoothed]"
"\nslitlet=" + str(islitlet) +
", grism=" + grism +
", filter=" + spfilter +
", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
debugplot=debugplot)
# apply 1d Savitzky-Golay filter (along the spatial direction)
# to compute first derivative
slitlet2d_savgol = savgol_filter(
slitlet2d_smooth, window_length=size_y_savgol, polyorder=2,
deriv=1, axis=0)
# compute basic statistics
q25, q50, q75 = np.percentile(slitlet2d_savgol, q=[25.0, 50.0, 75.0])
sigmag = 0.7413 * (q75 - q25) # robust standard deviation
if debugplot >= 10:
print("q50, sigmag:", q50, sigmag)
if debugplot % 10 != 0:
ximshow(slitlet2d_savgol,
title=sfilename + " [S.-G.filt.]"
"\nslitlet=" + str(islitlet) +
", grism=" + grism +
", filter=" + spfilter +
", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
z1z2=(q50-times_sigma_threshold*sigmag,
q50+times_sigma_threshold*sigmag),
debugplot=debugplot)
# identify objects in slitlet2d_savgol: pixels with positive
# derivatives are identify independently from pixels with
# negative derivaties; then the two set of potential features
# are merged; this approach avoids some problems when, in
# nearby regions, there are pixels with positive and negative
# derivatives (in those circumstances a single search as
# np.logical_or(
# slitlet2d_savgol < q50 - times_sigma_threshold * sigmag,
# slitlet2d_savgol > q50 + times_sigma_threshold * sigmag)
# led to erroneous detections!)
#
# search for positive derivatives
labels2d_objects_pos, no_objects_pos = ndimage.label(
slitlet2d_savgol > q50 + times_sigma_threshold * sigmag)
# search for negative derivatives
labels2d_objects_neg, no_objects_neg = ndimage.label(
slitlet2d_savgol < q50 - times_sigma_threshold * sigmag)
# merge both sets
non_zero_neg = np.where(labels2d_objects_neg > 0)
labels2d_objects = np.copy(labels2d_objects_pos)
labels2d_objects[non_zero_neg] += \
labels2d_objects_neg[non_zero_neg] + no_objects_pos
no_objects = no_objects_pos + no_objects_neg
if debugplot >= 10:
print("Number of objects with positive derivative:",
no_objects_pos)
print("Number of objects with negative derivative:",
no_objects_neg)
print("Total number of objects initially found...:", no_objects)
if debugplot % 10 != 0:
ximshow(labels2d_objects,
z1z2=(0, no_objects),
title=sfilename + " [objects]"
"\nslitlet=" + str(islitlet) +
", grism=" + grism +
", filter=" + spfilter +
", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
cbar_label="Object number",
debugplot=debugplot)
# select boundaries as the largest objects found with
# positive and negative derivatives
n_der_pos = 0 # number of pixels covered by the object with deriv > 0
i_der_pos = 0 # id of the object with deriv > 0
n_der_neg = 0 # number of pixels covered by the object with deriv < 0
i_der_neg = 0 # id of the object with deriv < 0
for i in range(1, no_objects+1):
xy_tmp = np.where(labels2d_objects == i)
n_pix = len(xy_tmp[0])
if i <= no_objects_pos:
if n_pix > n_der_pos:
i_der_pos = i
n_der_pos = n_pix
else:
if n_pix > n_der_neg:
i_der_neg = i
n_der_neg = n_pix
# determine which boundary is lower and which is upper
y_center_mass_der_pos = ndimage.center_of_mass(
slitlet2d_savgol, labels2d_objects, [i_der_pos])[0][0]
y_center_mass_der_neg = ndimage.center_of_mass(
slitlet2d_savgol, labels2d_objects, [i_der_neg])[0][0]
if y_center_mass_der_pos < y_center_mass_der_neg:
i_lower = i_der_pos
i_upper = i_der_neg
if debugplot >= 10:
print("-> lower boundary has positive derivatives")
else:
i_lower = i_der_neg
i_upper = i_der_pos
if debugplot >= 10:
print("-> lower boundary has negative derivatives")
list_slices_ok = [i_lower, i_upper]
# adjust individual boundaries passing the selection:
# - select points in the image belonging to a given boundary
# - compute weighted mean of the pixels of the boundary, column
# by column (this reduces dramatically the number of points
# to be fitted to determine the boundary)
list_boundaries = []
for k in range(2): # k=0 lower boundary, k=1 upper boundary
# select points to be fitted for a particular boundary
# (note: be careful with array indices and pixel
# coordinates)
xy_tmp = np.where(labels2d_objects == list_slices_ok[k])
xmin = xy_tmp[1].min() # array indices (integers)
xmax = xy_tmp[1].max() # array indices (integers)
xfit = []
yfit = []
# fix range for fit
if k == 0:
xmineff = max(sltlim.xmin_lower_boundary_fit, xmin)
xmaxeff = min(sltlim.xmax_lower_boundary_fit, xmax)
else:
xmineff = max(sltlim.xmin_upper_boundary_fit, xmin)
xmaxeff = min(sltlim.xmax_upper_boundary_fit, xmax)
# loop in columns of the image belonging to the boundary
for xdum in range(xmineff, xmaxeff + 1): # array indices (integer)
iok = np.where(xy_tmp[1] == xdum)
y_tmp = xy_tmp[0][iok] + sltlim.bb_ns1_orig # image pixel
weight = slitlet2d_savgol[xy_tmp[0][iok], xy_tmp[1][iok]]
y_wmean = sum(y_tmp * weight) / sum(weight)
xfit.append(xdum + sltlim.bb_nc1_orig)
yfit.append(y_wmean)
xfit = np.array(xfit)
yfit = np.array(yfit)
# declare new SpectrumTrail instance
boundary = SpectrumTrail()
# define new boundary
boundary.fit(x=xfit, y=yfit, deg=sltlim.deg_boundary,
times_sigma_reject=10,
title="slit:" + str(sltlim.islitlet) +
", deg=" + str(sltlim.deg_boundary),
debugplot=0)
list_boundaries.append(boundary)
if debugplot % 10 != 0:
for tmp_img, tmp_label in zip(
[slitlet2d_savgol, slitlet2d],
[' [S.-G.filt.]', ' [original]']
):
ax = ximshow(tmp_img,
title=sfilename + tmp_label +
"\nslitlet=" + str(islitlet) +
", grism=" + grism +
", filter=" + spfilter +
", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig,
sltlim.bb_ns1_orig),
show=False,
debugplot=debugplot)
for k in range(2):
xpol, ypol = list_boundaries[k].linspace_pix(
start=1, stop=EMIR_NAXIS1)
ax.plot(xpol, ypol, 'b--', linewidth=1)
for k in range(2):
xpol, ypol = list_boundaries[k].linspace_pix()
ax.plot(xpol, ypol, 'g--', linewidth=4)
# show plot
pause_debugplot(debugplot, pltshow=True)
# update bounddict
tmp_dict = {
'boundary_coef_lower':
list_boundaries[0].poly_funct.coef.tolist(),
'boundary_xmin_lower': list_boundaries[0].xlower_line,
'boundary_xmax_lower': list_boundaries[0].xupper_line,
'boundary_coef_upper':
list_boundaries[1].poly_funct.coef.tolist(),
'boundary_xmin_upper': list_boundaries[1].xlower_line,
'boundary_xmax_upper': list_boundaries[1].xupper_line,
'csu_bar_left': csu_config.csu_bar_left(islitlet),
'csu_bar_right': csu_config.csu_bar_right(islitlet),
'csu_bar_slit_center': csu_config.csu_bar_slit_center(islitlet),
'csu_bar_slit_width': csu_config.csu_bar_slit_width(islitlet),
'rotang': rotang,
'xdtu': dtu_config.xdtu,
'ydtu': dtu_config.ydtu,
'zdtu': dtu_config.zdtu,
'xdtu_0': dtu_config.xdtu_0,
'ydtu_0': dtu_config.ydtu_0,
'zdtu_0': dtu_config.zdtu_0,
'zzz_info1': os.getlogin() + '@' + socket.gethostname(),
'zzz_info2': datetime.now().isoformat()
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
if slitlet_label not in bounddict['contents']:
bounddict['contents'][slitlet_label] = {}
bounddict['contents'][slitlet_label][date_obs] = tmp_dict
if debugplot < 10:
print("")
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser(
description='description: compute median spectrum for each slitlet')
# positional arguments
parser.add_argument("fitsfile",
help="Input FITS file name",
type=argparse.FileType('rb'))
parser.add_argument("outfile",
help="Output FITS file name",
type=lambda x: arg_file_is_new(parser, x, mode='wb'))
# optional arguments
parser.add_argument("--mode",
help="Output type: 0 -> full frame (default), "
"1 -> individual slitlets, "
"2 -> collapsed single spectrum)",
default=0,
type=int,
choices=[0, 1, 2])
parser.add_argument("--minimum_slitlet_width_mm",
help="Minimum slitlet width (mm) for --mode 2 "
"(default=0)",
default=EMIR_MINIMUM_SLITLET_WIDTH_MM,
type=float)
parser.add_argument("--maximum_slitlet_width_mm",
help="Maximum slitlet width (mm) for --mode 2 "
"(default=" + str(EMIR_MAXIMUM_SLITLET_WIDTH_MM) + ")",
default=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
type=float)
parser.add_argument("--debugplot",
help="Integer indicating plotting/debugging" +
" (default=0)",
default=0,
type=int,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args(args=args)
if args.echo:
print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n')
# read input FITS file
hdulist = fits.open(args.fitsfile)
# determine useful slitlets
main_header = hdulist[0].header
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=args.minimum_slitlet_width_mm,
maxwidth=args.maximum_slitlet_width_mm)
image_median = median_slitlets_rectified(
hdulist,
mode=args.mode,
list_useful_slitlets=list_useful_slitlets,
debugplot=args.debugplot)
# save result
image_median.writeto(args.outfile, overwrite=True)
def compute_slitlet_boundaries(filename,
grism,
spfilter,
list_slitlets,
size_x_medfilt,
size_y_savgol,
times_sigma_threshold,
bounddict,
debugplot=0):
"""Compute slitlet boundaries using continuum lamp images.
Parameters
----------
filename : string
Input continumm lamp image.
grism : string
Grism name. It must be one in EMIR_VALID_GRISMS.
spfilter : string
Filter name. It must be one in EMIR_VALID_FILTERS.
list_slitlets : list of integers
Number of slitlets to be updated.
size_x_medfilt : int
Window in the X (spectral) direction, in pixels, to apply
the 1d median filter in order to remove bad pixels.
size_y_savgol : int
Window in the Y (spatial) direction to be used when using the
1d Savitzky-Golay filter.
times_sigma_threshold : float
Times sigma to detect peaks in derivatives.
bounddict : dictionary of dictionaries
Structure to store the boundaries.
debugplot : int
Determines whether intermediate computations and/or plots are
displayed.
"""
# read 2D image
hdulist = fits.open(filename)
image_header = hdulist[0].header
image2d = hdulist[0].data
naxis2, naxis1 = image2d.shape
hdulist.close()
if debugplot >= 10:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
# ToDo: replace this by application of cosmetic defect mask!
for j in range(1024):
image2d[1024, j] = (image2d[1023, j] + image2d[1025, j]) / 2
image2d[1023, j +
1024] = (image2d[1022, j + 1024] + image2d[1024, j + 1024]) / 2
# remove path from filename
sfilename = os.path.basename(filename)
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# read CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_fits(filename)
# read DTU configuration from FITS header
dtu_config = DtuConfiguration.define_from_fits(filename)
# read grism
grism_in_header = image_header['grism']
if grism != grism_in_header:
raise ValueError("GRISM keyword=" + grism_in_header +
" is not the expected value=" + grism)
# read filter
spfilter_in_header = image_header['filter']
if spfilter != spfilter_in_header:
raise ValueError("FILTER keyword=" + spfilter_in_header +
" is not the expected value=" + spfilter)
# read rotator position angle
rotang = image_header['rotang']
# read date-obs
date_obs = image_header['date-obs']
for islitlet in list_slitlets:
if debugplot < 10:
sys.stdout.write('.')
sys.stdout.flush()
sltlim = SlitletLimits(grism, spfilter, islitlet)
# extract slitlet2d
slitlet2d = extract_slitlet2d(image2d, sltlim)
if debugplot % 10 != 0:
ximshow(slitlet2d,
title=sfilename + " [original]"
"\nslitlet=" + str(islitlet) + ", grism=" + grism +
", filter=" + spfilter + ", rotang=" +
str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
debugplot=debugplot)
# apply 1d median filtering (along the spectral direction)
# to remove bad pixels
size_x = size_x_medfilt
size_y = 1
slitlet2d_smooth = ndimage.filters.median_filter(slitlet2d,
size=(size_y, size_x))
if debugplot % 10 != 0:
ximshow(slitlet2d_smooth,
title=sfilename + " [smoothed]"
"\nslitlet=" + str(islitlet) + ", grism=" + grism +
", filter=" + spfilter + ", rotang=" +
str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
debugplot=debugplot)
# apply 1d Savitzky-Golay filter (along the spatial direction)
# to compute first derivative
slitlet2d_savgol = savgol_filter(slitlet2d_smooth,
window_length=size_y_savgol,
polyorder=2,
deriv=1,
axis=0)
# compute basic statistics
q25, q50, q75 = np.percentile(slitlet2d_savgol, q=[25.0, 50.0, 75.0])
sigmag = 0.7413 * (q75 - q25) # robust standard deviation
if debugplot >= 10:
print("q50, sigmag:", q50, sigmag)
if debugplot % 10 != 0:
ximshow(slitlet2d_savgol,
title=sfilename + " [S.-G.filt.]"
"\nslitlet=" + str(islitlet) + ", grism=" + grism +
", filter=" + spfilter + ", rotang=" +
str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
z1z2=(q50 - times_sigma_threshold * sigmag,
q50 + times_sigma_threshold * sigmag),
debugplot=debugplot)
# identify objects in slitlet2d_savgol: pixels with positive
# derivatives are identify independently from pixels with
# negative derivaties; then the two set of potential features
# are merged; this approach avoids some problems when, in
# nearby regions, there are pixels with positive and negative
# derivatives (in those circumstances a single search as
# np.logical_or(
# slitlet2d_savgol < q50 - times_sigma_threshold * sigmag,
# slitlet2d_savgol > q50 + times_sigma_threshold * sigmag)
# led to erroneous detections!)
#
# search for positive derivatives
labels2d_objects_pos, no_objects_pos = ndimage.label(
slitlet2d_savgol > q50 + times_sigma_threshold * sigmag)
# search for negative derivatives
labels2d_objects_neg, no_objects_neg = ndimage.label(
slitlet2d_savgol < q50 - times_sigma_threshold * sigmag)
# merge both sets
non_zero_neg = np.where(labels2d_objects_neg > 0)
labels2d_objects = np.copy(labels2d_objects_pos)
labels2d_objects[non_zero_neg] += \
labels2d_objects_neg[non_zero_neg] + no_objects_pos
no_objects = no_objects_pos + no_objects_neg
if debugplot >= 10:
print("Number of objects with positive derivative:",
no_objects_pos)
print("Number of objects with negative derivative:",
no_objects_neg)
print("Total number of objects initially found...:", no_objects)
if debugplot % 10 != 0:
ximshow(labels2d_objects,
z1z2=(0, no_objects),
title=sfilename + " [objects]"
"\nslitlet=" + str(islitlet) + ", grism=" + grism +
", filter=" + spfilter + ", rotang=" +
str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig, sltlim.bb_ns1_orig),
cbar_label="Object number",
debugplot=debugplot)
# select boundaries as the largest objects found with
# positive and negative derivatives
n_der_pos = 0 # number of pixels covered by the object with deriv > 0
i_der_pos = 0 # id of the object with deriv > 0
n_der_neg = 0 # number of pixels covered by the object with deriv < 0
i_der_neg = 0 # id of the object with deriv < 0
for i in range(1, no_objects + 1):
xy_tmp = np.where(labels2d_objects == i)
n_pix = len(xy_tmp[0])
if i <= no_objects_pos:
if n_pix > n_der_pos:
i_der_pos = i
n_der_pos = n_pix
else:
if n_pix > n_der_neg:
i_der_neg = i
n_der_neg = n_pix
# determine which boundary is lower and which is upper
y_center_mass_der_pos = ndimage.center_of_mass(slitlet2d_savgol,
labels2d_objects,
[i_der_pos])[0][0]
y_center_mass_der_neg = ndimage.center_of_mass(slitlet2d_savgol,
labels2d_objects,
[i_der_neg])[0][0]
if y_center_mass_der_pos < y_center_mass_der_neg:
i_lower = i_der_pos
i_upper = i_der_neg
if debugplot >= 10:
print("-> lower boundary has positive derivatives")
else:
i_lower = i_der_neg
i_upper = i_der_pos
if debugplot >= 10:
print("-> lower boundary has negative derivatives")
list_slices_ok = [i_lower, i_upper]
# adjust individual boundaries passing the selection:
# - select points in the image belonging to a given boundary
# - compute weighted mean of the pixels of the boundary, column
# by column (this reduces dramatically the number of points
# to be fitted to determine the boundary)
list_boundaries = []
for k in range(2): # k=0 lower boundary, k=1 upper boundary
# select points to be fitted for a particular boundary
# (note: be careful with array indices and pixel
# coordinates)
xy_tmp = np.where(labels2d_objects == list_slices_ok[k])
xmin = xy_tmp[1].min() # array indices (integers)
xmax = xy_tmp[1].max() # array indices (integers)
xfit = []
yfit = []
# fix range for fit
if k == 0:
xmineff = max(sltlim.xmin_lower_boundary_fit, xmin)
xmaxeff = min(sltlim.xmax_lower_boundary_fit, xmax)
else:
xmineff = max(sltlim.xmin_upper_boundary_fit, xmin)
xmaxeff = min(sltlim.xmax_upper_boundary_fit, xmax)
# loop in columns of the image belonging to the boundary
for xdum in range(xmineff, xmaxeff + 1): # array indices (integer)
iok = np.where(xy_tmp[1] == xdum)
y_tmp = xy_tmp[0][iok] + sltlim.bb_ns1_orig # image pixel
weight = slitlet2d_savgol[xy_tmp[0][iok], xy_tmp[1][iok]]
y_wmean = sum(y_tmp * weight) / sum(weight)
xfit.append(xdum + sltlim.bb_nc1_orig)
yfit.append(y_wmean)
xfit = np.array(xfit)
yfit = np.array(yfit)
# declare new SpectrumTrail instance
boundary = SpectrumTrail()
# define new boundary
boundary.fit(x=xfit,
y=yfit,
deg=sltlim.deg_boundary,
times_sigma_reject=10,
title="slit:" + str(sltlim.islitlet) + ", deg=" +
str(sltlim.deg_boundary),
debugplot=0)
list_boundaries.append(boundary)
if debugplot % 10 != 0:
for tmp_img, tmp_label in zip([slitlet2d_savgol, slitlet2d],
[' [S.-G.filt.]', ' [original]']):
ax = ximshow(tmp_img,
title=sfilename + tmp_label + "\nslitlet=" +
str(islitlet) + ", grism=" + grism + ", filter=" +
spfilter + ", rotang=" + str(round(rotang, 2)),
first_pixel=(sltlim.bb_nc1_orig,
sltlim.bb_ns1_orig),
show=False,
debugplot=debugplot)
for k in range(2):
xpol, ypol = list_boundaries[k].linspace_pix(
start=1, stop=EMIR_NAXIS1)
ax.plot(xpol, ypol, 'b--', linewidth=1)
for k in range(2):
xpol, ypol = list_boundaries[k].linspace_pix()
ax.plot(xpol, ypol, 'g--', linewidth=4)
# show plot
pause_debugplot(debugplot, pltshow=True)
# update bounddict
tmp_dict = {
'boundary_coef_lower': list_boundaries[0].poly_funct.coef.tolist(),
'boundary_xmin_lower': list_boundaries[0].xlower_line,
'boundary_xmax_lower': list_boundaries[0].xupper_line,
'boundary_coef_upper': list_boundaries[1].poly_funct.coef.tolist(),
'boundary_xmin_upper': list_boundaries[1].xlower_line,
'boundary_xmax_upper': list_boundaries[1].xupper_line,
'csu_bar_left': csu_config.csu_bar_left(islitlet),
'csu_bar_right': csu_config.csu_bar_right(islitlet),
'csu_bar_slit_center': csu_config.csu_bar_slit_center(islitlet),
'csu_bar_slit_width': csu_config.csu_bar_slit_width(islitlet),
'rotang': rotang,
'xdtu': dtu_config.xdtu,
'ydtu': dtu_config.ydtu,
'zdtu': dtu_config.zdtu,
'xdtu_0': dtu_config.xdtu_0,
'ydtu_0': dtu_config.ydtu_0,
'zdtu_0': dtu_config.zdtu_0,
'zzz_info1': os.getlogin() + '@' + socket.gethostname(),
'zzz_info2': datetime.now().isoformat()
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
if slitlet_label not in bounddict['contents']:
bounddict['contents'][slitlet_label] = {}
bounddict['contents'][slitlet_label][date_obs] = tmp_dict
if debugplot < 10:
print("")
def run(self, rinput):
self.logger.info('starting generation of flatlowfreq')
self.logger.info('rectwv_coeff..........................: {}'.format(
rinput.rectwv_coeff))
self.logger.info('master_rectwv.........................: {}'.format(
rinput.master_rectwv))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
self.logger.info('nwindow_x_median......................: {}'.format(
rinput.nwindow_x_median))
self.logger.info('nwindow_y_median......................: {}'.format(
rinput.nwindow_y_median))
self.logger.info('Minimum fraction......................: {}'.format(
rinput.minimum_fraction))
self.logger.info('Minimum value in output...............: {}'.format(
rinput.minimum_value_in_output))
self.logger.info('Maximum value in output...............: {}'.format(
rinput.maximum_value_in_output))
# check rectification and wavelength calibration information
if rinput.master_rectwv is None and rinput.rectwv_coeff is None:
raise ValueError('No master_rectwv nor rectwv_coeff data have '
'been provided')
elif rinput.master_rectwv is not None and \
rinput.rectwv_coeff is not None:
self.logger.warning('rectwv_coeff will be used instead of '
'master_rectwv')
if rinput.rectwv_coeff is not None and \
(rinput.global_integer_offset_x_pix != 0 or
rinput.global_integer_offset_y_pix != 0):
raise ValueError('global_integer_offsets cannot be used '
'simultaneously with rectwv_coeff')
# check headers to detect lamp status (on/off)
list_lampincd = []
for fname in rinput.obresult.frames:
with fname.open() as f:
list_lampincd.append(f[0].header['lampincd'])
# check number of images
nimages = len(rinput.obresult.frames)
n_on = list_lampincd.count(1)
n_off = list_lampincd.count(0)
self.logger.info(
'Number of images with lamp ON.........: {}'.format(n_on))
self.logger.info(
'Number of images with lamp OFF........: {}'.format(n_off))
self.logger.info(
'Total number of images................: {}'.format(nimages))
if n_on == 0:
raise ValueError('Insufficient number of images with lamp ON')
if n_on + n_off != nimages:
raise ValueError('Number of images does not match!')
# check combination method
if rinput.method_kwargs == {}:
method_kwargs = None
else:
if rinput.method == 'sigmaclip':
method_kwargs = rinput.method_kwargs
else:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# build object to proceed with bpm, bias, and dark (not flat)
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
# basic reduction of images with lamp ON or OFF
lampmode = {0: 'off', 1: 'on'}
reduced_image_on = None
reduced_image_off = None
for imode in lampmode.keys():
self.logger.info('starting basic reduction of images with'
' lamp {}'.format(lampmode[imode]))
tmplist = [
rinput.obresult.frames[i]
for i, lampincd in enumerate(list_lampincd)
if lampincd == imode
]
if len(tmplist) > 0:
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open()) for fname in tmplist
]
reduced_image = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
if imode == 0:
reduced_image_off = flow(reduced_image)
hdr = reduced_image_off[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_off,
'reduced_image_off.fits')
elif imode == 1:
reduced_image_on = flow(reduced_image)
hdr = reduced_image_on[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_on,
'reduced_image_on.fits')
else:
raise ValueError('Unexpected imode={}'.format(imode))
# computation of ON-OFF
header_on = reduced_image_on[0].header
data_on = reduced_image_on[0].data.astype('float32')
if n_off > 0:
header_off = reduced_image_off[0].header
data_off = reduced_image_off[0].data.astype('float32')
else:
header_off = None
data_off = np.zeros_like(data_on)
reduced_data = data_on - data_off
# update reduced image header
reduced_image = self.create_reduced_image(rinput, reduced_data,
header_on, header_off,
list_lampincd)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# define rectification and wavelength calibration coefficients
if rinput.rectwv_coeff is None:
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
else:
rectwv_coeff = rinput.rectwv_coeff
# save as JSON in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# apply global offsets (to both, the original and the cleaned version)
image2d = apply_integer_offsets(
image2d=reduced_data,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix)
# load CSU configuration
csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header)
# determine (pseudo) longslits
dict_longslits = csu_conf.pseudo_longslits()
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
self.logger.info('-> Removing slitlet (not defined): ' + str(idel))
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf.csu_bar_slit_width(islitlet)
if (slitwidth < rinput.minimum_slitlet_width_mm) or \
(slitwidth > rinput.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
self.logger.info('-> Removing slitlet (width out of range): ' +
str(islitlet))
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
grism_name = rectwv_coeff.tags['grism']
filter_name = rectwv_coeff.tags['filter']
cout = '0'
debugplot = rinput.debugplot
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=debugplot)
if abs(slt.debugplot) > 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d,
subtitle='original image')
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=2,
subtitle='original (cleaned) rectified')
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
# get useful slitlet region (use boundaries)
spectrail = slt.list_spectrails[0]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig
spectrail = slt.list_spectrails[2]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :],
axis=0)
# smooth median spectrum along the spectral direction
sp_median = ndimage.median_filter(sp_collapsed,
5,
mode='nearest')
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * rinput.minimum_fraction
lremove = np.where(sp_median < y_threshold)
sp_median[lremove] = 0.0
if abs(slt.debugplot) % 10 != 0:
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1,
sp_collapsed,
title=title,
show=False,
**{'label': 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d],
2 * [y_threshold],
label='threshold')
# ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian)
pause_debugplot(slt.debugplot,
pltshow=True,
tight_layout=True)
# generate rectified slitlet region filled with the
# median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median,
(naxis2_slitlet2d, 1))
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum')
# compute smooth surface
# clipped region
slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy()
slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0
slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0
# unrectified clipped image
slitlet2d_unrect_clipped = slt.rectify(
slitlet2d=slitlet2d_rect_clipped,
resampling=2,
inverse=True,
subtitle='unrectified, filled with median spectrum '
'(clipped)')
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm_clipped = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_clipped[i, j]
if den == 0:
slitlet2d_norm_clipped[i, j] = 1.0
else:
slitlet2d_norm_clipped[i, j] = \
slitlet2d[i, j] / den
# set to 1.0 one additional pixel at each side (since
# 'den' above is small at the borders and generates wrong
# bright pixels)
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_smooth = ndimage.median_filter(
slitlet2d_norm_clipped,
size=(rinput.nwindow_y_median, rinput.nwindow_x_median),
mode='nearest')
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_clipped,
subtitle='unrectified, pixel-to-pixel (clipped)')
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_smooth,
subtitle='unrectified, pixel-to-pixel (smoothed)')
# ---
# check for (pseudo) longslit with previous and next slitlet
imin = dict_longslits[islitlet].imin()
imax = dict_longslits[islitlet].imax()
if islitlet > 1:
same_slitlet_below = (islitlet - 1) >= imin
else:
same_slitlet_below = False
if islitlet < EMIR_NBARS:
same_slitlet_above = (islitlet + 1) <= imax
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm_smooth[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
cout += '.'
else:
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
self.logger.info(cout)
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix)
# set pixels below minimum value to 1.0
filtered = np.where(
image2d_flatfielded < rinput.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(
image2d_flatfielded > rinput.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# update image header
reduced_flatlowfreq = self.create_reduced_image(
rinput, image2d_flatfielded, header_on, header_off, list_lampincd)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# save results in results directory
self.logger.info('end of flatlowfreq generation')
result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq)
return result
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser(
description='description: compute pixel-to-pixel flatfield'
)
# required arguments
parser.add_argument("fitsfile",
help="Input FITS file (flat ON-OFF)",
type=argparse.FileType('rb'))
parser.add_argument("--rectwv_coeff", required=True,
help="Input JSON file with rectification and "
"wavelength calibration coefficients",
type=argparse.FileType('rt'))
parser.add_argument("--minimum_slitlet_width_mm", required=True,
help="Minimum slitlet width in mm",
type=float)
parser.add_argument("--maximum_slitlet_width_mm", required=True,
help="Maximum slitlet width in mm",
type=float)
parser.add_argument("--minimum_fraction", required=True,
help="Minimum allowed flatfielding value",
type=float, default=0.01)
parser.add_argument("--minimum_value_in_output",
help="Minimum value allowed in output file: pixels "
"below this value are set to 1.0 (default=0.01)",
type=float, default=0.01)
parser.add_argument("--maximum_value_in_output",
help="Maximum value allowed in output file: pixels "
"above this value are set to 1.0 (default=10.0)",
type=float, default=10.0)
# parser.add_argument("--nwindow_median", required=True,
# help="Window size to smooth median spectrum in the "
# "spectral direction",
# type=int)
parser.add_argument("--outfile", required=True,
help="Output FITS file",
type=lambda x: arg_file_is_new(parser, x, mode='wb'))
# optional arguments
parser.add_argument("--delta_global_integer_offset_x_pix",
help="Delta global integer offset in the X direction "
"(default=0)",
default=0, type=int)
parser.add_argument("--delta_global_integer_offset_y_pix",
help="Delta global integer offset in the Y direction "
"(default=0)",
default=0, type=int)
parser.add_argument("--resampling",
help="Resampling method: 1 -> nearest neighbor, "
"2 -> linear interpolation (default)",
default=2, type=int,
choices=(1, 2))
parser.add_argument("--ignore_DTUconf",
help="Ignore DTU configurations differences between "
"model and input image",
action="store_true")
parser.add_argument("--debugplot",
help="Integer indicating plotting & debugging options"
" (default=0)",
default=0, type=int,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args(args)
if args.echo:
print('\033[1m\033[31m% ' + ' '.join(sys.argv) + '\033[0m\n')
# read calibration structure from JSON file
rectwv_coeff = RectWaveCoeff._datatype_load(args.rectwv_coeff.name)
# modify (when requested) global offsets
rectwv_coeff.global_integer_offset_x_pix += \
args.delta_global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix += \
args.delta_global_integer_offset_y_pix
# read FITS image and its corresponding header
hdulist = fits.open(args.fitsfile)
header = hdulist[0].header
image2d = hdulist[0].data
hdulist.close()
# apply global offsets
image2d = apply_integer_offsets(
image2d=image2d,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix
)
# protections
naxis2, naxis1 = image2d.shape
if naxis1 != header['naxis1'] or naxis2 != header['naxis2']:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
raise ValueError('Something is wrong with NAXIS1 and/or NAXIS2')
if abs(args.debugplot) >= 10:
print('>>> NAXIS1:', naxis1)
print('>>> NAXIS2:', naxis2)
# check that the input FITS file grism and filter match
filter_name = header['filter']
if filter_name != rectwv_coeff.tags['filter']:
raise ValueError("Filter name does not match!")
grism_name = header['grism']
if grism_name != rectwv_coeff.tags['grism']:
raise ValueError("Filter name does not match!")
if abs(args.debugplot) >= 10:
print('>>> grism.......:', grism_name)
print('>>> filter......:', filter_name)
# check that the DTU configurations are compatible
dtu_conf_fitsfile = DtuConfiguration.define_from_fits(args.fitsfile)
dtu_conf_jsonfile = DtuConfiguration.define_from_dictionary(
rectwv_coeff.meta_info['dtu_configuration'])
if dtu_conf_fitsfile != dtu_conf_jsonfile:
print('DTU configuration (FITS file):\n\t', dtu_conf_fitsfile)
print('DTU configuration (JSON file):\n\t', dtu_conf_jsonfile)
if args.ignore_DTUconf:
print('WARNING: DTU configuration differences found!')
else:
raise ValueError('DTU configurations do not match')
else:
if abs(args.debugplot) >= 10:
print('>>> DTU Configuration match!')
print(dtu_conf_fitsfile)
# load CSU configuration
csu_conf_fitsfile = CsuConfiguration.define_from_fits(args.fitsfile)
if abs(args.debugplot) >= 10:
print(csu_conf_fitsfile)
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
print('-> Removing slitlet (not defined):', idel)
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
if (slitwidth < args.minimum_slitlet_width_mm) or \
(slitwidth > args.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
print('-> Removing slitlet (invalid width):', islitlet)
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
print('>>> valid slitlet numbers:\n', list_valid_islitlets)
# ---
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=False)
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=args.debugplot)
if abs(args.debugplot) >= 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(
image_2k2k=image2d,
subtitle='original image'
)
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=args.resampling,
subtitle='original rectified'
)
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
# get useful slitlet region (use boundaries instead of frontiers;
# note that the nscan_minmax_frontiers() works well independently
# of using frontiers of boundaries as arguments)
nscan_min, nscan_max = nscan_minmax_frontiers(
slt.y0_reference_lower,
slt.y0_reference_upper,
resize=False
)
ii1 = nscan_min - slt.bb_ns1_orig
ii2 = nscan_max - slt.bb_ns1_orig + 1
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :], axis=0)
# smooth median spectrum along the spectral direction
# sp_median = ndimage.median_filter(
# sp_collapsed,
# args.nwindow_median,
# mode='nearest'
# )
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
nremove = 5
spl = AdaptiveLSQUnivariateSpline(
x=xaxis1[nremove:-nremove],
y=sp_collapsed[nremove:-nremove],
t=11,
adaptive=True
)
xknots = spl.get_knots()
yknots = spl(xknots)
sp_median = spl(xaxis1)
# compute rms within each knot interval
nknots = len(xknots)
rms_array = np.zeros(nknots - 1, dtype=float)
for iknot in range(nknots - 1):
residuals = []
for xdum, ydum, yydum in \
zip(xaxis1, sp_collapsed, sp_median):
if xknots[iknot] <= xdum <= xknots[iknot + 1]:
residuals.append(abs(ydum - yydum))
if len(residuals) > 5:
rms_array[iknot] = np.std(residuals)
else:
rms_array[iknot] = 0
# determine in which knot interval falls each pixel
iknot_array = np.zeros(len(xaxis1), dtype=int)
for idum, xdum in enumerate(xaxis1):
for iknot in range(nknots - 1):
if xknots[iknot] <= xdum <= xknots[iknot + 1]:
iknot_array[idum] = iknot
# compute new fit removing deviant points (with fixed knots)
xnewfit = []
ynewfit = []
for idum in range(len(xaxis1)):
delta_sp = abs(sp_collapsed[idum] - sp_median[idum])
rms_tmp = rms_array[iknot_array[idum]]
if idum == 0 or idum == (len(xaxis1) - 1):
lok = True
elif rms_tmp > 0:
if delta_sp < 3.0 * rms_tmp:
lok = True
else:
lok = False
else:
lok = True
if lok:
xnewfit.append(xaxis1[idum])
ynewfit.append(sp_collapsed[idum])
nremove = 5
splnew = AdaptiveLSQUnivariateSpline(
x=xnewfit[nremove:-nremove],
y=ynewfit[nremove:-nremove],
t=xknots[1:-1],
adaptive=False
)
sp_median = splnew(xaxis1)
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * args.minimum_fraction
sp_median[np.where(sp_median < y_threshold)] = 0.0
if abs(args.debugplot) > 10:
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1, sp_collapsed,
title=title,
show=False, **{'label' : 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d], 2*[y_threshold],
label='threshold')
ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05*ymax_spmedian, 1.05*ymax_spmedian)
pause_debugplot(args.debugplot,
pltshow=True, tight_layout=True)
# generate rectified slitlet region filled with the median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median, (naxis2_slitlet2d, 1))
if abs(args.debugplot) > 10:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum'
)
# unrectified image
slitlet2d_unrect_spmedian = slt.rectify(
slitlet2d=slitlet2d_rect_spmedian,
resampling=args.resampling,
inverse=True,
subtitle='unrectified, filled with median spectrum'
)
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_spmedian[i, j]
if den == 0:
slitlet2d_norm[i, j] = 1.0
else:
slitlet2d_norm[i, j] = slitlet2d[i, j] / den
if abs(args.debugplot) > 10:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm,
subtitle='unrectified, pixel-to-pixel'
)
# check for pseudo-longslit with previous slitlet
if islitlet > 1:
if (islitlet - 1) in list_valid_islitlets:
c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet - 1)
w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet - 1)
c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet)
w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
if abs(w1-w2)/w1 < 0.25:
wmean = (w1 + w2) / 2.0
if abs(c1 - c2) < wmean/4.0:
same_slitlet_below = True
else:
same_slitlet_below = False
else:
same_slitlet_below = False
else:
same_slitlet_below = False
else:
same_slitlet_below = False
# check for pseudo-longslit with previous slitlet
if islitlet < EMIR_NBARS:
if (islitlet + 1) in list_valid_islitlets:
c1 = csu_conf_fitsfile.csu_bar_slit_center(islitlet)
w1 = csu_conf_fitsfile.csu_bar_slit_width(islitlet)
c2 = csu_conf_fitsfile.csu_bar_slit_center(islitlet + 1)
w2 = csu_conf_fitsfile.csu_bar_slit_width(islitlet + 1)
if abs(w1-w2)/w1 < 0.25:
wmean = (w1 + w2) / 2.0
if abs(c1 - c2) < wmean/4.0:
same_slitlet_above = True
else:
same_slitlet_above = False
else:
same_slitlet_above = False
else:
same_slitlet_above = False
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
else:
if args.debugplot == 0:
islitlet_progress(islitlet, EMIR_NBARS, ignore=True)
if args.debugplot == 0:
print('OK!')
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded ,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix
)
# set pixels below minimum value to 1.0
filtered = np.where(image2d_flatfielded < args.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(image2d_flatfielded > args.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# save output file
save_ndarray_to_fits(
array=image2d_flatfielded,
file_name=args.outfile,
main_header=header,
overwrite=True
)
print('>>> Saving file ' + args.outfile.name)
def rectwv_coeff_from_mos_library(reduced_image,
master_rectwv,
ignore_dtu_configuration=True,
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from MOS library
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
master_rectwv : MasterRectWave instance
Rectification and Wavelength Calibrartion Library product.
Contains the library of polynomial coefficients necessary
to generate an instance of RectWaveCoeff with the rectification
and wavelength calibration coefficients for the particular
CSU configuration.
ignore_dtu_configuration : bool
If True, ignore differences in DTU configuration.
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
"""
logger = logging.getLogger(__name__)
logger.info('Computing expected RectWaveCoeff from CSU configuration')
# header
header = reduced_image[0].header
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
# retrieve DTU configuration from MasterRectWave object
dtu_conf_calib = DtuConfiguration.define_from_dictionary(
master_rectwv.meta_info['dtu_configuration']
)
# check that the DTU configuration employed to obtain the calibration
# corresponds to the DTU configuration in the input FITS file
if dtu_conf != dtu_conf_calib:
if ignore_dtu_configuration:
logger.warning('DTU configuration differences found!')
else:
logger.info('DTU configuration from image header:')
logger.info(dtu_conf)
logger.info('DTU configuration from master calibration:')
logger.info(dtu_conf_calib)
raise ValueError("DTU configurations do not match!")
else:
logger.info('DTU configuration match!')
# check grism and filter
filter_name = header['filter']
logger.debug('Filter: ' + filter_name)
if filter_name != master_rectwv.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.debug('Grism: ' + grism_name)
if grism_name != master_rectwv.tags['grism']:
raise ValueError('Grism name does not match!')
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in master_rectwv.missing_slitlets:
list_valid_islitlets.remove(idel)
logger.debug('valid slitlet numbers: ' + str(list_valid_islitlets))
# initialize intermediate dictionary with relevant information
# (note: this dictionary corresponds to an old structure employed to
# store the information in a JSON file; this is no longer necessary,
# but here we reuse that dictionary for convenience)
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration from a MOS model '
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['fits_frame_uuid'] = 'TBD'
outdict['meta_info']['origin']['rect_wpoly_mos_uuid'] = \
master_rectwv.uuid
outdict['meta_info']['origin']['fitted_boundary_param_uuid'] = \
master_rectwv.meta_info['origin']['bound_param']
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
# compute rectification and wavelength calibration coefficients for each
# slitlet according to its csu_bar_slit_center value
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# input data structure
tmpdict = master_rectwv.contents[islitlet - 1]
list_csu_bar_slit_center = tmpdict['list_csu_bar_slit_center']
# check extrapolations
if csu_bar_slit_center < min(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('minimum tabulated value: ' +
str(min(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
if csu_bar_slit_center > max(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('maximum tabulated value: ' +
str(max(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
# rectification coefficients
ttd_order = tmpdict['ttd_order']
ncoef = ncoef_fmap(ttd_order)
outdict['contents'][cslitlet] = {}
outdict['contents'][cslitlet]['ttd_order'] = ttd_order
outdict['contents'][cslitlet]['ttd_order_longslit_model'] = None
for keycoef in ['ttd_aij', 'ttd_bij', 'tti_aij', 'tti_bij']:
coef_out = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_' + keycoef + '_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
coef_out.append(dum[0])
outdict['contents'][cslitlet][keycoef] = coef_out
outdict['contents'][cslitlet][keycoef + '_longslit_model'] = None
# wavelength calibration coefficients
ncoef = tmpdict['wpoly_degree'] + 1
wpoly_coeff = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_wpoly_coeff_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
wpoly_coeff.append(dum[0])
outdict['contents'][cslitlet]['wpoly_coeff'] = wpoly_coeff
outdict['contents'][cslitlet]['wpoly_coeff_longslit_model'] = None
# update cdelt1_linear and crval1_linear
wpoly_function = np.polynomial.Polynomial(wpoly_coeff)
crmin1_linear = wpoly_function(1)
crmax1_linear = wpoly_function(EMIR_NAXIS1)
cdelt1_linear = (crmax1_linear - crmin1_linear) / (EMIR_NAXIS1 - 1)
crval1_linear = crmin1_linear
outdict['contents'][cslitlet]['crval1_linear'] = crval1_linear
outdict['contents'][cslitlet]['cdelt1_linear'] = cdelt1_linear
# update CSU keywords
outdict['contents'][cslitlet]['csu_bar_left'] = \
csu_conf.csu_bar_left(islitlet)
outdict['contents'][cslitlet]['csu_bar_right'] = \
csu_conf.csu_bar_right(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_center'] = \
csu_conf.csu_bar_slit_center(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_width'] = \
csu_conf.csu_bar_slit_width(islitlet)
# for each slitlet compute spectrum trails and frontiers using the
# fitted boundary parameters
fitted_bound_param_json = {
'contents': master_rectwv.meta_info['refined_boundary_model']
}
parmodel = fitted_bound_param_json['contents']['parmodel']
fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}})
params = bound_params_from_dict(fitted_bound_param_json)
if abs(debugplot) >= 10:
logger.debug('Fitted boundary parameters:')
logger.debug(params.pretty_print())
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# compute and store x0_reference value
x0_reference = float(EMIR_NAXIS1) / 2.0 + 0.5
outdict['contents'][cslitlet]['x0_reference'] = x0_reference
# compute spectrum trails (lower, middle and upper)
list_spectrails = expected_distorted_boundaries(
islitlet, csu_bar_slit_center,
[0, 0.5, 1], params, parmodel,
numpts=101, deg=5, debugplot=0
)
# store spectrails in output JSON file
outdict['contents'][cslitlet]['spectrail'] = {}
for idum, cdum in zip(range(3), ['lower', 'middle', 'upper']):
outdict['contents'][cslitlet]['spectrail']['poly_coef_' + cdum] = \
list_spectrails[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_reference_' + cdum] = \
list_spectrails[idum].poly_funct(x0_reference)
# compute frontiers (lower, upper)
list_frontiers = expected_distorted_frontiers(
islitlet, csu_bar_slit_center,
params, parmodel,
numpts=101, deg=5, debugplot=0
)
# store frontiers in output JSON
outdict['contents'][cslitlet]['frontier'] = {}
for idum, cdum in zip(range(2), ['lower', 'upper']):
outdict['contents'][cslitlet]['frontier']['poly_coef_' + cdum] = \
list_frontiers[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_frontier_' + cdum] = \
list_frontiers[idum].poly_funct(x0_reference)
# store bounding box parameters for each slitlet
xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# parameters already available in the input JSON file
for par in ['bb_nc1_orig', 'bb_nc2_orig', 'ymargin_bb']:
outdict['contents'][cslitlet][par] = \
master_rectwv.contents[islitlet - 1][par]
# estimate bb_ns1_orig and bb_ns2_orig using the already computed
# frontiers and the value of ymargin_bb, following the same approach
# employed in Slitlet2dArc.__init__()
poly_lower_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_lower']
)
poly_upper_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_upper']
)
ylower = poly_lower_frontier(xdum)
yupper = poly_upper_frontier(xdum)
ymargin_bb = master_rectwv.contents[islitlet - 1]['ymargin_bb']
bb_ns1_orig = int(ylower.min() + 0.5) - ymargin_bb
if bb_ns1_orig < 1:
bb_ns1_orig = 1
bb_ns2_orig = int(yupper.max() + 0.5) + ymargin_bb
if bb_ns2_orig > EMIR_NAXIS2:
bb_ns2_orig = EMIR_NAXIS2
outdict['contents'][cslitlet]['bb_ns1_orig'] = bb_ns1_orig
outdict['contents'][cslitlet]['bb_ns2_orig'] = bb_ns2_orig
# additional parameters (see Slitlet2dArc.__init__)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# define expected frontier ordinates at x0_reference for the rectified
# image imposing the vertical length of the slitlet to be constant
# and equal to EMIR_NPIXPERSLIT_RECTIFIED
outdict['contents'][cslitlet]['y0_frontier_lower_expected'] = \
expected_y0_lower_frontier(islitlet)
outdict['contents'][cslitlet]['y0_frontier_upper_expected'] = \
expected_y0_upper_frontier(islitlet)
# compute linear transformation to place the rectified slitlet at
# the center of the current slitlet bounding box
tmpdict = outdict['contents'][cslitlet]
xdum1 = tmpdict['y0_frontier_lower']
ydum1 = tmpdict['y0_frontier_lower_expected']
xdum2 = tmpdict['y0_frontier_upper']
ydum2 = tmpdict['y0_frontier_upper_expected']
corr_yrect_b = (ydum2 - ydum1) / (xdum2 - xdum1)
corr_yrect_a = ydum1 - corr_yrect_b * xdum1
# compute expected location of rectified boundaries
y0_reference_lower_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_lower']
y0_reference_middle_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_middle']
y0_reference_upper_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_upper']
# shift transformation to center the rectified slitlet within the
# slitlet bounding box
ydummid = (ydum1 + ydum2) / 2
ioffset = int(
ydummid - (tmpdict['bb_ns1_orig'] + tmpdict['bb_ns2_orig']) / 2.0)
corr_yrect_a -= ioffset
# minimum and maximum row in the rectified slitlet encompassing
# EMIR_NPIXPERSLIT_RECTIFIED pixels
# a) scan number (in pixels, from 1 to NAXIS2)
xdum1 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_lower']
xdum2 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_upper']
# b) row number (starting from zero)
min_row_rectified = \
int((round(xdum1 * 10) + 5) / 10) - tmpdict['bb_ns1_orig']
max_row_rectified = \
int((round(xdum2 * 10) - 5) / 10) - tmpdict['bb_ns1_orig']
# save previous results in outdict
outdict['contents'][cslitlet]['y0_reference_lower_expected'] = \
y0_reference_lower_expected
outdict['contents'][cslitlet]['y0_reference_middle_expected'] = \
y0_reference_middle_expected
outdict['contents'][cslitlet]['y0_reference_upper_expected'] = \
y0_reference_upper_expected
outdict['contents'][cslitlet]['corr_yrect_a'] = corr_yrect_a
outdict['contents'][cslitlet]['corr_yrect_b'] = corr_yrect_b
outdict['contents'][cslitlet]['min_row_rectified'] = min_row_rectified
outdict['contents'][cslitlet]['max_row_rectified'] = max_row_rectified
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
master_rectwv.meta_info['origin']['bound_param']
rectwv_coeff.meta_info['origin']['master_rectwv'] = \
'uuid' + master_rectwv.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
dumdict.update({
'csu_bar_left': csu_conf.csu_bar_left(islitlet),
'csu_bar_right': csu_conf.csu_bar_right(islitlet),
'csu_bar_slit_center': csu_conf.csu_bar_slit_center(islitlet),
'csu_bar_slit_width': csu_conf.csu_bar_slit_width(islitlet),
'x0_reference': float(EMIR_NAXIS1) / 2.0 + 0.5,
'y0_frontier_lower_expected':
expected_y0_lower_frontier(islitlet),
'y0_frontier_upper_expected':
expected_y0_upper_frontier(islitlet)
})
rectwv_coeff.missing_slitlets.append(islitlet)
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_rect_wpoly.name)
# print('>>> Saving file ' + args.out_rect_wpoly.name)
# check_setstate_getstate(rectwv_coeff, args.out_rect_wpoly.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff
def rectwv_coeff_from_mos_library(reduced_image,
master_rectwv,
ignore_dtu_configuration=True,
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from MOS library
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
master_rectwv : MasterRectWave instance
Rectification and Wavelength Calibrartion Library product.
Contains the library of polynomial coefficients necessary
to generate an instance of RectWaveCoeff with the rectification
and wavelength calibration coefficients for the particular
CSU configuration.
ignore_dtu_configuration : bool
If True, ignore differences in DTU configuration.
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
"""
logger = logging.getLogger(__name__)
logger.info('Computing expected RectWaveCoeff from CSU configuration')
# header
header = reduced_image[0].header
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
# retrieve DTU configuration from MasterRectWave object
dtu_conf_calib = DtuConfiguration.define_from_dictionary(
master_rectwv.meta_info['dtu_configuration'])
# check that the DTU configuration employed to obtain the calibration
# corresponds to the DTU configuration in the input FITS file
if dtu_conf != dtu_conf_calib:
if ignore_dtu_configuration:
logger.warning('DTU configuration differences found!')
else:
logger.info('DTU configuration from image header:')
logger.info(dtu_conf)
logger.info('DTU configuration from master calibration:')
logger.info(dtu_conf_calib)
raise ValueError("DTU configurations do not match!")
else:
logger.info('DTU configuration match!')
# check grism and filter
filter_name = header['filter']
logger.debug('Filter: ' + filter_name)
if filter_name != master_rectwv.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.debug('Grism: ' + grism_name)
if grism_name != master_rectwv.tags['grism']:
raise ValueError('Grism name does not match!')
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in master_rectwv.missing_slitlets:
list_valid_islitlets.remove(idel)
logger.debug('valid slitlet numbers: ' + str(list_valid_islitlets))
# initialize intermediate dictionary with relevant information
# (note: this dictionary corresponds to an old structure employed to
# store the information in a JSON file; this is no longer necessary,
# but here we reuse that dictionary for convenience)
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration from a MOS model '
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['fits_frame_uuid'] = 'TBD'
outdict['meta_info']['origin']['rect_wpoly_mos_uuid'] = \
master_rectwv.uuid
outdict['meta_info']['origin']['fitted_boundary_param_uuid'] = \
master_rectwv.meta_info['origin']['bound_param']
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
# compute rectification and wavelength calibration coefficients for each
# slitlet according to its csu_bar_slit_center value
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# input data structure
tmpdict = master_rectwv.contents[islitlet - 1]
list_csu_bar_slit_center = tmpdict['list_csu_bar_slit_center']
# check extrapolations
if csu_bar_slit_center < min(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('minimum tabulated value: ' +
str(min(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
if csu_bar_slit_center > max(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('maximum tabulated value: ' +
str(max(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
# rectification coefficients
ttd_order = tmpdict['ttd_order']
ncoef = ncoef_fmap(ttd_order)
outdict['contents'][cslitlet] = {}
outdict['contents'][cslitlet]['ttd_order'] = ttd_order
outdict['contents'][cslitlet]['ttd_order_longslit_model'] = None
for keycoef in ['ttd_aij', 'ttd_bij', 'tti_aij', 'tti_bij']:
coef_out = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_' + keycoef + '_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
coef_out.append(dum[0])
outdict['contents'][cslitlet][keycoef] = coef_out
outdict['contents'][cslitlet][keycoef + '_longslit_model'] = None
# wavelength calibration coefficients
ncoef = tmpdict['wpoly_degree'] + 1
wpoly_coeff = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_wpoly_coeff_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
wpoly_coeff.append(dum[0])
outdict['contents'][cslitlet]['wpoly_coeff'] = wpoly_coeff
outdict['contents'][cslitlet]['wpoly_coeff_longslit_model'] = None
# update cdelt1_linear and crval1_linear
wpoly_function = np.polynomial.Polynomial(wpoly_coeff)
crmin1_linear = wpoly_function(1)
crmax1_linear = wpoly_function(EMIR_NAXIS1)
cdelt1_linear = (crmax1_linear - crmin1_linear) / (EMIR_NAXIS1 - 1)
crval1_linear = crmin1_linear
outdict['contents'][cslitlet]['crval1_linear'] = crval1_linear
outdict['contents'][cslitlet]['cdelt1_linear'] = cdelt1_linear
# update CSU keywords
outdict['contents'][cslitlet]['csu_bar_left'] = \
csu_conf.csu_bar_left(islitlet)
outdict['contents'][cslitlet]['csu_bar_right'] = \
csu_conf.csu_bar_right(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_center'] = \
csu_conf.csu_bar_slit_center(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_width'] = \
csu_conf.csu_bar_slit_width(islitlet)
# for each slitlet compute spectrum trails and frontiers using the
# fitted boundary parameters
fitted_bound_param_json = {
'contents': master_rectwv.meta_info['refined_boundary_model']
}
parmodel = fitted_bound_param_json['contents']['parmodel']
fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}})
params = bound_params_from_dict(fitted_bound_param_json)
if abs(debugplot) >= 10:
logger.debug('Fitted boundary parameters:')
logger.debug(params.pretty_print())
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# compute and store x0_reference value
x0_reference = float(EMIR_NAXIS1) / 2.0 + 0.5
outdict['contents'][cslitlet]['x0_reference'] = x0_reference
# compute spectrum trails (lower, middle and upper)
list_spectrails = expected_distorted_boundaries(islitlet,
csu_bar_slit_center,
[0, 0.5, 1],
params,
parmodel,
numpts=101,
deg=5,
debugplot=0)
# store spectrails in output JSON file
outdict['contents'][cslitlet]['spectrail'] = {}
for idum, cdum in zip(range(3), ['lower', 'middle', 'upper']):
outdict['contents'][cslitlet]['spectrail']['poly_coef_' + cdum] = \
list_spectrails[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_reference_' + cdum] = \
list_spectrails[idum].poly_funct(x0_reference)
# compute frontiers (lower, upper)
list_frontiers = expected_distorted_frontiers(islitlet,
csu_bar_slit_center,
params,
parmodel,
numpts=101,
deg=5,
debugplot=0)
# store frontiers in output JSON
outdict['contents'][cslitlet]['frontier'] = {}
for idum, cdum in zip(range(2), ['lower', 'upper']):
outdict['contents'][cslitlet]['frontier']['poly_coef_' + cdum] = \
list_frontiers[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_frontier_' + cdum] = \
list_frontiers[idum].poly_funct(x0_reference)
# store bounding box parameters for each slitlet
xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# parameters already available in the input JSON file
for par in ['bb_nc1_orig', 'bb_nc2_orig', 'ymargin_bb']:
outdict['contents'][cslitlet][par] = \
master_rectwv.contents[islitlet - 1][par]
# estimate bb_ns1_orig and bb_ns2_orig using the already computed
# frontiers and the value of ymargin_bb, following the same approach
# employed in Slitlet2dArc.__init__()
poly_lower_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_lower'])
poly_upper_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_upper'])
ylower = poly_lower_frontier(xdum)
yupper = poly_upper_frontier(xdum)
ymargin_bb = master_rectwv.contents[islitlet - 1]['ymargin_bb']
bb_ns1_orig = int(ylower.min() + 0.5) - ymargin_bb
if bb_ns1_orig < 1:
bb_ns1_orig = 1
bb_ns2_orig = int(yupper.max() + 0.5) + ymargin_bb
if bb_ns2_orig > EMIR_NAXIS2:
bb_ns2_orig = EMIR_NAXIS2
outdict['contents'][cslitlet]['bb_ns1_orig'] = bb_ns1_orig
outdict['contents'][cslitlet]['bb_ns2_orig'] = bb_ns2_orig
# additional parameters (see Slitlet2dArc.__init__)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# define expected frontier ordinates at x0_reference for the rectified
# image imposing the vertical length of the slitlet to be constant
# and equal to EMIR_NPIXPERSLIT_RECTIFIED
outdict['contents'][cslitlet]['y0_frontier_lower_expected'] = \
expected_y0_lower_frontier(islitlet)
outdict['contents'][cslitlet]['y0_frontier_upper_expected'] = \
expected_y0_upper_frontier(islitlet)
# compute linear transformation to place the rectified slitlet at
# the center of the current slitlet bounding box
tmpdict = outdict['contents'][cslitlet]
xdum1 = tmpdict['y0_frontier_lower']
ydum1 = tmpdict['y0_frontier_lower_expected']
xdum2 = tmpdict['y0_frontier_upper']
ydum2 = tmpdict['y0_frontier_upper_expected']
corr_yrect_b = (ydum2 - ydum1) / (xdum2 - xdum1)
corr_yrect_a = ydum1 - corr_yrect_b * xdum1
# compute expected location of rectified boundaries
y0_reference_lower_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_lower']
y0_reference_middle_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_middle']
y0_reference_upper_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_upper']
# shift transformation to center the rectified slitlet within the
# slitlet bounding box
ydummid = (ydum1 + ydum2) / 2
ioffset = int(ydummid -
(tmpdict['bb_ns1_orig'] + tmpdict['bb_ns2_orig']) / 2.0)
corr_yrect_a -= ioffset
# minimum and maximum row in the rectified slitlet encompassing
# EMIR_NPIXPERSLIT_RECTIFIED pixels
# a) scan number (in pixels, from 1 to NAXIS2)
xdum1 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_lower']
xdum2 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_upper']
# b) row number (starting from zero)
min_row_rectified = \
int((round(xdum1 * 10) + 5) / 10) - tmpdict['bb_ns1_orig']
max_row_rectified = \
int((round(xdum2 * 10) - 5) / 10) - tmpdict['bb_ns1_orig']
# save previous results in outdict
outdict['contents'][cslitlet]['y0_reference_lower_expected'] = \
y0_reference_lower_expected
outdict['contents'][cslitlet]['y0_reference_middle_expected'] = \
y0_reference_middle_expected
outdict['contents'][cslitlet]['y0_reference_upper_expected'] = \
y0_reference_upper_expected
outdict['contents'][cslitlet]['corr_yrect_a'] = corr_yrect_a
outdict['contents'][cslitlet]['corr_yrect_b'] = corr_yrect_b
outdict['contents'][cslitlet]['min_row_rectified'] = min_row_rectified
outdict['contents'][cslitlet]['max_row_rectified'] = max_row_rectified
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
master_rectwv.meta_info['origin']['bound_param']
rectwv_coeff.meta_info['origin']['master_rectwv'] = \
'uuid' + master_rectwv.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
dumdict.update({
'csu_bar_left':
csu_conf.csu_bar_left(islitlet),
'csu_bar_right':
csu_conf.csu_bar_right(islitlet),
'csu_bar_slit_center':
csu_conf.csu_bar_slit_center(islitlet),
'csu_bar_slit_width':
csu_conf.csu_bar_slit_width(islitlet),
'x0_reference':
float(EMIR_NAXIS1) / 2.0 + 0.5,
'y0_frontier_lower_expected':
expected_y0_lower_frontier(islitlet),
'y0_frontier_upper_expected':
expected_y0_upper_frontier(islitlet)
})
rectwv_coeff.missing_slitlets.append(islitlet)
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_rect_wpoly.name)
# print('>>> Saving file ' + args.out_rect_wpoly.name)
# check_setstate_getstate(rectwv_coeff, args.out_rect_wpoly.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff
def median_slitlets_rectified(
input_image,
mode=0,
minimum_slitlet_width_mm=EMIR_MINIMUM_SLITLET_WIDTH_MM,
maximum_slitlet_width_mm=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
debugplot=0):
"""Compute median spectrum for each slitlet.
Parameters
----------
input_image : HDUList object
Input 2D image.
mode : int
Indicate desired result:
0 : image with the same size as the input image, with the
median spectrum of each slitlet spanning all the spectra
of the corresponding slitlet
1 : image with 55 spectra, containing the median spectra of
each slitlet
2 : single collapsed median spectrum, using exclusively the
useful slitlets from the input image
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
image_median : HDUList object
Output image.
"""
image_header = input_image[0].header
image2d = input_image[0].data
# check image dimensions
naxis2_expected = EMIR_NBARS * EMIR_NPIXPERSLIT_RECTIFIED
naxis2, naxis1 = image2d.shape
if naxis2 != naxis2_expected:
raise ValueError("NAXIS2={0} should be {1}".format(
naxis2, naxis2_expected))
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# initialize output image
if mode == 0:
image2d_median = np.zeros((naxis2, naxis1))
else:
image2d_median = np.zeros((EMIR_NBARS, naxis1))
# main loop
for i in range(EMIR_NBARS):
ns1 = i * EMIR_NPIXPERSLIT_RECTIFIED + 1
ns2 = ns1 + EMIR_NPIXPERSLIT_RECTIFIED - 1
sp_median = np.median(image2d[(ns1 - 1):ns2, :], axis=0)
if mode == 0:
image2d_median[(ns1 - 1):ns2, :] = np.tile(
sp_median, (EMIR_NPIXPERSLIT_RECTIFIED, 1))
else:
image2d_median[i] = np.copy(sp_median)
if mode == 2:
# get CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_header(image_header)
# define wavelength calibration parameters
crpix1 = image_header['crpix1']
crval1 = image_header['crval1']
cdelt1 = image_header['cdelt1']
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
if abs(debugplot) != 0:
print('>>> list_useful_slitlets....:', list_useful_slitlets)
print('>>> list_not_useful_slitlets:', list_not_useful_slitlets)
# define mask from array data
mask2d, borders = define_mask_borders(image2d_median, sought_value=0)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int),
z1z2=(-.2, 1.2),
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# update mask with unused slitlets
for islitlet in list_not_useful_slitlets:
mask2d[islitlet - 1, :] = np.array([True] * naxis1)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int),
z1z2=(-.2, 1.2),
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# useful image pixels
image2d_masked = image2d_median * (1 - mask2d.astype(int))
if abs(debugplot) % 10 != 0:
ximshow(image2d_masked,
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# masked image
image2d_masked = np.ma.masked_array(image2d_median, mask=mask2d)
# median spectrum
image1d_median = np.ma.median(image2d_masked, axis=0).data
image_median = fits.PrimaryHDU(data=image1d_median,
header=image_header)
else:
image_median = fits.PrimaryHDU(data=image2d_median,
header=image_header)
return fits.HDUList([image_median])
def median_slitlets_rectified(
input_image,
mode=0,
minimum_slitlet_width_mm=EMIR_MINIMUM_SLITLET_WIDTH_MM,
maximum_slitlet_width_mm=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
debugplot=0
):
"""Compute median spectrum for each slitlet.
Parameters
----------
input_image : HDUList object
Input 2D image.
mode : int
Indicate desired result:
0 : image with the same size as the input image, with the
median spectrum of each slitlet spanning all the spectra
of the corresponding slitlet
1 : image with 55 spectra, containing the median spectra of
each slitlet
2 : single collapsed median spectrum, using exclusively the
useful slitlets from the input image
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
image_median : HDUList object
Output image.
"""
image_header = input_image[0].header
image2d = input_image[0].data
# check image dimensions
naxis2_expected = EMIR_NBARS * EMIR_NPIXPERSLIT_RECTIFIED
naxis2, naxis1 = image2d.shape
if naxis2 != naxis2_expected:
raise ValueError("NAXIS2={0} should be {1}".format(
naxis2, naxis2_expected
))
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# initialize output image
if mode == 0:
image2d_median = np.zeros((naxis2, naxis1))
else:
image2d_median = np.zeros((EMIR_NBARS, naxis1))
# main loop
for i in range(EMIR_NBARS):
ns1 = i * EMIR_NPIXPERSLIT_RECTIFIED + 1
ns2 = ns1 + EMIR_NPIXPERSLIT_RECTIFIED - 1
sp_median = np.median(image2d[(ns1-1):ns2, :], axis=0)
if mode == 0:
image2d_median[(ns1-1):ns2, :] = np.tile(
sp_median, (EMIR_NPIXPERSLIT_RECTIFIED, 1)
)
else:
image2d_median[i] = np.copy(sp_median)
if mode == 2:
# get CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_header(image_header)
# define wavelength calibration parameters
crpix1 = image_header['crpix1']
crval1 = image_header['crval1']
cdelt1 = image_header['cdelt1']
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm
)
list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets]
if abs(debugplot) != 0:
print('>>> list_useful_slitlets....:', list_useful_slitlets)
print('>>> list_not_useful_slitlets:', list_not_useful_slitlets)
# define mask from array data
mask2d, borders = define_mask_borders(image2d_median, sought_value=0)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int), z1z2=(-.2, 1.2), crpix1=crpix1,
crval1=crval1, cdelt1=cdelt1, debugplot=debugplot)
# update mask with unused slitlets
for islitlet in list_not_useful_slitlets:
mask2d[islitlet - 1, :] = np.array([True] * naxis1)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int), z1z2=(-.2, 1.2), crpix1=crpix1,
crval1=crval1, cdelt1=cdelt1, debugplot=debugplot)
# useful image pixels
image2d_masked = image2d_median * (1 - mask2d.astype(int))
if abs(debugplot) % 10 != 0:
ximshow(image2d_masked, crpix1=crpix1, crval1=crval1,
cdelt1=cdelt1, debugplot=debugplot)
# masked image
image2d_masked = np.ma.masked_array(image2d_median, mask=mask2d)
# median spectrum
image1d_median = np.ma.median(image2d_masked, axis=0).data
image_median = fits.PrimaryHDU(data=image1d_median,
header=image_header)
else:
image_median = fits.PrimaryHDU(data=image2d_median,
header=image_header)
return fits.HDUList([image_median])