def display_slitlet_histogram(csu_bar_slit_width,
n_clusters=2,
geometry=None,
debugplot=0):
"""
Find separations between groups of slitlet widths.
Parameters
----------
csu_bar_slit_width : numpy array
Array containing the csu_bar_slit_center values.
n_clusters : int
Number of slitlet groups to be sought.
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
TBD
"""
# protections
if n_clusters < 2:
raise ValueError("n_clusters must be >= 2")
# determine useful slitlets from their widths
km = KMeans(n_clusters=n_clusters)
fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1))
separator_list = []
for i in range(n_clusters - 1):
peak1 = fit.cluster_centers_[i][0]
peak2 = fit.cluster_centers_[i + 1][0]
separator = (peak1 + peak2) / 2
separator_list.append(separator)
print('---> separator: {0:7.3f}'.format(separator))
# display histogram
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
ax.hist(csu_bar_slit_width, bins=100)
ax.set_xlabel('slit width (mm)')
ax.set_ylabel('number of slitlets')
for separator in separator_list:
ax.axvline(separator, color='C1', linestyle='dashed')
pause_debugplot(debugplot, pltshow=True)
def display_slitlet_histogram(csu_bar_slit_width,
n_clusters=2,
geometry=None,
debugplot=0):
"""
Find separations between groups of slitlet widths.
Parameters
----------
csu_bar_slit_width : numpy array
Array containing the csu_bar_slit_center values.
n_clusters : int
Number of slitlet groups to be sought.
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
TBD
"""
# protections
if n_clusters < 2:
raise ValueError("n_clusters must be >= 2")
# determine useful slitlets from their widths
km = KMeans(n_clusters=n_clusters)
fit = km.fit(np.array(csu_bar_slit_width).reshape(-1, 1))
separator_list = []
for i in range(n_clusters - 1):
peak1 = fit.cluster_centers_[i][0]
peak2 = fit.cluster_centers_[i+1][0]
separator = (peak1 + peak2) / 2
separator_list.append(separator)
print('---> separator: {0:7.3f}'.format(separator))
# display histogram
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
ax.hist(csu_bar_slit_width, bins=100)
ax.set_xlabel('slit width (mm)')
ax.set_ylabel('number of slitlets')
for separator in separator_list:
ax.axvline(separator, color='C1', linestyle='dashed')
pause_debugplot(debugplot, pltshow=True)
def refine_rectwv_coeff(input_image,
rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'.format(
refine_wavecalib_mode))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data('emirdrp.instrument.configs',
'Oliva_etal_2013.dat')
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1 * lok2]
catlines_reference_flux = catlines_all_flux[lok1 * lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
logger.info('list of useful slitlets: {}'.format(list_useful_slitlets))
logger.info(
'list of not useful slitlets: {}'.format(list_not_useful_slitlets))
tempwidths = np.array([
csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets
])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave,
sp_median,
'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum',
show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave,
catlines_reference_flux,
'C4-',
markerfmt=' ',
basefmt='C4-',
label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference,
sp_reference,
'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset * cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot)
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i] * cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1 - offset_array[i] * cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(
stat_summary['robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot,
yplot,
linestyle='',
marker='o',
color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False,
**{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset,
linestyle='--',
color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image
def display_slitlet_arrangement(fileobj,
grism=None,
spfilter=None,
bbox=None,
adjust=None,
geometry=None,
debugplot=0):
"""Display slitlet arrangment from CSUP keywords in FITS header.
Parameters
----------
fileobj : file object
FITS or TXT file object.
grism : str
Grism.
grism : str
Filter.
bbox : tuple of 4 floats
If not None, values for xmin, xmax, ymin and ymax.
adjust : bool
Adjust X range according to minimum and maximum csu_bar_left
and csu_bar_right (note that this option is overriden by 'bbox')
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
csu_bar_left : list of floats
Location (mm) of the left bar for each slitlet.
csu_bar_right : list of floats
Location (mm) of the right bar for each slitlet, using the
same origin employed for csu_bar_left (which is not the
value stored in the FITS keywords.
csu_bar_slit_center : list of floats
Middle point (mm) in between the two bars defining a slitlet.
csu_bar_slit_width : list of floats
Slitlet width (mm), computed as the distance between the two
bars defining the slitlet.
"""
if fileobj.name[-4:] == ".txt":
if grism is None:
raise ValueError("Undefined grism!")
if spfilter is None:
raise ValueError("Undefined filter!")
# define CsuConfiguration object
csu_config = CsuConfiguration()
csu_config._csu_bar_left = []
csu_config._csu_bar_right = []
csu_config._csu_bar_slit_center = []
csu_config._csu_bar_slit_width = []
# since the input filename has been opened with argparse in binary
# mode, it is necessary to close it and open it in text mode
fileobj.close()
# read TXT file
with open(fileobj.name, mode='rt') as f:
file_content = f.read().splitlines()
next_id_bar = 1
for line in file_content:
if len(line) > 0:
if line[0] not in ['#']:
line_contents = line.split()
id_bar = int(line_contents[0])
position = float(line_contents[1])
if id_bar == next_id_bar:
if id_bar <= EMIR_NBARS:
csu_config._csu_bar_left.append(position)
next_id_bar = id_bar + EMIR_NBARS
else:
csu_config._csu_bar_right.append(341.5 - position)
next_id_bar = id_bar - EMIR_NBARS + 1
else:
raise ValueError("Unexpected id_bar:" + str(id_bar))
# compute slit width and center
for i in range(EMIR_NBARS):
csu_config._csu_bar_slit_center.append(
(csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i]) /
2)
csu_config._csu_bar_slit_width.append(
csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i])
else:
# read input FITS file
hdulist = fits.open(fileobj.name)
image_header = hdulist[0].header
hdulist.close()
# additional info from header
grism = image_header['grism']
spfilter = image_header['filter']
# define slitlet arrangement
csu_config = CsuConfiguration.define_from_fits(fileobj)
# determine calibration
if grism in ["J", "OPEN"] and spfilter == "J":
wv_parameters = set_wv_parameters("J", "J")
elif grism in ["H", "OPEN"] and spfilter == "H":
wv_parameters = set_wv_parameters("H", "H")
elif grism in ["K", "OPEN"] and spfilter == "Ksp":
wv_parameters = set_wv_parameters("Ksp", "K")
elif grism in ["LR", "OPEN"] and spfilter == "YJ":
wv_parameters = set_wv_parameters("YJ", "LR")
elif grism in ["LR", "OPEN"] and spfilter == "HK":
wv_parameters = set_wv_parameters("HK", "LR")
else:
raise ValueError("Invalid grism + filter configuration")
crval1 = wv_parameters['poly_crval1_linear']
cdelt1 = wv_parameters['poly_cdelt1_linear']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# display arrangement
if abs(debugplot) >= 10:
print("slit left right center width min.wave max.wave")
print("==== ======= ======= ======= ===== ======== ========")
for i in range(EMIR_NBARS):
ibar = i + 1
csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar))
csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar))
csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1
if wvmin_useful is not None:
csu_crval1 = np.amax([csu_crval1, wvmin_useful])
if wvmax_useful is not None:
csu_crvaln = np.amin([csu_crvaln, wvmax_useful])
print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} "
"{5:8.2f} {6:8.2f}".format(
ibar, csu_config.csu_bar_left(ibar),
csu_config.csu_bar_right(ibar),
csu_config.csu_bar_slit_center(ibar),
csu_config.csu_bar_slit_width(ibar), csu_crval1,
csu_crvaln))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format(
np.mean(csu_config._csu_bar_left),
np.mean(csu_config._csu_bar_right),
np.mean(csu_config._csu_bar_slit_center),
np.mean(csu_config._csu_bar_slit_width)))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format(
np.mean(csu_config._csu_bar_left[::2]),
np.mean(csu_config._csu_bar_right[::2]),
np.mean(csu_config._csu_bar_slit_center[::2]),
np.mean(csu_config._csu_bar_slit_width[::2])))
print("---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format(
np.mean(csu_config._csu_bar_left[1::2]),
np.mean(csu_config._csu_bar_right[1::2]),
np.mean(csu_config._csu_bar_slit_center[1::2]),
np.mean(csu_config._csu_bar_slit_width[1::2])))
# display slit arrangement
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
if bbox is None:
if adjust:
xmin = min(csu_config._csu_bar_left)
xmax = max(csu_config._csu_bar_right)
dx = xmax - xmin
if dx == 0:
dx = 1
xmin -= dx / 20
xmax += dx / 20
ax.set_xlim(xmin, xmax)
else:
ax.set_xlim(0., 341.5)
ax.set_ylim(0, 56)
else:
ax.set_xlim(bbox[0], bbox[1])
ax.set_ylim(bbox[2], bbox[3])
ax.set_xlabel('csu_bar_position (mm)')
ax.set_ylabel('slit number')
for i in range(EMIR_NBARS):
ibar = i + 1
ax.add_patch(
patches.Rectangle((csu_config.csu_bar_left(ibar), ibar - 0.5),
csu_config.csu_bar_slit_width(ibar), 1.0))
ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar],
'-',
color='gray')
ax.plot([csu_config.csu_bar_right(ibar), 341.5], [ibar, ibar],
'-',
color='gray')
plt.title("File: " + fileobj.name + "\ngrism=" + grism + ", filter=" +
spfilter)
pause_debugplot(debugplot, pltshow=True)
# return results
return csu_config._csu_bar_left, csu_config._csu_bar_right, \
csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def display_slitlet_arrangement(fileobj,
grism=None,
spfilter=None,
bbox=None,
adjust=None,
geometry=None,
debugplot=0):
"""Display slitlet arrangment from CSUP keywords in FITS header.
Parameters
----------
fileobj : file object
FITS or TXT file object.
grism : str
Grism.
grism : str
Filter.
bbox : tuple of 4 floats
If not None, values for xmin, xmax, ymin and ymax.
adjust : bool
Adjust X range according to minimum and maximum csu_bar_left
and csu_bar_right (note that this option is overriden by 'bbox')
geometry : tuple (4 integers) or None
x, y, dx, dy values employed to set the Qt backend geometry.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot
Returns
-------
csu_bar_left : list of floats
Location (mm) of the left bar for each slitlet.
csu_bar_right : list of floats
Location (mm) of the right bar for each slitlet, using the
same origin employed for csu_bar_left (which is not the
value stored in the FITS keywords.
csu_bar_slit_center : list of floats
Middle point (mm) in between the two bars defining a slitlet.
csu_bar_slit_width : list of floats
Slitlet width (mm), computed as the distance between the two
bars defining the slitlet.
"""
if fileobj.name[-4:] == ".txt":
if grism is None:
raise ValueError("Undefined grism!")
if spfilter is None:
raise ValueError("Undefined filter!")
# define CsuConfiguration object
csu_config = CsuConfiguration()
csu_config._csu_bar_left = []
csu_config._csu_bar_right = []
csu_config._csu_bar_slit_center = []
csu_config._csu_bar_slit_width = []
# since the input filename has been opened with argparse in binary
# mode, it is necessary to close it and open it in text mode
fileobj.close()
# read TXT file
with open(fileobj.name, mode='rt') as f:
file_content = f.read().splitlines()
next_id_bar = 1
for line in file_content:
if len(line) > 0:
if line[0] not in ['#']:
line_contents = line.split()
id_bar = int(line_contents[0])
position = float(line_contents[1])
if id_bar == next_id_bar:
if id_bar <= EMIR_NBARS:
csu_config._csu_bar_left.append(position)
next_id_bar = id_bar + EMIR_NBARS
else:
csu_config._csu_bar_right.append(341.5 - position)
next_id_bar = id_bar - EMIR_NBARS + 1
else:
raise ValueError("Unexpected id_bar:" + str(id_bar))
# compute slit width and center
for i in range(EMIR_NBARS):
csu_config._csu_bar_slit_center.append(
(csu_config._csu_bar_left[i] + csu_config._csu_bar_right[i])/2
)
csu_config._csu_bar_slit_width.append(
csu_config._csu_bar_right[i] - csu_config._csu_bar_left[i]
)
else:
# read input FITS file
hdulist = fits.open(fileobj.name)
image_header = hdulist[0].header
hdulist.close()
# additional info from header
grism = image_header['grism']
spfilter = image_header['filter']
# define slitlet arrangement
csu_config = CsuConfiguration.define_from_fits(fileobj)
# determine calibration
if grism in ["J", "OPEN"] and spfilter == "J":
wv_parameters = set_wv_parameters("J", "J")
elif grism in ["H", "OPEN"] and spfilter == "H":
wv_parameters = set_wv_parameters("H", "H")
elif grism in ["K", "OPEN"] and spfilter == "Ksp":
wv_parameters = set_wv_parameters("Ksp", "K")
elif grism in ["LR", "OPEN"] and spfilter == "YJ":
wv_parameters = set_wv_parameters("YJ", "LR")
elif grism in ["LR", "OPEN"] and spfilter == "HK":
wv_parameters = set_wv_parameters("HK", "LR")
else:
raise ValueError("Invalid grism + filter configuration")
crval1 = wv_parameters['poly_crval1_linear']
cdelt1 = wv_parameters['poly_cdelt1_linear']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# display arrangement
if abs(debugplot) >= 10:
print("slit left right center width min.wave max.wave")
print("==== ======= ======= ======= ===== ======== ========")
for i in range(EMIR_NBARS):
ibar = i + 1
csu_crval1 = crval1(csu_config.csu_bar_slit_center(ibar))
csu_cdelt1 = cdelt1(csu_config.csu_bar_slit_center(ibar))
csu_crvaln = csu_crval1 + (EMIR_NAXIS1 - 1) * csu_cdelt1
if wvmin_useful is not None:
csu_crval1 = np.amax([csu_crval1, wvmin_useful])
if wvmax_useful is not None:
csu_crvaln = np.amin([csu_crvaln, wvmax_useful])
print("{0:4d} {1:8.3f} {2:8.3f} {3:8.3f} {4:7.3f} "
"{5:8.2f} {6:8.2f}".format(
ibar, csu_config.csu_bar_left(ibar),
csu_config.csu_bar_right(ibar),
csu_config.csu_bar_slit_center(ibar),
csu_config.csu_bar_slit_width(ibar),
csu_crval1, csu_crvaln)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (all)".format(
np.mean(csu_config._csu_bar_left),
np.mean(csu_config._csu_bar_right),
np.mean(csu_config._csu_bar_slit_center),
np.mean(csu_config._csu_bar_slit_width)
)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (odd)".format(
np.mean(csu_config._csu_bar_left[::2]),
np.mean(csu_config._csu_bar_right[::2]),
np.mean(csu_config._csu_bar_slit_center[::2]),
np.mean(csu_config._csu_bar_slit_width[::2])
)
)
print(
"---> {0:8.3f} {1:8.3f} {2:8.3f} {3:7.3f} <- mean (even)".format(
np.mean(csu_config._csu_bar_left[1::2]),
np.mean(csu_config._csu_bar_right[1::2]),
np.mean(csu_config._csu_bar_slit_center[1::2]),
np.mean(csu_config._csu_bar_slit_width[1::2])
)
)
# display slit arrangement
if abs(debugplot) % 10 != 0:
fig = plt.figure()
set_window_geometry(geometry)
ax = fig.add_subplot(111)
if bbox is None:
if adjust:
xmin = min(csu_config._csu_bar_left)
xmax = max(csu_config._csu_bar_right)
dx = xmax - xmin
if dx == 0:
dx = 1
xmin -= dx/20
xmax += dx/20
ax.set_xlim(xmin, xmax)
else:
ax.set_xlim(0., 341.5)
ax.set_ylim(0, 56)
else:
ax.set_xlim(bbox[0], bbox[1])
ax.set_ylim(bbox[2], bbox[3])
ax.set_xlabel('csu_bar_position (mm)')
ax.set_ylabel('slit number')
for i in range(EMIR_NBARS):
ibar = i + 1
ax.add_patch(patches.Rectangle(
(csu_config.csu_bar_left(ibar), ibar-0.5),
csu_config.csu_bar_slit_width(ibar), 1.0))
ax.plot([0., csu_config.csu_bar_left(ibar)], [ibar, ibar],
'-', color='gray')
ax.plot([csu_config.csu_bar_right(ibar), 341.5],
[ibar, ibar], '-', color='gray')
plt.title("File: " + fileobj.name + "\ngrism=" + grism +
", filter=" + spfilter)
pause_debugplot(debugplot, pltshow=True)
# return results
return csu_config._csu_bar_left, csu_config._csu_bar_right, \
csu_config._csu_bar_slit_center, csu_config._csu_bar_slit_width
def refine_rectwv_coeff(input_image, rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'. format(
refine_wavecalib_mode
))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data(
'emirdrp.instrument.configs',
'Oliva_etal_2013.dat'
)
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm
)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1*lok2]
catlines_reference_flux = catlines_all_flux[lok1*lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm
)
# remove missing slitlets
if len(refined_rectwv_coeff.missing_slitlets) > 0:
for iremove in refined_rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets]
logger.info('list of useful slitlets: {}'.format(
list_useful_slitlets))
logger.info('list of not useful slitlets: {}'.format(
list_not_useful_slitlets))
tempwidths = np.array([csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1
)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave, sp_median, 'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum', show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-',
markerfmt=' ', basefmt='C4-', label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference, sp_reference, 'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot
)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset*cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
if np.any(lok):
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot
)
else:
offset_array[i] = 0.0
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i]*cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1-offset_array[i]*cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot
)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(stat_summary[
'robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot, yplot,
linestyle='', marker='o', color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False, **{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset, linestyle='--', color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image