def rectwv_coeff_from_arc_image(reduced_image,
bound_param,
lines_catalog,
args_nbrightlines=None,
args_ymargin_bb=2,
args_remove_sp_background=True,
args_times_sigma_threshold=10,
args_order_fmap=2,
args_sigma_gaussian_filtering=2,
args_margin_npix=50,
args_poldeg_initial=3,
args_poldeg_refined=5,
args_interactive=False,
args_threshold_wv=0,
args_ylogscale=False,
args_pdf=None,
args_geometry=(0, 0, 640, 480),
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from arc image
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
bound_param : RefinedBoundaryModelParam instance
Refined boundary model.
lines_catalog : Numpy array
2D numpy array with the contents of the master file with the
expected arc line wavelengths.
args_nbrightlines : int
TBD
args_ymargin_bb : int
TBD
args_remove_sp_background : bool
TBD
args_times_sigma_threshold : float
TBD
args_order_fmap : int
TBD
args_sigma_gaussian_filtering : float
TBD
args_margin_npix : int
TBD
args_poldeg_initial : int
TBD
args_poldeg_refined : int
TBD
args_interactive : bool
TBD
args_threshold_wv : float
TBD
args_ylogscale : bool
TBD
args_pdf : TBD
args_geometry : TBD
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration of the input arc image.
reduced_55sp : HDUList object
Image with 55 spectra corresponding to the median spectrum for
each slitlet, employed to derived the wavelength calibration
polynomial.
"""
logger = logging.getLogger(__name__)
# protections
if args_interactive and args_pdf is not None:
logger.error('--interactive and --pdf are incompatible options')
raise ValueError('--interactive and --pdf are incompatible options')
# header and data array
header = reduced_image[0].header
image2d = reduced_image[0].data
# check grism and filter
filter_name = header['filter']
logger.info('Filter: ' + filter_name)
if filter_name != bound_param.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.info('Grism: ' + grism_name)
if grism_name != bound_param.tags['grism']:
raise ValueError('Grism name does not match!')
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
logger.debug(csu_conf)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
logger.debug(dtu_conf)
# set boundary parameters
parmodel = bound_param.meta_info['parmodel']
params = bound_params_from_dict(bound_param.__getstate__())
if abs(debugplot) >= 10:
print('-' * 83)
print('* FITTED BOUND PARAMETERS')
params.pretty_print()
pause_debugplot(debugplot)
# determine parameters according to grism+filter combination
wv_parameters = set_wv_parameters(filter_name, grism_name)
islitlet_min = wv_parameters['islitlet_min']
islitlet_max = wv_parameters['islitlet_max']
if args_nbrightlines is None:
nbrightlines = wv_parameters['nbrightlines']
else:
nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')]
poly_crval1_linear = wv_parameters['poly_crval1_linear']
poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear']
wvmin_expected = wv_parameters['wvmin_expected']
wvmax_expected = wv_parameters['wvmax_expected']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# list of slitlets to be computed
logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' +
str(islitlet_max) + ']')
# read master arc line wavelengths (only brightest lines)
wv_master = read_wv_master_from_array(master_table=lines_catalog,
lines='brightest',
debugplot=debugplot)
# read master arc line wavelengths (whole data set)
wv_master_all = read_wv_master_from_array(master_table=lines_catalog,
lines='all',
debugplot=debugplot)
# check that the arc lines in the master file are properly sorted
# in ascending order
for i in range(len(wv_master_all) - 1):
if wv_master_all[i] >= wv_master_all[i + 1]:
logger.error('>>> wavelengths: ' + str(wv_master_all[i]) + ' ' +
str(wv_master_all[i + 1]))
raise ValueError('Arc lines are not sorted in master file')
# ---
image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
# compute rectification transformation and wavelength calibration
# polynomials
measured_slitlets = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet_min <= islitlet <= islitlet_max:
# define Slitlet2dArc object
slt = Slitlet2dArc(islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=params,
parmodel=parmodel,
debugplot=debugplot)
# extract 2D image corresponding to the selected slitlet, clipping
# the image beyond the unrectified slitlet (in order to isolate
# the arc lines of the current slitlet; otherwise there are
# problems with arc lines from neighbour slitlets)
image2d_tmp = select_unrectified_slitlet(
image2d=image2d,
islitlet=islitlet,
csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet),
params=params,
parmodel=parmodel,
maskonly=False)
slitlet2d = slt.extract_slitlet2d(image2d_tmp)
# subtract smooth background computed as follows:
# - median collapsed spectrum of the whole slitlet2d
# - independent median filtering of the previous spectrum in the
# two halves in the spectral direction
if args_remove_sp_background:
spmedian = np.median(slitlet2d, axis=0)
naxis1_tmp = spmedian.shape[0]
jmidpoint = naxis1_tmp // 2
sp1 = medfilt(spmedian[:jmidpoint], [201])
sp2 = medfilt(spmedian[jmidpoint:], [201])
spbackground = np.concatenate((sp1, sp2))
slitlet2d -= spbackground
# locate unknown arc lines
slt.locate_unknown_arc_lines(
slitlet2d=slitlet2d,
times_sigma_threshold=args_times_sigma_threshold)
# continue working with current slitlet only if arc lines have
# been detected
if slt.list_arc_lines is not None:
# compute intersections between spectrum trails and arc lines
slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d)
# compute rectification transformation
slt.estimate_tt_to_rectify(order=args_order_fmap,
slitlet2d=slitlet2d)
# rectify image
slitlet2d_rect = slt.rectify(slitlet2d,
resampling=2,
transformation=1)
# median spectrum and line peaks from rectified image
sp_median, fxpeaks = slt.median_spectrum_from_rectified_image(
slitlet2d_rect,
sigma_gaussian_filtering=args_sigma_gaussian_filtering,
nwinwidth_initial=5,
nwinwidth_refined=5,
times_sigma_threshold=5,
npix_avoid_border=6,
nbrightlines=nbrightlines)
image2d_55sp[islitlet - 1, :] = sp_median
# determine expected wavelength limits prior to the wavelength
# calibration
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
crval1_linear = poly_crval1_linear(csu_bar_slit_center)
cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center)
expected_wvmin = crval1_linear - \
args_margin_npix * cdelt1_linear
naxis1_linear = sp_median.shape[0]
crvaln_linear = crval1_linear + \
(naxis1_linear - 1) * cdelt1_linear
expected_wvmax = crvaln_linear + \
args_margin_npix * cdelt1_linear
# override previous estimates when necessary
if wvmin_expected is not None:
expected_wvmin = wvmin_expected
if wvmax_expected is not None:
expected_wvmax = wvmax_expected
# clip initial master arc line list with bright lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master
lok2 = wv_master <= expected_wvmax
lok = lok1 * lok2
wv_master_eff = wv_master[lok]
# perform initial wavelength calibration
solution_wv = wvcal_spectrum(
sp=sp_median,
fxpeaks=fxpeaks,
poly_degree_wfit=args_poldeg_initial,
wv_master=wv_master_eff,
wv_ini_search=expected_wvmin,
wv_end_search=expected_wvmax,
wvmin_useful=wvmin_useful,
wvmax_useful=wvmax_useful,
geometry=args_geometry,
debugplot=slt.debugplot)
# store initial wavelength calibration polynomial in current
# slitlet instance
slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff)
pause_debugplot(debugplot)
# clip initial master arc line list with all the lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master_all
lok2 = wv_master_all <= expected_wvmax
lok = lok1 * lok2
wv_master_all_eff = wv_master_all[lok]
# clip master arc line list to useful region
if wvmin_useful is not None:
lok = wvmin_useful <= wv_master_all_eff
wv_master_all_eff = wv_master_all_eff[lok]
if wvmax_useful is not None:
lok = wv_master_all_eff <= wvmax_useful
wv_master_all_eff = wv_master_all_eff[lok]
# refine wavelength calibration
if args_poldeg_refined > 0:
plottitle = '[slitlet#{}, refined]'.format(islitlet)
poly_refined, yres_summary = refine_arccalibration(
sp=sp_median,
poly_initial=slt.wpoly,
wv_master=wv_master_all_eff,
poldeg=args_poldeg_refined,
ntimes_match_wv=1,
interactive=args_interactive,
threshold=args_threshold_wv,
plottitle=plottitle,
ylogscale=args_ylogscale,
geometry=args_geometry,
pdf=args_pdf,
debugplot=slt.debugplot)
# store refined wavelength calibration polynomial in
# current slitlet instance
slt.wpoly = poly_refined
# compute approximate linear values for CRVAL1 and CDELT1
naxis1_linear = sp_median.shape[0]
crmin1_linear = slt.wpoly(1)
crmax1_linear = slt.wpoly(naxis1_linear)
slt.crval1_linear = crmin1_linear
slt.cdelt1_linear = \
(crmax1_linear - crmin1_linear) / (naxis1_linear - 1)
# check that the trimming of wv_master and wv_master_all has
# preserved the wavelength range [crmin1_linear, crmax1_linear]
if crmin1_linear < expected_wvmin:
logger.warning(">>> islitlet: " + str(islitlet))
logger.warning("expected_wvmin: " + str(expected_wvmin))
logger.warning("crmin1_linear.: " + str(crmin1_linear))
logger.warning("WARNING: Unexpected crmin1_linear < "
"expected_wvmin")
if crmax1_linear > expected_wvmax:
logger.warning(">>> islitlet: " + str(islitlet))
logger.warning("expected_wvmax: " + str(expected_wvmax))
logger.warning("crmax1_linear.: " + str(crmax1_linear))
logger.warning("WARNING: Unexpected crmax1_linear > "
"expected_wvmax")
cout += '.'
else:
cout += 'x'
if islitlet % 10 == 0:
if cout != 'x':
cout = str(islitlet // 10)
if debugplot != 0:
pause_debugplot(debugplot)
else:
# define Slitlet2dArc object
slt = Slitlet2dArc(islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=None,
parmodel=None,
debugplot=debugplot)
cout += 'i'
# store current slitlet in list of measured slitlets
measured_slitlets.append(slt)
logger.info(cout)
# ---
# generate FITS file structure with 55 spectra corresponding to the
# median spectrum for each slitlet
reduced_55sp = fits.PrimaryHDU(data=image2d_55sp)
reduced_55sp.header['crpix1'] = (0.0, 'reference pixel')
reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt1'] = (1.0, 'increment')
reduced_55sp.header['ctype1'] = 'PIXEL'
reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2')
reduced_55sp.header['crpix2'] = (0.0, 'reference pixel')
reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt2'] = (1.0, 'increment')
reduced_55sp.header['ctype2'] = 'PIXEL'
reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2')
# ---
# Generate structure to store intermediate results
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration'
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['bound_param_uuid'] = \
bound_param.uuid
outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined'
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['tags']['islitlet_min'] = islitlet_min
outdict['tags']['islitlet_max'] = islitlet_max
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
missing_slitlets = []
for slt in measured_slitlets:
islitlet = slt.islitlet
if islitlet_min <= islitlet <= islitlet_max:
# avoid error when creating a python list of coefficients from
# numpy polynomials when the polynomials do not exist (note that
# the JSON format doesn't handle numpy arrays and such arrays must
# be transformed into native python lists)
if slt.wpoly is None:
wpoly_coeff = None
else:
wpoly_coeff = slt.wpoly.coef.tolist()
if slt.wpoly_longslit_model is None:
wpoly_coeff_longslit_model = None
else:
wpoly_coeff_longslit_model = \
slt.wpoly_longslit_model.coef.tolist()
# avoid similar error when creating a python list of coefficients
# when the numpy array does not exist; note that this problem
# does not happen with tt?_aij_longslit_model and
# tt?_bij_longslit_model because the latter have already been
# created as native python lists
if slt.ttd_aij is None:
ttd_aij = None
else:
ttd_aij = slt.ttd_aij.tolist()
if slt.ttd_bij is None:
ttd_bij = None
else:
ttd_bij = slt.ttd_bij.tolist()
if slt.tti_aij is None:
tti_aij = None
else:
tti_aij = slt.tti_aij.tolist()
if slt.tti_bij is None:
tti_bij = None
else:
tti_bij = slt.tti_bij.tolist()
# creating temporary dictionary with the information corresponding
# to the current slitlett that will be saved in the JSON file
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_reference_lower': slt.y0_reference_lower,
'y0_reference_middle': slt.y0_reference_middle,
'y0_reference_upper': slt.y0_reference_upper,
'y0_reference_lower_expected': slt.y0_reference_lower_expected,
'y0_reference_middle_expected':
slt.y0_reference_middle_expected,
'y0_reference_upper_expected': slt.y0_reference_upper_expected,
'y0_frontier_lower': slt.y0_frontier_lower,
'y0_frontier_upper': slt.y0_frontier_upper,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected,
'corr_yrect_a': slt.corr_yrect_a,
'corr_yrect_b': slt.corr_yrect_b,
'min_row_rectified': slt.min_row_rectified,
'max_row_rectified': slt.max_row_rectified,
'ymargin_bb': slt.ymargin_bb,
'bb_nc1_orig': slt.bb_nc1_orig,
'bb_nc2_orig': slt.bb_nc2_orig,
'bb_ns1_orig': slt.bb_ns1_orig,
'bb_ns2_orig': slt.bb_ns2_orig,
'spectrail': {
'poly_coef_lower':
slt.list_spectrails[
slt.i_lower_spectrail].poly_funct.coef.tolist(),
'poly_coef_middle':
slt.list_spectrails[
slt.i_middle_spectrail].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_spectrails[
slt.i_upper_spectrail].poly_funct.coef.tolist(),
},
'frontier': {
'poly_coef_lower':
slt.list_frontiers[0].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_frontiers[1].poly_funct.coef.tolist(),
},
'ttd_order': slt.ttd_order,
'ttd_aij': ttd_aij,
'ttd_bij': ttd_bij,
'tti_aij': tti_aij,
'tti_bij': tti_bij,
'ttd_order_longslit_model': slt.ttd_order_longslit_model,
'ttd_aij_longslit_model': slt.ttd_aij_longslit_model,
'ttd_bij_longslit_model': slt.ttd_bij_longslit_model,
'tti_aij_longslit_model': slt.tti_aij_longslit_model,
'tti_bij_longslit_model': slt.tti_bij_longslit_model,
'wpoly_coeff': wpoly_coeff,
'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model,
'crval1_linear': slt.crval1_linear,
'cdelt1_linear': slt.cdelt1_linear
}
else:
missing_slitlets.append(islitlet)
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
outdict['contents'][slitlet_label] = tmp_dict
# ---
# OBSOLETE
'''
# save JSON file needed to compute the MOS model
with open(args.out_json.name, 'w') as fstream:
json.dump(outdict, fstream, indent=2, sort_keys=True)
print('>>> Saving file ' + args.out_json.name)
'''
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
'uuid' + bound_param.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
rectwv_coeff.missing_slitlets = missing_slitlets
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
raise ValueError("Unexpected error")
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_json.name)
# print('>>> Saving file ' + args.out_json.name)
# check_setstate_getstate(rectwv_coeff, args.out_json.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff, reduced_55sp
def median_slitlets_rectified(
input_image,
mode=0,
minimum_slitlet_width_mm=EMIR_MINIMUM_SLITLET_WIDTH_MM,
maximum_slitlet_width_mm=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
debugplot=0):
"""Compute median spectrum for each slitlet.
Parameters
----------
input_image : HDUList object
Input 2D image.
mode : int
Indicate desired result:
0 : image with the same size as the input image, with the
median spectrum of each slitlet spanning all the spectra
of the corresponding slitlet
1 : image with 55 spectra, containing the median spectra of
each slitlet
2 : single collapsed median spectrum, using exclusively the
useful slitlets from the input image
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
image_median : HDUList object
Output image.
"""
image_header = input_image[0].header
image2d = input_image[0].data
# check image dimensions
naxis2_expected = EMIR_NBARS * EMIR_NPIXPERSLIT_RECTIFIED
naxis2, naxis1 = image2d.shape
if naxis2 != naxis2_expected:
raise ValueError("NAXIS2={0} should be {1}".format(
naxis2, naxis2_expected))
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# initialize output image
if mode == 0:
image2d_median = np.zeros((naxis2, naxis1))
else:
image2d_median = np.zeros((EMIR_NBARS, naxis1))
# main loop
for i in range(EMIR_NBARS):
ns1 = i * EMIR_NPIXPERSLIT_RECTIFIED + 1
ns2 = ns1 + EMIR_NPIXPERSLIT_RECTIFIED - 1
sp_median = np.median(image2d[(ns1 - 1):ns2, :], axis=0)
if mode == 0:
image2d_median[(ns1 - 1):ns2, :] = np.tile(
sp_median, (EMIR_NPIXPERSLIT_RECTIFIED, 1))
else:
image2d_median[i] = np.copy(sp_median)
if mode == 2:
# get CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_header(image_header)
# define wavelength calibration parameters
crpix1 = image_header['crpix1']
crval1 = image_header['crval1']
cdelt1 = image_header['cdelt1']
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
if abs(debugplot) != 0:
print('>>> list_useful_slitlets....:', list_useful_slitlets)
print('>>> list_not_useful_slitlets:', list_not_useful_slitlets)
# define mask from array data
mask2d, borders = define_mask_borders(image2d_median, sought_value=0)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int),
z1z2=(-.2, 1.2),
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# update mask with unused slitlets
for islitlet in list_not_useful_slitlets:
mask2d[islitlet - 1, :] = np.array([True] * naxis1)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int),
z1z2=(-.2, 1.2),
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# useful image pixels
image2d_masked = image2d_median * (1 - mask2d.astype(int))
if abs(debugplot) % 10 != 0:
ximshow(image2d_masked,
crpix1=crpix1,
crval1=crval1,
cdelt1=cdelt1,
debugplot=debugplot)
# masked image
image2d_masked = np.ma.masked_array(image2d_median, mask=mask2d)
# median spectrum
image1d_median = np.ma.median(image2d_masked, axis=0).data
image_median = fits.PrimaryHDU(data=image1d_median,
header=image_header)
else:
image_median = fits.PrimaryHDU(data=image2d_median,
header=image_header)
return fits.HDUList([image_median])
def run(self, rinput):
self.logger.info('starting generation of flatlowfreq')
self.logger.info('rectwv_coeff..........................: {}'.format(
rinput.rectwv_coeff))
self.logger.info('master_rectwv.........................: {}'.format(
rinput.master_rectwv))
self.logger.info('Minimum slitlet width (mm)............: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info('Maximum slitlet width (mm)............: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info('Global offset X direction (pixels)....: {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info('Global offset Y direction (pixels)....: {}'.format(
rinput.global_integer_offset_y_pix))
self.logger.info('nwindow_x_median......................: {}'.format(
rinput.nwindow_x_median))
self.logger.info('nwindow_y_median......................: {}'.format(
rinput.nwindow_y_median))
self.logger.info('Minimum fraction......................: {}'.format(
rinput.minimum_fraction))
self.logger.info('Minimum value in output...............: {}'.format(
rinput.minimum_value_in_output))
self.logger.info('Maximum value in output...............: {}'.format(
rinput.maximum_value_in_output))
# check rectification and wavelength calibration information
if rinput.master_rectwv is None and rinput.rectwv_coeff is None:
raise ValueError('No master_rectwv nor rectwv_coeff data have '
'been provided')
elif rinput.master_rectwv is not None and \
rinput.rectwv_coeff is not None:
self.logger.warning('rectwv_coeff will be used instead of '
'master_rectwv')
if rinput.rectwv_coeff is not None and \
(rinput.global_integer_offset_x_pix != 0 or
rinput.global_integer_offset_y_pix != 0):
raise ValueError('global_integer_offsets cannot be used '
'simultaneously with rectwv_coeff')
# check headers to detect lamp status (on/off)
list_lampincd = []
for fname in rinput.obresult.frames:
with fname.open() as f:
list_lampincd.append(f[0].header['lampincd'])
# check number of images
nimages = len(rinput.obresult.frames)
n_on = list_lampincd.count(1)
n_off = list_lampincd.count(0)
self.logger.info(
'Number of images with lamp ON.........: {}'.format(n_on))
self.logger.info(
'Number of images with lamp OFF........: {}'.format(n_off))
self.logger.info(
'Total number of images................: {}'.format(nimages))
if n_on == 0:
raise ValueError('Insufficient number of images with lamp ON')
if n_on + n_off != nimages:
raise ValueError('Number of images does not match!')
# check combination method
if rinput.method_kwargs == {}:
method_kwargs = None
else:
if rinput.method == 'sigmaclip':
method_kwargs = rinput.method_kwargs
else:
raise ValueError('Unexpected method_kwargs={}'.format(
rinput.method_kwargs))
# build object to proceed with bpm, bias, and dark (not flat)
flow = self.init_filters(rinput)
# available combination methods
method = getattr(combine, rinput.method)
# basic reduction of images with lamp ON or OFF
lampmode = {0: 'off', 1: 'on'}
reduced_image_on = None
reduced_image_off = None
for imode in lampmode.keys():
self.logger.info('starting basic reduction of images with'
' lamp {}'.format(lampmode[imode]))
tmplist = [
rinput.obresult.frames[i]
for i, lampincd in enumerate(list_lampincd)
if lampincd == imode
]
if len(tmplist) > 0:
with contextlib.ExitStack() as stack:
hduls = [
stack.enter_context(fname.open()) for fname in tmplist
]
reduced_image = combine_imgs(hduls,
method=method,
method_kwargs=method_kwargs,
errors=False,
prolog=None)
if imode == 0:
reduced_image_off = flow(reduced_image)
hdr = reduced_image_off[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_off,
'reduced_image_off.fits')
elif imode == 1:
reduced_image_on = flow(reduced_image)
hdr = reduced_image_on[0].header
self.set_base_headers(hdr)
self.save_intermediate_img(reduced_image_on,
'reduced_image_on.fits')
else:
raise ValueError('Unexpected imode={}'.format(imode))
# computation of ON-OFF
header_on = reduced_image_on[0].header
data_on = reduced_image_on[0].data.astype('float32')
if n_off > 0:
header_off = reduced_image_off[0].header
data_off = reduced_image_off[0].data.astype('float32')
else:
header_off = None
data_off = np.zeros_like(data_on)
reduced_data = data_on - data_off
# update reduced image header
reduced_image = self.create_reduced_image(rinput, reduced_data,
header_on, header_off,
list_lampincd)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# define rectification and wavelength calibration coefficients
if rinput.rectwv_coeff is None:
rectwv_coeff = rectwv_coeff_from_mos_library(
reduced_image, rinput.master_rectwv)
# set global offsets
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
else:
rectwv_coeff = rinput.rectwv_coeff
# save as JSON in work directory
self.save_structured_as_json(rectwv_coeff, 'rectwv_coeff.json')
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# apply global offsets (to both, the original and the cleaned version)
image2d = apply_integer_offsets(
image2d=reduced_data,
offx=rectwv_coeff.global_integer_offset_x_pix,
offy=rectwv_coeff.global_integer_offset_y_pix)
# load CSU configuration
csu_conf = CsuConfiguration.define_from_header(reduced_image[0].header)
# determine (pseudo) longslits
dict_longslits = csu_conf.pseudo_longslits()
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in rectwv_coeff.missing_slitlets:
self.logger.info('-> Removing slitlet (not defined): ' + str(idel))
list_valid_islitlets.remove(idel)
# filter out slitlets with widths outside valid range
list_outside_valid_width = []
for islitlet in list_valid_islitlets:
slitwidth = csu_conf.csu_bar_slit_width(islitlet)
if (slitwidth < rinput.minimum_slitlet_width_mm) or \
(slitwidth > rinput.maximum_slitlet_width_mm):
list_outside_valid_width.append(islitlet)
self.logger.info('-> Removing slitlet (width out of range): ' +
str(islitlet))
if len(list_outside_valid_width) > 0:
for idel in list_outside_valid_width:
list_valid_islitlets.remove(idel)
# initialize rectified image
image2d_flatfielded = np.zeros((EMIR_NAXIS2, EMIR_NAXIS1))
# main loop
grism_name = rectwv_coeff.tags['grism']
filter_name = rectwv_coeff.tags['filter']
cout = '0'
debugplot = rinput.debugplot
for islitlet in list(range(1, EMIR_NBARS + 1)):
if islitlet in list_valid_islitlets:
# define Slitlet2D object
slt = Slitlet2D(islitlet=islitlet,
rectwv_coeff=rectwv_coeff,
debugplot=debugplot)
if abs(slt.debugplot) > 10:
print(slt)
# extract (distorted) slitlet from the initial image
slitlet2d = slt.extract_slitlet2d(image_2k2k=image2d,
subtitle='original image')
# rectify slitlet
slitlet2d_rect = slt.rectify(
slitlet2d=slitlet2d,
resampling=2,
subtitle='original (cleaned) rectified')
naxis2_slitlet2d, naxis1_slitlet2d = slitlet2d_rect.shape
if naxis1_slitlet2d != EMIR_NAXIS1:
print('naxis1_slitlet2d: ', naxis1_slitlet2d)
print('EMIR_NAXIS1.....: ', EMIR_NAXIS1)
raise ValueError("Unexpected naxis1_slitlet2d")
# get useful slitlet region (use boundaries)
spectrail = slt.list_spectrails[0]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii1 = int(yy0 + 0.5) - slt.bb_ns1_orig
spectrail = slt.list_spectrails[2]
yy0 = slt.corr_yrect_a + \
slt.corr_yrect_b * spectrail(slt.x0_reference)
ii2 = int(yy0 + 0.5) - slt.bb_ns1_orig
# median spectrum
sp_collapsed = np.median(slitlet2d_rect[ii1:(ii2 + 1), :],
axis=0)
# smooth median spectrum along the spectral direction
sp_median = ndimage.median_filter(sp_collapsed,
5,
mode='nearest')
ymax_spmedian = sp_median.max()
y_threshold = ymax_spmedian * rinput.minimum_fraction
lremove = np.where(sp_median < y_threshold)
sp_median[lremove] = 0.0
if abs(slt.debugplot) % 10 != 0:
xaxis1 = np.arange(1, naxis1_slitlet2d + 1)
title = 'Slitlet#' + str(islitlet) + ' (median spectrum)'
ax = ximplotxy(xaxis1,
sp_collapsed,
title=title,
show=False,
**{'label': 'collapsed spectrum'})
ax.plot(xaxis1, sp_median, label='fitted spectrum')
ax.plot([1, naxis1_slitlet2d],
2 * [y_threshold],
label='threshold')
# ax.plot(xknots, yknots, 'o', label='knots')
ax.legend()
ax.set_ylim(-0.05 * ymax_spmedian, 1.05 * ymax_spmedian)
pause_debugplot(slt.debugplot,
pltshow=True,
tight_layout=True)
# generate rectified slitlet region filled with the
# median spectrum
slitlet2d_rect_spmedian = np.tile(sp_median,
(naxis2_slitlet2d, 1))
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_rectified(
slitlet2d_rect=slitlet2d_rect_spmedian,
subtitle='rectified, filled with median spectrum')
# compute smooth surface
# clipped region
slitlet2d_rect_clipped = slitlet2d_rect_spmedian.copy()
slitlet2d_rect_clipped[:(ii1 - 1), :] = 0.0
slitlet2d_rect_clipped[(ii2 + 2):, :] = 0.0
# unrectified clipped image
slitlet2d_unrect_clipped = slt.rectify(
slitlet2d=slitlet2d_rect_clipped,
resampling=2,
inverse=True,
subtitle='unrectified, filled with median spectrum '
'(clipped)')
# normalize initial slitlet image (avoid division by zero)
slitlet2d_norm_clipped = np.zeros_like(slitlet2d)
for j in range(naxis1_slitlet2d):
for i in range(naxis2_slitlet2d):
den = slitlet2d_unrect_clipped[i, j]
if den == 0:
slitlet2d_norm_clipped[i, j] = 1.0
else:
slitlet2d_norm_clipped[i, j] = \
slitlet2d[i, j] / den
# set to 1.0 one additional pixel at each side (since
# 'den' above is small at the borders and generates wrong
# bright pixels)
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_clipped = fix_pix_borders(
image2d=slitlet2d_norm_clipped,
nreplace=1,
sought_value=1.0,
replacement_value=1.0)
slitlet2d_norm_clipped = slitlet2d_norm_clipped.transpose()
slitlet2d_norm_smooth = ndimage.median_filter(
slitlet2d_norm_clipped,
size=(rinput.nwindow_y_median, rinput.nwindow_x_median),
mode='nearest')
if abs(slt.debugplot) % 10 != 0:
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_clipped,
subtitle='unrectified, pixel-to-pixel (clipped)')
slt.ximshow_unrectified(
slitlet2d=slitlet2d_norm_smooth,
subtitle='unrectified, pixel-to-pixel (smoothed)')
# ---
# check for (pseudo) longslit with previous and next slitlet
imin = dict_longslits[islitlet].imin()
imax = dict_longslits[islitlet].imax()
if islitlet > 1:
same_slitlet_below = (islitlet - 1) >= imin
else:
same_slitlet_below = False
if islitlet < EMIR_NBARS:
same_slitlet_above = (islitlet + 1) <= imax
else:
same_slitlet_above = False
for j in range(EMIR_NAXIS1):
xchannel = j + 1
y0_lower = slt.list_frontiers[0](xchannel)
y0_upper = slt.list_frontiers[1](xchannel)
n1, n2 = nscan_minmax_frontiers(y0_frontier_lower=y0_lower,
y0_frontier_upper=y0_upper,
resize=True)
# note that n1 and n2 are scans (ranging from 1 to NAXIS2)
nn1 = n1 - slt.bb_ns1_orig + 1
nn2 = n2 - slt.bb_ns1_orig + 1
image2d_flatfielded[(n1 - 1):n2, j] = \
slitlet2d_norm_smooth[(nn1 - 1):nn2, j]
# force to 1.0 region around frontiers
if not same_slitlet_below:
image2d_flatfielded[(n1 - 1):(n1 + 2), j] = 1
if not same_slitlet_above:
image2d_flatfielded[(n2 - 5):n2, j] = 1
cout += '.'
else:
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
self.logger.info(cout)
# restore global offsets
image2d_flatfielded = apply_integer_offsets(
image2d=image2d_flatfielded,
offx=-rectwv_coeff.global_integer_offset_x_pix,
offy=-rectwv_coeff.global_integer_offset_y_pix)
# set pixels below minimum value to 1.0
filtered = np.where(
image2d_flatfielded < rinput.minimum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# set pixels above maximum value to 1.0
filtered = np.where(
image2d_flatfielded > rinput.maximum_value_in_output)
image2d_flatfielded[filtered] = 1.0
# update image header
reduced_flatlowfreq = self.create_reduced_image(
rinput, image2d_flatfielded, header_on, header_off, list_lampincd)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# save results in results directory
self.logger.info('end of flatlowfreq generation')
result = self.create_result(reduced_flatlowfreq=reduced_flatlowfreq)
return result
def rectwv_coeff_from_mos_library(reduced_image,
master_rectwv,
ignore_dtu_configuration=True,
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from MOS library
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
master_rectwv : MasterRectWave instance
Rectification and Wavelength Calibrartion Library product.
Contains the library of polynomial coefficients necessary
to generate an instance of RectWaveCoeff with the rectification
and wavelength calibration coefficients for the particular
CSU configuration.
ignore_dtu_configuration : bool
If True, ignore differences in DTU configuration.
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
"""
logger = logging.getLogger(__name__)
logger.info('Computing expected RectWaveCoeff from CSU configuration')
# header
header = reduced_image[0].header
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
# retrieve DTU configuration from MasterRectWave object
dtu_conf_calib = DtuConfiguration.define_from_dictionary(
master_rectwv.meta_info['dtu_configuration']
)
# check that the DTU configuration employed to obtain the calibration
# corresponds to the DTU configuration in the input FITS file
if dtu_conf != dtu_conf_calib:
if ignore_dtu_configuration:
logger.warning('DTU configuration differences found!')
else:
logger.info('DTU configuration from image header:')
logger.info(dtu_conf)
logger.info('DTU configuration from master calibration:')
logger.info(dtu_conf_calib)
raise ValueError("DTU configurations do not match!")
else:
logger.info('DTU configuration match!')
# check grism and filter
filter_name = header['filter']
logger.debug('Filter: ' + filter_name)
if filter_name != master_rectwv.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.debug('Grism: ' + grism_name)
if grism_name != master_rectwv.tags['grism']:
raise ValueError('Grism name does not match!')
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in master_rectwv.missing_slitlets:
list_valid_islitlets.remove(idel)
logger.debug('valid slitlet numbers: ' + str(list_valid_islitlets))
# initialize intermediate dictionary with relevant information
# (note: this dictionary corresponds to an old structure employed to
# store the information in a JSON file; this is no longer necessary,
# but here we reuse that dictionary for convenience)
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration from a MOS model '
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['fits_frame_uuid'] = 'TBD'
outdict['meta_info']['origin']['rect_wpoly_mos_uuid'] = \
master_rectwv.uuid
outdict['meta_info']['origin']['fitted_boundary_param_uuid'] = \
master_rectwv.meta_info['origin']['bound_param']
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
# compute rectification and wavelength calibration coefficients for each
# slitlet according to its csu_bar_slit_center value
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# input data structure
tmpdict = master_rectwv.contents[islitlet - 1]
list_csu_bar_slit_center = tmpdict['list_csu_bar_slit_center']
# check extrapolations
if csu_bar_slit_center < min(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('minimum tabulated value: ' +
str(min(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
if csu_bar_slit_center > max(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('maximum tabulated value: ' +
str(max(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
# rectification coefficients
ttd_order = tmpdict['ttd_order']
ncoef = ncoef_fmap(ttd_order)
outdict['contents'][cslitlet] = {}
outdict['contents'][cslitlet]['ttd_order'] = ttd_order
outdict['contents'][cslitlet]['ttd_order_longslit_model'] = None
for keycoef in ['ttd_aij', 'ttd_bij', 'tti_aij', 'tti_bij']:
coef_out = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_' + keycoef + '_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
coef_out.append(dum[0])
outdict['contents'][cslitlet][keycoef] = coef_out
outdict['contents'][cslitlet][keycoef + '_longslit_model'] = None
# wavelength calibration coefficients
ncoef = tmpdict['wpoly_degree'] + 1
wpoly_coeff = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_wpoly_coeff_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
wpoly_coeff.append(dum[0])
outdict['contents'][cslitlet]['wpoly_coeff'] = wpoly_coeff
outdict['contents'][cslitlet]['wpoly_coeff_longslit_model'] = None
# update cdelt1_linear and crval1_linear
wpoly_function = np.polynomial.Polynomial(wpoly_coeff)
crmin1_linear = wpoly_function(1)
crmax1_linear = wpoly_function(EMIR_NAXIS1)
cdelt1_linear = (crmax1_linear - crmin1_linear) / (EMIR_NAXIS1 - 1)
crval1_linear = crmin1_linear
outdict['contents'][cslitlet]['crval1_linear'] = crval1_linear
outdict['contents'][cslitlet]['cdelt1_linear'] = cdelt1_linear
# update CSU keywords
outdict['contents'][cslitlet]['csu_bar_left'] = \
csu_conf.csu_bar_left(islitlet)
outdict['contents'][cslitlet]['csu_bar_right'] = \
csu_conf.csu_bar_right(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_center'] = \
csu_conf.csu_bar_slit_center(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_width'] = \
csu_conf.csu_bar_slit_width(islitlet)
# for each slitlet compute spectrum trails and frontiers using the
# fitted boundary parameters
fitted_bound_param_json = {
'contents': master_rectwv.meta_info['refined_boundary_model']
}
parmodel = fitted_bound_param_json['contents']['parmodel']
fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}})
params = bound_params_from_dict(fitted_bound_param_json)
if abs(debugplot) >= 10:
logger.debug('Fitted boundary parameters:')
logger.debug(params.pretty_print())
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# compute and store x0_reference value
x0_reference = float(EMIR_NAXIS1) / 2.0 + 0.5
outdict['contents'][cslitlet]['x0_reference'] = x0_reference
# compute spectrum trails (lower, middle and upper)
list_spectrails = expected_distorted_boundaries(
islitlet, csu_bar_slit_center,
[0, 0.5, 1], params, parmodel,
numpts=101, deg=5, debugplot=0
)
# store spectrails in output JSON file
outdict['contents'][cslitlet]['spectrail'] = {}
for idum, cdum in zip(range(3), ['lower', 'middle', 'upper']):
outdict['contents'][cslitlet]['spectrail']['poly_coef_' + cdum] = \
list_spectrails[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_reference_' + cdum] = \
list_spectrails[idum].poly_funct(x0_reference)
# compute frontiers (lower, upper)
list_frontiers = expected_distorted_frontiers(
islitlet, csu_bar_slit_center,
params, parmodel,
numpts=101, deg=5, debugplot=0
)
# store frontiers in output JSON
outdict['contents'][cslitlet]['frontier'] = {}
for idum, cdum in zip(range(2), ['lower', 'upper']):
outdict['contents'][cslitlet]['frontier']['poly_coef_' + cdum] = \
list_frontiers[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_frontier_' + cdum] = \
list_frontiers[idum].poly_funct(x0_reference)
# store bounding box parameters for each slitlet
xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# parameters already available in the input JSON file
for par in ['bb_nc1_orig', 'bb_nc2_orig', 'ymargin_bb']:
outdict['contents'][cslitlet][par] = \
master_rectwv.contents[islitlet - 1][par]
# estimate bb_ns1_orig and bb_ns2_orig using the already computed
# frontiers and the value of ymargin_bb, following the same approach
# employed in Slitlet2dArc.__init__()
poly_lower_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_lower']
)
poly_upper_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_upper']
)
ylower = poly_lower_frontier(xdum)
yupper = poly_upper_frontier(xdum)
ymargin_bb = master_rectwv.contents[islitlet - 1]['ymargin_bb']
bb_ns1_orig = int(ylower.min() + 0.5) - ymargin_bb
if bb_ns1_orig < 1:
bb_ns1_orig = 1
bb_ns2_orig = int(yupper.max() + 0.5) + ymargin_bb
if bb_ns2_orig > EMIR_NAXIS2:
bb_ns2_orig = EMIR_NAXIS2
outdict['contents'][cslitlet]['bb_ns1_orig'] = bb_ns1_orig
outdict['contents'][cslitlet]['bb_ns2_orig'] = bb_ns2_orig
# additional parameters (see Slitlet2dArc.__init__)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# define expected frontier ordinates at x0_reference for the rectified
# image imposing the vertical length of the slitlet to be constant
# and equal to EMIR_NPIXPERSLIT_RECTIFIED
outdict['contents'][cslitlet]['y0_frontier_lower_expected'] = \
expected_y0_lower_frontier(islitlet)
outdict['contents'][cslitlet]['y0_frontier_upper_expected'] = \
expected_y0_upper_frontier(islitlet)
# compute linear transformation to place the rectified slitlet at
# the center of the current slitlet bounding box
tmpdict = outdict['contents'][cslitlet]
xdum1 = tmpdict['y0_frontier_lower']
ydum1 = tmpdict['y0_frontier_lower_expected']
xdum2 = tmpdict['y0_frontier_upper']
ydum2 = tmpdict['y0_frontier_upper_expected']
corr_yrect_b = (ydum2 - ydum1) / (xdum2 - xdum1)
corr_yrect_a = ydum1 - corr_yrect_b * xdum1
# compute expected location of rectified boundaries
y0_reference_lower_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_lower']
y0_reference_middle_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_middle']
y0_reference_upper_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_upper']
# shift transformation to center the rectified slitlet within the
# slitlet bounding box
ydummid = (ydum1 + ydum2) / 2
ioffset = int(
ydummid - (tmpdict['bb_ns1_orig'] + tmpdict['bb_ns2_orig']) / 2.0)
corr_yrect_a -= ioffset
# minimum and maximum row in the rectified slitlet encompassing
# EMIR_NPIXPERSLIT_RECTIFIED pixels
# a) scan number (in pixels, from 1 to NAXIS2)
xdum1 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_lower']
xdum2 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_upper']
# b) row number (starting from zero)
min_row_rectified = \
int((round(xdum1 * 10) + 5) / 10) - tmpdict['bb_ns1_orig']
max_row_rectified = \
int((round(xdum2 * 10) - 5) / 10) - tmpdict['bb_ns1_orig']
# save previous results in outdict
outdict['contents'][cslitlet]['y0_reference_lower_expected'] = \
y0_reference_lower_expected
outdict['contents'][cslitlet]['y0_reference_middle_expected'] = \
y0_reference_middle_expected
outdict['contents'][cslitlet]['y0_reference_upper_expected'] = \
y0_reference_upper_expected
outdict['contents'][cslitlet]['corr_yrect_a'] = corr_yrect_a
outdict['contents'][cslitlet]['corr_yrect_b'] = corr_yrect_b
outdict['contents'][cslitlet]['min_row_rectified'] = min_row_rectified
outdict['contents'][cslitlet]['max_row_rectified'] = max_row_rectified
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
master_rectwv.meta_info['origin']['bound_param']
rectwv_coeff.meta_info['origin']['master_rectwv'] = \
'uuid' + master_rectwv.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
dumdict.update({
'csu_bar_left': csu_conf.csu_bar_left(islitlet),
'csu_bar_right': csu_conf.csu_bar_right(islitlet),
'csu_bar_slit_center': csu_conf.csu_bar_slit_center(islitlet),
'csu_bar_slit_width': csu_conf.csu_bar_slit_width(islitlet),
'x0_reference': float(EMIR_NAXIS1) / 2.0 + 0.5,
'y0_frontier_lower_expected':
expected_y0_lower_frontier(islitlet),
'y0_frontier_upper_expected':
expected_y0_upper_frontier(islitlet)
})
rectwv_coeff.missing_slitlets.append(islitlet)
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_rect_wpoly.name)
# print('>>> Saving file ' + args.out_rect_wpoly.name)
# check_setstate_getstate(rectwv_coeff, args.out_rect_wpoly.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff
def rectwv_coeff_from_mos_library(reduced_image,
master_rectwv,
ignore_dtu_configuration=True,
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from MOS library
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
master_rectwv : MasterRectWave instance
Rectification and Wavelength Calibrartion Library product.
Contains the library of polynomial coefficients necessary
to generate an instance of RectWaveCoeff with the rectification
and wavelength calibration coefficients for the particular
CSU configuration.
ignore_dtu_configuration : bool
If True, ignore differences in DTU configuration.
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
"""
logger = logging.getLogger(__name__)
logger.info('Computing expected RectWaveCoeff from CSU configuration')
# header
header = reduced_image[0].header
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
# retrieve DTU configuration from MasterRectWave object
dtu_conf_calib = DtuConfiguration.define_from_dictionary(
master_rectwv.meta_info['dtu_configuration'])
# check that the DTU configuration employed to obtain the calibration
# corresponds to the DTU configuration in the input FITS file
if dtu_conf != dtu_conf_calib:
if ignore_dtu_configuration:
logger.warning('DTU configuration differences found!')
else:
logger.info('DTU configuration from image header:')
logger.info(dtu_conf)
logger.info('DTU configuration from master calibration:')
logger.info(dtu_conf_calib)
raise ValueError("DTU configurations do not match!")
else:
logger.info('DTU configuration match!')
# check grism and filter
filter_name = header['filter']
logger.debug('Filter: ' + filter_name)
if filter_name != master_rectwv.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.debug('Grism: ' + grism_name)
if grism_name != master_rectwv.tags['grism']:
raise ValueError('Grism name does not match!')
# valid slitlet numbers
list_valid_islitlets = list(range(1, EMIR_NBARS + 1))
for idel in master_rectwv.missing_slitlets:
list_valid_islitlets.remove(idel)
logger.debug('valid slitlet numbers: ' + str(list_valid_islitlets))
# initialize intermediate dictionary with relevant information
# (note: this dictionary corresponds to an old structure employed to
# store the information in a JSON file; this is no longer necessary,
# but here we reuse that dictionary for convenience)
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration from a MOS model '
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['fits_frame_uuid'] = 'TBD'
outdict['meta_info']['origin']['rect_wpoly_mos_uuid'] = \
master_rectwv.uuid
outdict['meta_info']['origin']['fitted_boundary_param_uuid'] = \
master_rectwv.meta_info['origin']['bound_param']
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
# compute rectification and wavelength calibration coefficients for each
# slitlet according to its csu_bar_slit_center value
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# input data structure
tmpdict = master_rectwv.contents[islitlet - 1]
list_csu_bar_slit_center = tmpdict['list_csu_bar_slit_center']
# check extrapolations
if csu_bar_slit_center < min(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('minimum tabulated value: ' +
str(min(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
if csu_bar_slit_center > max(list_csu_bar_slit_center):
logger.warning('extrapolating table with ' + cslitlet)
logger.warning('maximum tabulated value: ' +
str(max(list_csu_bar_slit_center)))
logger.warning('sought value...........: ' +
str(csu_bar_slit_center))
# rectification coefficients
ttd_order = tmpdict['ttd_order']
ncoef = ncoef_fmap(ttd_order)
outdict['contents'][cslitlet] = {}
outdict['contents'][cslitlet]['ttd_order'] = ttd_order
outdict['contents'][cslitlet]['ttd_order_longslit_model'] = None
for keycoef in ['ttd_aij', 'ttd_bij', 'tti_aij', 'tti_bij']:
coef_out = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_' + keycoef + '_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
coef_out.append(dum[0])
outdict['contents'][cslitlet][keycoef] = coef_out
outdict['contents'][cslitlet][keycoef + '_longslit_model'] = None
# wavelength calibration coefficients
ncoef = tmpdict['wpoly_degree'] + 1
wpoly_coeff = []
for icoef in range(ncoef):
ccoef = str(icoef).zfill(2)
list_cij = tmpdict['list_wpoly_coeff_' + ccoef]
funinterp_coef = interp1d(list_csu_bar_slit_center,
list_cij,
kind='linear',
fill_value='extrapolate')
# note: funinterp_coef expects a numpy array
dum = funinterp_coef([csu_bar_slit_center])
wpoly_coeff.append(dum[0])
outdict['contents'][cslitlet]['wpoly_coeff'] = wpoly_coeff
outdict['contents'][cslitlet]['wpoly_coeff_longslit_model'] = None
# update cdelt1_linear and crval1_linear
wpoly_function = np.polynomial.Polynomial(wpoly_coeff)
crmin1_linear = wpoly_function(1)
crmax1_linear = wpoly_function(EMIR_NAXIS1)
cdelt1_linear = (crmax1_linear - crmin1_linear) / (EMIR_NAXIS1 - 1)
crval1_linear = crmin1_linear
outdict['contents'][cslitlet]['crval1_linear'] = crval1_linear
outdict['contents'][cslitlet]['cdelt1_linear'] = cdelt1_linear
# update CSU keywords
outdict['contents'][cslitlet]['csu_bar_left'] = \
csu_conf.csu_bar_left(islitlet)
outdict['contents'][cslitlet]['csu_bar_right'] = \
csu_conf.csu_bar_right(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_center'] = \
csu_conf.csu_bar_slit_center(islitlet)
outdict['contents'][cslitlet]['csu_bar_slit_width'] = \
csu_conf.csu_bar_slit_width(islitlet)
# for each slitlet compute spectrum trails and frontiers using the
# fitted boundary parameters
fitted_bound_param_json = {
'contents': master_rectwv.meta_info['refined_boundary_model']
}
parmodel = fitted_bound_param_json['contents']['parmodel']
fitted_bound_param_json.update({'meta_info': {'parmodel': parmodel}})
params = bound_params_from_dict(fitted_bound_param_json)
if abs(debugplot) >= 10:
logger.debug('Fitted boundary parameters:')
logger.debug(params.pretty_print())
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# csu_bar_slit_center of current slitlet in initial FITS image
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
# compute and store x0_reference value
x0_reference = float(EMIR_NAXIS1) / 2.0 + 0.5
outdict['contents'][cslitlet]['x0_reference'] = x0_reference
# compute spectrum trails (lower, middle and upper)
list_spectrails = expected_distorted_boundaries(islitlet,
csu_bar_slit_center,
[0, 0.5, 1],
params,
parmodel,
numpts=101,
deg=5,
debugplot=0)
# store spectrails in output JSON file
outdict['contents'][cslitlet]['spectrail'] = {}
for idum, cdum in zip(range(3), ['lower', 'middle', 'upper']):
outdict['contents'][cslitlet]['spectrail']['poly_coef_' + cdum] = \
list_spectrails[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_reference_' + cdum] = \
list_spectrails[idum].poly_funct(x0_reference)
# compute frontiers (lower, upper)
list_frontiers = expected_distorted_frontiers(islitlet,
csu_bar_slit_center,
params,
parmodel,
numpts=101,
deg=5,
debugplot=0)
# store frontiers in output JSON
outdict['contents'][cslitlet]['frontier'] = {}
for idum, cdum in zip(range(2), ['lower', 'upper']):
outdict['contents'][cslitlet]['frontier']['poly_coef_' + cdum] = \
list_frontiers[idum].poly_funct.coef.tolist()
outdict['contents'][cslitlet]['y0_frontier_' + cdum] = \
list_frontiers[idum].poly_funct(x0_reference)
# store bounding box parameters for each slitlet
xdum = np.linspace(1, EMIR_NAXIS1, num=EMIR_NAXIS1)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# parameters already available in the input JSON file
for par in ['bb_nc1_orig', 'bb_nc2_orig', 'ymargin_bb']:
outdict['contents'][cslitlet][par] = \
master_rectwv.contents[islitlet - 1][par]
# estimate bb_ns1_orig and bb_ns2_orig using the already computed
# frontiers and the value of ymargin_bb, following the same approach
# employed in Slitlet2dArc.__init__()
poly_lower_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_lower'])
poly_upper_frontier = np.polynomial.Polynomial(
outdict['contents'][cslitlet]['frontier']['poly_coef_upper'])
ylower = poly_lower_frontier(xdum)
yupper = poly_upper_frontier(xdum)
ymargin_bb = master_rectwv.contents[islitlet - 1]['ymargin_bb']
bb_ns1_orig = int(ylower.min() + 0.5) - ymargin_bb
if bb_ns1_orig < 1:
bb_ns1_orig = 1
bb_ns2_orig = int(yupper.max() + 0.5) + ymargin_bb
if bb_ns2_orig > EMIR_NAXIS2:
bb_ns2_orig = EMIR_NAXIS2
outdict['contents'][cslitlet]['bb_ns1_orig'] = bb_ns1_orig
outdict['contents'][cslitlet]['bb_ns2_orig'] = bb_ns2_orig
# additional parameters (see Slitlet2dArc.__init__)
for islitlet in list_valid_islitlets:
cslitlet = 'slitlet' + str(islitlet).zfill(2)
# define expected frontier ordinates at x0_reference for the rectified
# image imposing the vertical length of the slitlet to be constant
# and equal to EMIR_NPIXPERSLIT_RECTIFIED
outdict['contents'][cslitlet]['y0_frontier_lower_expected'] = \
expected_y0_lower_frontier(islitlet)
outdict['contents'][cslitlet]['y0_frontier_upper_expected'] = \
expected_y0_upper_frontier(islitlet)
# compute linear transformation to place the rectified slitlet at
# the center of the current slitlet bounding box
tmpdict = outdict['contents'][cslitlet]
xdum1 = tmpdict['y0_frontier_lower']
ydum1 = tmpdict['y0_frontier_lower_expected']
xdum2 = tmpdict['y0_frontier_upper']
ydum2 = tmpdict['y0_frontier_upper_expected']
corr_yrect_b = (ydum2 - ydum1) / (xdum2 - xdum1)
corr_yrect_a = ydum1 - corr_yrect_b * xdum1
# compute expected location of rectified boundaries
y0_reference_lower_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_lower']
y0_reference_middle_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_middle']
y0_reference_upper_expected = \
corr_yrect_a + corr_yrect_b * tmpdict['y0_reference_upper']
# shift transformation to center the rectified slitlet within the
# slitlet bounding box
ydummid = (ydum1 + ydum2) / 2
ioffset = int(ydummid -
(tmpdict['bb_ns1_orig'] + tmpdict['bb_ns2_orig']) / 2.0)
corr_yrect_a -= ioffset
# minimum and maximum row in the rectified slitlet encompassing
# EMIR_NPIXPERSLIT_RECTIFIED pixels
# a) scan number (in pixels, from 1 to NAXIS2)
xdum1 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_lower']
xdum2 = corr_yrect_a + \
corr_yrect_b * tmpdict['y0_frontier_upper']
# b) row number (starting from zero)
min_row_rectified = \
int((round(xdum1 * 10) + 5) / 10) - tmpdict['bb_ns1_orig']
max_row_rectified = \
int((round(xdum2 * 10) - 5) / 10) - tmpdict['bb_ns1_orig']
# save previous results in outdict
outdict['contents'][cslitlet]['y0_reference_lower_expected'] = \
y0_reference_lower_expected
outdict['contents'][cslitlet]['y0_reference_middle_expected'] = \
y0_reference_middle_expected
outdict['contents'][cslitlet]['y0_reference_upper_expected'] = \
y0_reference_upper_expected
outdict['contents'][cslitlet]['corr_yrect_a'] = corr_yrect_a
outdict['contents'][cslitlet]['corr_yrect_b'] = corr_yrect_b
outdict['contents'][cslitlet]['min_row_rectified'] = min_row_rectified
outdict['contents'][cslitlet]['max_row_rectified'] = max_row_rectified
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
master_rectwv.meta_info['origin']['bound_param']
rectwv_coeff.meta_info['origin']['master_rectwv'] = \
'uuid' + master_rectwv.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
dumdict.update({
'csu_bar_left':
csu_conf.csu_bar_left(islitlet),
'csu_bar_right':
csu_conf.csu_bar_right(islitlet),
'csu_bar_slit_center':
csu_conf.csu_bar_slit_center(islitlet),
'csu_bar_slit_width':
csu_conf.csu_bar_slit_width(islitlet),
'x0_reference':
float(EMIR_NAXIS1) / 2.0 + 0.5,
'y0_frontier_lower_expected':
expected_y0_lower_frontier(islitlet),
'y0_frontier_upper_expected':
expected_y0_upper_frontier(islitlet)
})
rectwv_coeff.missing_slitlets.append(islitlet)
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_rect_wpoly.name)
# print('>>> Saving file ' + args.out_rect_wpoly.name)
# check_setstate_getstate(rectwv_coeff, args.out_rect_wpoly.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff
def refine_rectwv_coeff(input_image,
rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'.format(
refine_wavecalib_mode))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data('emirdrp.instrument.configs',
'Oliva_etal_2013.dat')
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1 * lok2]
catlines_reference_flux = catlines_all_flux[lok1 * lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm, maxwidth=maximum_slitlet_width_mm)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
logger.info('list of useful slitlets: {}'.format(list_useful_slitlets))
logger.info(
'list of not useful slitlets: {}'.format(list_not_useful_slitlets))
tempwidths = np.array([
csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets
])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave,
sp_median,
'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum',
show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave,
catlines_reference_flux,
'C4-',
markerfmt=' ',
basefmt='C4-',
label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference,
sp_reference,
'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset * cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot)
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i] * cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1 - offset_array[i] * cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(
stat_summary['robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot,
yplot,
linestyle='',
marker='o',
color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False,
**{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset,
linestyle='--',
color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image
def run(self, rinput):
self.logger.info('starting rect.+wavecal. reduction of stare spectra')
self.logger.info(rinput.master_rectwv)
self.logger.info(
'Wavelength calibration refinement mode....: {}'.format(
rinput.refine_wavecalib_mode))
self.logger.info(
'Minimum slitlet width (mm)................: {}'.format(
rinput.minimum_slitlet_width_mm))
self.logger.info(
'Maximum slitlet width (mm)................: {}'.format(
rinput.maximum_slitlet_width_mm))
self.logger.info(
'Global integer offsets mode...............: {}'.format(
rinput.global_integer_offsets_mode))
self.logger.info(
'Global integer offset X direction (pixels): {}'.format(
rinput.global_integer_offset_x_pix))
self.logger.info(
'Global integer offset Y direction (pixels): {}'.format(
rinput.global_integer_offset_y_pix))
# build object to proceed with bpm, bias, dark and flat
flow = self.init_filters(rinput)
# apply bpm, bias, dark and flat
reduced_image = basic_processing_with_combination(rinput,
flow,
method=sigmaclip)
# update header with additional info
hdr = reduced_image[0].header
self.set_base_headers(hdr)
# save intermediate image in work directory
self.save_intermediate_img(reduced_image, 'reduced_image.fits')
# RectWaveCoeff object with rectification and wavelength
# calibration coefficients for the particular CSU configuration
rectwv_coeff = rectwv_coeff_from_mos_library(reduced_image,
rinput.master_rectwv)
# wavelength calibration refinement
# 0 -> no refinement
# 1 -> apply global offset to all the slitlets (using ARC lines)
# 2 -> apply individual offset to each slitlet (using ARC lines)
# 11 -> apply global offset to all the slitlets (using OH lines)
# 12 -> apply individual offset to each slitlet (using OH lines)
if rinput.refine_wavecalib_mode != 0:
main_header = reduced_image[0].header
# determine useful slitlets
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=rinput.minimum_slitlet_width_mm,
maxwidth=rinput.maximum_slitlet_width_mm)
# remove missing slitlets
if len(rectwv_coeff.missing_slitlets) > 0:
for iremove in rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [
i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets
]
self.logger.info(
'list of useful slitlets: {}'.format(list_useful_slitlets))
self.logger.info('list of unusable slitlets: {}'.format(
list_not_useful_slitlets))
# retrieve arc/OH lines
catlines_all_wave, catlines_all_flux = retrieve_catlines(
rinput.refine_wavecalib_mode, main_header['grism'])
# global integer offsets
if rinput.global_integer_offsets_mode == 'auto':
if (rinput.global_integer_offset_x_pix != 0) or \
(rinput.global_integer_offset_y_pix != 0):
raise ValueError('Global integer offsets must be zero when'
' mode=auto')
# ToDo: include additional airglow emission lines
self.logger.info('computing synthetic image')
# generate synthetic image
synthetic_raw_data = synthetic_lines_rawdata(
catlines_all_wave, catlines_all_flux, list_useful_slitlets,
rectwv_coeff)
synthetic_raw_header = main_header.copy()
synthetic_raw_header['DATE-OBS'] = \
datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
chistory = 'Synthetic image'
synthetic_raw_header.add_history(chistory)
hdu = fits.PrimaryHDU(synthetic_raw_data.astype('float32'),
header=synthetic_raw_header)
synthetic_raw_image = fits.HDUList([hdu])
if self.intermediate_results:
self.save_intermediate_img(synthetic_raw_image,
'synthetic_raw_image.fits')
# cross-correlation to determine global integer offsets
# (rescaling data arrays to [0, 1] before using skimage
# function)
data1_rs, coef1_rs = rescale_array_to_z1z2(
reduced_image[0].data, (0, 1))
data2_rs, coef2_rs = rescale_array_to_z1z2(
synthetic_raw_data, (0, 1))
shifts, error, diffphase = register_translation(
data1_rs, data2_rs, 100)
self.logger.info(
'global_float_offset_x_pix..: {}'.format(-shifts[1]))
self.logger.info(
'global_float_offset_y_pix..: {}'.format(-shifts[0]))
rectwv_coeff.global_integer_offset_x_pix = \
-int(round(shifts[1]))
rectwv_coeff.global_integer_offset_y_pix = \
-int(round(shifts[0]))
self.logger.info('global_integer_offset_x_pix: {}'.format(
rectwv_coeff.global_integer_offset_x_pix))
self.logger.info('global_integer_offset_y_pix: {}'.format(
rectwv_coeff.global_integer_offset_y_pix))
if self.intermediate_results:
data_product = np.fft.fft2(data1_rs) * \
np.fft.fft2(data2_rs).conj()
cc_image = np.fft.fftshift(np.fft.ifft2(data_product))
power = np.log10(cc_image.real)
hdu_power = fits.PrimaryHDU(power)
hdul_power = fits.HDUList([hdu_power])
hdul_power.writeto('power.fits', overwrite=True)
else:
rectwv_coeff.global_integer_offset_x_pix = \
rinput.global_integer_offset_x_pix
rectwv_coeff.global_integer_offset_y_pix = \
rinput.global_integer_offset_y_pix
# apply initial rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
self.logger.info(
'Refining wavelength calibration (mode={})'.format(
rinput.refine_wavecalib_mode))
# refine RectWaveCoeff object
rectwv_coeff, expected_catalog_lines = refine_rectwv_coeff(
reduced_mos,
rectwv_coeff,
catlines_all_wave,
catlines_all_flux,
rinput.refine_wavecalib_mode,
list_useful_slitlets,
save_intermediate_results=self.intermediate_results)
self.save_intermediate_img(expected_catalog_lines,
'expected_catalog_lines.fits')
# apply rectification and wavelength calibration
reduced_mos = apply_rectwv_coeff(reduced_image, rectwv_coeff)
# ds9 region files (to be saved in the work directory)
if self.intermediate_results:
save_four_ds9(rectwv_coeff)
save_spectral_lines_ds9(rectwv_coeff)
# compute median spectra employing the useful region of the
# rectified image
if self.intermediate_results:
for imode, outfile in enumerate([
'median_spectra_full', 'median_spectra_slitlets',
'median_spectrum_slitlets'
]):
median_image = median_slitlets_rectified(reduced_mos,
mode=imode)
self.save_intermediate_img(median_image, outfile + '.fits')
# save results in results directory
self.logger.info('end rect.+wavecal. reduction of stare spectra')
result = self.create_result(reduced_mos=reduced_mos,
rectwv_coeff=rectwv_coeff)
return result
def rectwv_coeff_from_arc_image(reduced_image,
bound_param,
lines_catalog,
args_nbrightlines=None,
args_ymargin_bb=2,
args_remove_sp_background=True,
args_times_sigma_threshold=10,
args_order_fmap=2,
args_sigma_gaussian_filtering=2,
args_margin_npix=50,
args_poldeg_initial=3,
args_poldeg_refined=5,
args_interactive=False,
args_threshold_wv=0,
args_ylogscale=False,
args_pdf=None,
args_geometry=(0,0,640,480),
debugplot=0):
"""Evaluate rect.+wavecal. coefficients from arc image
Parameters
----------
reduced_image : HDUList object
Image with preliminary basic reduction: bpm, bias, dark and
flatfield.
bound_param : RefinedBoundaryModelParam instance
Refined boundary model.
lines_catalog : Numpy array
2D numpy array with the contents of the master file with the
expected arc line wavelengths.
args_nbrightlines : int
TBD
args_ymargin_bb : int
TBD
args_remove_sp_background : bool
TBD
args_times_sigma_threshold : float
TBD
args_order_fmap : int
TBD
args_sigma_gaussian_filtering : float
TBD
args_margin_npix : int
TBD
args_poldeg_initial : int
TBD
args_poldeg_refined : int
TBD
args_interactive : bool
TBD
args_threshold_wv : float
TBD
args_ylogscale : bool
TBD
args_pdf : TBD
args_geometry : TBD
debugplot : int
Debugging level for messages and plots. For details see
'numina.array.display.pause_debugplot.py'.
Returns
-------
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration of the input arc image.
reduced_55sp : HDUList object
Image with 55 spectra corresponding to the median spectrum for
each slitlet, employed to derived the wavelength calibration
polynomial.
"""
logger = logging.getLogger(__name__)
# protections
if args_interactive and args_pdf is not None:
logger.error('--interactive and --pdf are incompatible options')
raise ValueError('--interactive and --pdf are incompatible options')
# header and data array
header = reduced_image[0].header
image2d = reduced_image[0].data
# check grism and filter
filter_name = header['filter']
logger.info('Filter: ' + filter_name)
if filter_name != bound_param.tags['filter']:
raise ValueError('Filter name does not match!')
grism_name = header['grism']
logger.info('Grism: ' + grism_name)
if grism_name != bound_param.tags['grism']:
raise ValueError('Grism name does not match!')
# read the CSU configuration from the image header
csu_conf = CsuConfiguration.define_from_header(header)
logger.debug(csu_conf)
# read the DTU configuration from the image header
dtu_conf = DtuConfiguration.define_from_header(header)
logger.debug(dtu_conf)
# set boundary parameters
parmodel = bound_param.meta_info['parmodel']
params = bound_params_from_dict(bound_param.__getstate__())
if abs(debugplot) >= 10:
print('-' * 83)
print('* FITTED BOUND PARAMETERS')
params.pretty_print()
pause_debugplot(debugplot)
# determine parameters according to grism+filter combination
wv_parameters = set_wv_parameters(filter_name, grism_name)
islitlet_min = wv_parameters['islitlet_min']
islitlet_max = wv_parameters['islitlet_max']
if args_nbrightlines is None:
nbrightlines = wv_parameters['nbrightlines']
else:
nbrightlines = [int(idum) for idum in args_nbrightlines.split(',')]
poly_crval1_linear = wv_parameters['poly_crval1_linear']
poly_cdelt1_linear = wv_parameters['poly_cdelt1_linear']
wvmin_expected = wv_parameters['wvmin_expected']
wvmax_expected = wv_parameters['wvmax_expected']
wvmin_useful = wv_parameters['wvmin_useful']
wvmax_useful = wv_parameters['wvmax_useful']
# list of slitlets to be computed
logger.info('list_slitlets: [' + str(islitlet_min) + ',... ' +
str(islitlet_max) + ']')
# read master arc line wavelengths (only brightest lines)
wv_master = read_wv_master_from_array(
master_table=lines_catalog, lines='brightest', debugplot=debugplot
)
# read master arc line wavelengths (whole data set)
wv_master_all = read_wv_master_from_array(
master_table=lines_catalog, lines='all', debugplot=debugplot
)
# check that the arc lines in the master file are properly sorted
# in ascending order
for i in range(len(wv_master_all) - 1):
if wv_master_all[i] >= wv_master_all[i + 1]:
logger.error('>>> wavelengths: ' +
str(wv_master_all[i]) + ' ' +
str(wv_master_all[i+1]))
raise ValueError('Arc lines are not sorted in master file')
# ---
image2d_55sp = np.zeros((EMIR_NBARS, EMIR_NAXIS1))
# compute rectification transformation and wavelength calibration
# polynomials
measured_slitlets = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet_min <= islitlet <= islitlet_max:
# define Slitlet2dArc object
slt = Slitlet2dArc(
islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=params,
parmodel=parmodel,
debugplot=debugplot
)
# extract 2D image corresponding to the selected slitlet, clipping
# the image beyond the unrectified slitlet (in order to isolate
# the arc lines of the current slitlet; otherwise there are
# problems with arc lines from neighbour slitlets)
image2d_tmp = select_unrectified_slitlet(
image2d=image2d,
islitlet=islitlet,
csu_bar_slit_center=csu_conf.csu_bar_slit_center(islitlet),
params=params,
parmodel=parmodel,
maskonly=False
)
slitlet2d = slt.extract_slitlet2d(image2d_tmp)
# subtract smooth background computed as follows:
# - median collapsed spectrum of the whole slitlet2d
# - independent median filtering of the previous spectrum in the
# two halves in the spectral direction
if args_remove_sp_background:
spmedian = np.median(slitlet2d, axis=0)
naxis1_tmp = spmedian.shape[0]
jmidpoint = naxis1_tmp // 2
sp1 = medfilt(spmedian[:jmidpoint], [201])
sp2 = medfilt(spmedian[jmidpoint:], [201])
spbackground = np.concatenate((sp1, sp2))
slitlet2d -= spbackground
# locate unknown arc lines
slt.locate_unknown_arc_lines(
slitlet2d=slitlet2d,
times_sigma_threshold=args_times_sigma_threshold)
# continue working with current slitlet only if arc lines have
# been detected
if slt.list_arc_lines is not None:
# compute intersections between spectrum trails and arc lines
slt.xy_spectrail_arc_intersections(slitlet2d=slitlet2d)
# compute rectification transformation
slt.estimate_tt_to_rectify(order=args_order_fmap,
slitlet2d=slitlet2d)
# rectify image
slitlet2d_rect = slt.rectify(slitlet2d,
resampling=2,
transformation=1)
# median spectrum and line peaks from rectified image
sp_median, fxpeaks = slt.median_spectrum_from_rectified_image(
slitlet2d_rect,
sigma_gaussian_filtering=args_sigma_gaussian_filtering,
nwinwidth_initial=5,
nwinwidth_refined=5,
times_sigma_threshold=5,
npix_avoid_border=6,
nbrightlines=nbrightlines
)
image2d_55sp[islitlet - 1, :] = sp_median
# determine expected wavelength limits prior to the wavelength
# calibration
csu_bar_slit_center = csu_conf.csu_bar_slit_center(islitlet)
crval1_linear = poly_crval1_linear(csu_bar_slit_center)
cdelt1_linear = poly_cdelt1_linear(csu_bar_slit_center)
expected_wvmin = crval1_linear - \
args_margin_npix * cdelt1_linear
naxis1_linear = sp_median.shape[0]
crvaln_linear = crval1_linear + \
(naxis1_linear - 1) * cdelt1_linear
expected_wvmax = crvaln_linear + \
args_margin_npix * cdelt1_linear
# override previous estimates when necessary
if wvmin_expected is not None:
expected_wvmin = wvmin_expected
if wvmax_expected is not None:
expected_wvmax = wvmax_expected
# clip initial master arc line list with bright lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master
lok2 = wv_master <= expected_wvmax
lok = lok1 * lok2
wv_master_eff = wv_master[lok]
# perform initial wavelength calibration
solution_wv = wvcal_spectrum(
sp=sp_median,
fxpeaks=fxpeaks,
poly_degree_wfit=args_poldeg_initial,
wv_master=wv_master_eff,
wv_ini_search=expected_wvmin,
wv_end_search=expected_wvmax,
wvmin_useful=wvmin_useful,
wvmax_useful=wvmax_useful,
geometry=args_geometry,
debugplot=slt.debugplot
)
# store initial wavelength calibration polynomial in current
# slitlet instance
slt.wpoly = np.polynomial.Polynomial(solution_wv.coeff)
pause_debugplot(debugplot)
# clip initial master arc line list with all the lines to
# the expected wavelength range
lok1 = expected_wvmin <= wv_master_all
lok2 = wv_master_all <= expected_wvmax
lok = lok1 * lok2
wv_master_all_eff = wv_master_all[lok]
# clip master arc line list to useful region
if wvmin_useful is not None:
lok = wvmin_useful <= wv_master_all_eff
wv_master_all_eff = wv_master_all_eff[lok]
if wvmax_useful is not None:
lok = wv_master_all_eff <= wvmax_useful
wv_master_all_eff = wv_master_all_eff[lok]
# refine wavelength calibration
if args_poldeg_refined > 0:
plottitle = '[slitlet#{}, refined]'.format(islitlet)
poly_refined, yres_summary = refine_arccalibration(
sp=sp_median,
poly_initial=slt.wpoly,
wv_master=wv_master_all_eff,
poldeg=args_poldeg_refined,
ntimes_match_wv=1,
interactive=args_interactive,
threshold=args_threshold_wv,
plottitle=plottitle,
ylogscale=args_ylogscale,
geometry=args_geometry,
pdf=args_pdf,
debugplot=slt.debugplot
)
# store refined wavelength calibration polynomial in
# current slitlet instance
slt.wpoly = poly_refined
# compute approximate linear values for CRVAL1 and CDELT1
naxis1_linear = sp_median.shape[0]
crmin1_linear = slt.wpoly(1)
crmax1_linear = slt.wpoly(naxis1_linear)
slt.crval1_linear = crmin1_linear
slt.cdelt1_linear = \
(crmax1_linear - crmin1_linear) / (naxis1_linear - 1)
# check that the trimming of wv_master and wv_master_all has
# preserved the wavelength range [crmin1_linear, crmax1_linear]
if crmin1_linear < expected_wvmin:
logger.warning(">>> islitlet: " +str(islitlet))
logger.warning("expected_wvmin: " + str(expected_wvmin))
logger.warning("crmin1_linear.: " + str(crmin1_linear))
logger.warning("WARNING: Unexpected crmin1_linear < "
"expected_wvmin")
if crmax1_linear > expected_wvmax:
logger.warning(">>> islitlet: " +str(islitlet))
logger.warning("expected_wvmax: " + str(expected_wvmax))
logger.warning("crmax1_linear.: " + str(crmax1_linear))
logger.warning("WARNING: Unexpected crmax1_linear > "
"expected_wvmax")
cout += '.'
else:
cout += 'x'
if islitlet % 10 == 0:
if cout != 'x':
cout = str(islitlet // 10)
if debugplot != 0:
pause_debugplot(debugplot)
else:
# define Slitlet2dArc object
slt = Slitlet2dArc(
islitlet=islitlet,
csu_conf=csu_conf,
ymargin_bb=args_ymargin_bb,
params=None,
parmodel=None,
debugplot=debugplot
)
cout += 'i'
# store current slitlet in list of measured slitlets
measured_slitlets.append(slt)
logger.info(cout)
# ---
# generate FITS file structure with 55 spectra corresponding to the
# median spectrum for each slitlet
reduced_55sp = fits.PrimaryHDU(data=image2d_55sp)
reduced_55sp.header['crpix1'] = (0.0, 'reference pixel')
reduced_55sp.header['crval1'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt1'] = (1.0, 'increment')
reduced_55sp.header['ctype1'] = 'PIXEL'
reduced_55sp.header['cunit1'] = ('Pixel', 'units along axis2')
reduced_55sp.header['crpix2'] = (0.0, 'reference pixel')
reduced_55sp.header['crval2'] = (0.0, 'central value at crpix2')
reduced_55sp.header['cdelt2'] = (1.0, 'increment')
reduced_55sp.header['ctype2'] = 'PIXEL'
reduced_55sp.header['cunit2'] = ('Pixel', 'units along axis2')
# ---
# Generate structure to store intermediate results
outdict = {}
outdict['instrument'] = 'EMIR'
outdict['meta_info'] = {}
outdict['meta_info']['creation_date'] = datetime.now().isoformat()
outdict['meta_info']['description'] = \
'computation of rectification and wavelength calibration polynomial ' \
'coefficients for a particular CSU configuration'
outdict['meta_info']['recipe_name'] = 'undefined'
outdict['meta_info']['origin'] = {}
outdict['meta_info']['origin']['bound_param_uuid'] = \
bound_param.uuid
outdict['meta_info']['origin']['arc_image_uuid'] = 'undefined'
outdict['tags'] = {}
outdict['tags']['grism'] = grism_name
outdict['tags']['filter'] = filter_name
outdict['tags']['islitlet_min'] = islitlet_min
outdict['tags']['islitlet_max'] = islitlet_max
outdict['dtu_configuration'] = dtu_conf.outdict()
outdict['uuid'] = str(uuid4())
outdict['contents'] = {}
missing_slitlets = []
for slt in measured_slitlets:
islitlet = slt.islitlet
if islitlet_min <= islitlet <= islitlet_max:
# avoid error when creating a python list of coefficients from
# numpy polynomials when the polynomials do not exist (note that
# the JSON format doesn't handle numpy arrays and such arrays must
# be transformed into native python lists)
if slt.wpoly is None:
wpoly_coeff = None
else:
wpoly_coeff = slt.wpoly.coef.tolist()
if slt.wpoly_longslit_model is None:
wpoly_coeff_longslit_model = None
else:
wpoly_coeff_longslit_model = \
slt.wpoly_longslit_model.coef.tolist()
# avoid similar error when creating a python list of coefficients
# when the numpy array does not exist; note that this problem
# does not happen with tt?_aij_longslit_model and
# tt?_bij_longslit_model because the latter have already been
# created as native python lists
if slt.ttd_aij is None:
ttd_aij = None
else:
ttd_aij = slt.ttd_aij.tolist()
if slt.ttd_bij is None:
ttd_bij = None
else:
ttd_bij = slt.ttd_bij.tolist()
if slt.tti_aij is None:
tti_aij = None
else:
tti_aij = slt.tti_aij.tolist()
if slt.tti_bij is None:
tti_bij = None
else:
tti_bij = slt.tti_bij.tolist()
# creating temporary dictionary with the information corresponding
# to the current slitlett that will be saved in the JSON file
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_reference_lower': slt.y0_reference_lower,
'y0_reference_middle': slt.y0_reference_middle,
'y0_reference_upper': slt.y0_reference_upper,
'y0_reference_lower_expected':
slt.y0_reference_lower_expected,
'y0_reference_middle_expected':
slt.y0_reference_middle_expected,
'y0_reference_upper_expected':
slt.y0_reference_upper_expected,
'y0_frontier_lower': slt.y0_frontier_lower,
'y0_frontier_upper': slt.y0_frontier_upper,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected,
'corr_yrect_a': slt.corr_yrect_a,
'corr_yrect_b': slt.corr_yrect_b,
'min_row_rectified': slt.min_row_rectified,
'max_row_rectified': slt.max_row_rectified,
'ymargin_bb': slt.ymargin_bb,
'bb_nc1_orig': slt.bb_nc1_orig,
'bb_nc2_orig': slt.bb_nc2_orig,
'bb_ns1_orig': slt.bb_ns1_orig,
'bb_ns2_orig': slt.bb_ns2_orig,
'spectrail': {
'poly_coef_lower':
slt.list_spectrails[
slt.i_lower_spectrail].poly_funct.coef.tolist(),
'poly_coef_middle':
slt.list_spectrails[
slt.i_middle_spectrail].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_spectrails[
slt.i_upper_spectrail].poly_funct.coef.tolist(),
},
'frontier': {
'poly_coef_lower':
slt.list_frontiers[0].poly_funct.coef.tolist(),
'poly_coef_upper':
slt.list_frontiers[1].poly_funct.coef.tolist(),
},
'ttd_order': slt.ttd_order,
'ttd_aij': ttd_aij,
'ttd_bij': ttd_bij,
'tti_aij': tti_aij,
'tti_bij': tti_bij,
'ttd_order_longslit_model': slt.ttd_order_longslit_model,
'ttd_aij_longslit_model': slt.ttd_aij_longslit_model,
'ttd_bij_longslit_model': slt.ttd_bij_longslit_model,
'tti_aij_longslit_model': slt.tti_aij_longslit_model,
'tti_bij_longslit_model': slt.tti_bij_longslit_model,
'wpoly_coeff': wpoly_coeff,
'wpoly_coeff_longslit_model': wpoly_coeff_longslit_model,
'crval1_linear': slt.crval1_linear,
'cdelt1_linear': slt.cdelt1_linear
}
else:
missing_slitlets.append(islitlet)
tmp_dict = {
'csu_bar_left': slt.csu_bar_left,
'csu_bar_right': slt.csu_bar_right,
'csu_bar_slit_center': slt.csu_bar_slit_center,
'csu_bar_slit_width': slt.csu_bar_slit_width,
'x0_reference': slt.x0_reference,
'y0_frontier_lower_expected': slt.y0_frontier_lower_expected,
'y0_frontier_upper_expected': slt.y0_frontier_upper_expected
}
slitlet_label = "slitlet" + str(islitlet).zfill(2)
outdict['contents'][slitlet_label] = tmp_dict
# ---
# OBSOLETE
'''
# save JSON file needed to compute the MOS model
with open(args.out_json.name, 'w') as fstream:
json.dump(outdict, fstream, indent=2, sort_keys=True)
print('>>> Saving file ' + args.out_json.name)
'''
# ---
# Create object of type RectWaveCoeff with coefficients for
# rectification and wavelength calibration
rectwv_coeff = RectWaveCoeff(instrument='EMIR')
rectwv_coeff.quality_control = numina.types.qc.QC.GOOD
rectwv_coeff.tags['grism'] = grism_name
rectwv_coeff.tags['filter'] = filter_name
rectwv_coeff.meta_info['origin']['bound_param'] = \
'uuid' + bound_param.uuid
rectwv_coeff.meta_info['dtu_configuration'] = outdict['dtu_configuration']
rectwv_coeff.total_slitlets = EMIR_NBARS
rectwv_coeff.missing_slitlets = missing_slitlets
for i in range(EMIR_NBARS):
islitlet = i + 1
dumdict = {'islitlet': islitlet}
cslitlet = 'slitlet' + str(islitlet).zfill(2)
if cslitlet in outdict['contents']:
dumdict.update(outdict['contents'][cslitlet])
else:
raise ValueError("Unexpected error")
rectwv_coeff.contents.append(dumdict)
# debugging __getstate__ and __setstate__
# rectwv_coeff.writeto(args.out_json.name)
# print('>>> Saving file ' + args.out_json.name)
# check_setstate_getstate(rectwv_coeff, args.out_json.name)
logger.info('Generating RectWaveCoeff object with uuid=' +
rectwv_coeff.uuid)
return rectwv_coeff, reduced_55sp
def refine_rectwv_coeff(input_image, rectwv_coeff,
refine_wavecalib_mode,
minimum_slitlet_width_mm,
maximum_slitlet_width_mm,
save_intermediate_results=False,
debugplot=0):
"""Refine RectWaveCoeff object using a catalogue of lines
One and only one among refine_with_oh_lines_mode and
refine_with_arc_lines must be different from zero.
Parameters
----------
input_image : HDUList object
Input 2D image.
rectwv_coeff : RectWaveCoeff instance
Rectification and wavelength calibration coefficients for the
particular CSU configuration.
refine_wavecalib_mode : int
Integer, indicating the type of refinement:
0 : no refinement
1 : apply the same global offset to all the slitlets (using ARC lines)
2 : apply individual offset to each slitlet (using ARC lines)
11 : apply the same global offset to all the slitlets (using OH lines)
12 : apply individual offset to each slitlet (using OH lines)
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
save_intermediate_results : bool
If True, save plots in PDF files
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
refined_rectwv_coeff : RectWaveCoeff instance
Refined rectification and wavelength calibration coefficients
for the particular CSU configuration.
expected_cat_image : HDUList object
Output 2D image with the expected catalogued lines.
"""
logger = logging.getLogger(__name__)
if save_intermediate_results:
from matplotlib.backends.backend_pdf import PdfPages
pdf = PdfPages('crosscorrelation.pdf')
else:
pdf = None
# image header
main_header = input_image[0].header
filter_name = main_header['filter']
grism_name = main_header['grism']
# protections
if refine_wavecalib_mode not in [1, 2, 11, 12]:
logger.error('Wavelength calibration refinemente mode={}'. format(
refine_wavecalib_mode
))
raise ValueError("Invalid wavelength calibration refinement mode")
# read tabulated lines
if refine_wavecalib_mode in [1, 2]: # ARC lines
if grism_name == 'LR':
catlines_file = 'lines_argon_neon_xenon_empirical_LR.dat'
else:
catlines_file = 'lines_argon_neon_xenon_empirical.dat'
dumdata = pkgutil.get_data('emirdrp.instrument.configs', catlines_file)
arc_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(arc_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = catlines[:, 0]
catlines_all_flux = catlines[:, 1]
mode = refine_wavecalib_mode
elif refine_wavecalib_mode in [11, 12]: # OH lines
dumdata = pkgutil.get_data(
'emirdrp.instrument.configs',
'Oliva_etal_2013.dat'
)
oh_lines_tmpfile = StringIO(dumdata.decode('utf8'))
catlines = np.genfromtxt(oh_lines_tmpfile)
# define wavelength and flux as separate arrays
catlines_all_wave = np.concatenate((catlines[:, 1], catlines[:, 0]))
catlines_all_flux = np.concatenate((catlines[:, 2], catlines[:, 2]))
mode = refine_wavecalib_mode - 10
else:
raise ValueError('Unexpected mode={}'.format(refine_wavecalib_mode))
# initialize output
refined_rectwv_coeff = deepcopy(rectwv_coeff)
logger.info('Computing median spectrum')
# compute median spectrum and normalize it
sp_median = median_slitlets_rectified(
input_image,
mode=2,
minimum_slitlet_width_mm=minimum_slitlet_width_mm,
maximum_slitlet_width_mm=maximum_slitlet_width_mm
)[0].data
sp_median /= sp_median.max()
# determine minimum and maximum useful wavelength
jmin, jmax = find_pix_borders(sp_median, 0)
naxis1 = main_header['naxis1']
naxis2 = main_header['naxis2']
crpix1 = main_header['crpix1']
crval1 = main_header['crval1']
cdelt1 = main_header['cdelt1']
xwave = crval1 + (np.arange(naxis1) + 1.0 - crpix1) * cdelt1
if grism_name == 'LR':
wv_parameters = set_wv_parameters(filter_name, grism_name)
wave_min = wv_parameters['wvmin_useful']
wave_max = wv_parameters['wvmax_useful']
else:
wave_min = crval1 + (jmin + 1 - crpix1) * cdelt1
wave_max = crval1 + (jmax + 1 - crpix1) * cdelt1
logger.info('Setting wave_min to {}'.format(wave_min))
logger.info('Setting wave_max to {}'.format(wave_max))
# extract subset of catalogue lines within current wavelength range
lok1 = catlines_all_wave >= wave_min
lok2 = catlines_all_wave <= wave_max
catlines_reference_wave = catlines_all_wave[lok1*lok2]
catlines_reference_flux = catlines_all_flux[lok1*lok2]
catlines_reference_flux /= catlines_reference_flux.max()
# estimate sigma to broaden catalogue lines
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm
)
# remove missing slitlets
if len(refined_rectwv_coeff.missing_slitlets) > 0:
for iremove in refined_rectwv_coeff.missing_slitlets:
if iremove in list_useful_slitlets:
list_useful_slitlets.remove(iremove)
list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets]
logger.info('list of useful slitlets: {}'.format(
list_useful_slitlets))
logger.info('list of not useful slitlets: {}'.format(
list_not_useful_slitlets))
tempwidths = np.array([csu_config.csu_bar_slit_width(islitlet)
for islitlet in list_useful_slitlets])
widths_summary = summary(tempwidths)
logger.info('Statistics of useful slitlet widths (mm):')
logger.info('- npoints....: {0:d}'.format(widths_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(widths_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(widths_summary['median']))
logger.info('- std........: {0:7.3f}'.format(widths_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(widths_summary['robust_std']))
# empirical transformation of slit width (mm) to pixels
sigma_broadening = cdelt1 * widths_summary['median']
# convolve location of catalogue lines to generate expected spectrum
xwave_reference, sp_reference = convolve_comb_lines(
catlines_reference_wave, catlines_reference_flux, sigma_broadening,
crpix1, crval1, cdelt1, naxis1
)
sp_reference /= sp_reference.max()
# generate image2d with expected lines
image2d_expected_lines = np.tile(sp_reference, (naxis2, 1))
hdu = fits.PrimaryHDU(data=image2d_expected_lines, header=main_header)
expected_cat_image = fits.HDUList([hdu])
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xwave, sp_median, 'C1-',
xlabel='Wavelength (Angstroms, in vacuum)',
ylabel='Normalized number of counts',
title='Median spectrum',
label='observed spectrum', show=False)
# overplot reference catalogue lines
ax.stem(catlines_reference_wave, catlines_reference_flux, 'C4-',
markerfmt=' ', basefmt='C4-', label='tabulated lines')
# overplot convolved reference lines
ax.plot(xwave_reference, sp_reference, 'C0-',
label='expected spectrum')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
# compute baseline signal in sp_median
baseline = np.percentile(sp_median[sp_median > 0], q=10)
if (abs(debugplot) % 10 != 0) or (pdf is not None):
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(sp_median, bins=1000, log=True)
ax.set_xlabel('Normalized number of counts')
ax.set_ylabel('Number of pixels')
ax.set_title('Median spectrum')
ax.axvline(float(baseline), linestyle='--', color='grey')
if pdf is not None:
pdf.savefig()
else:
geometry = (0, 0, 640, 480)
set_window_geometry(geometry)
plt.show()
# subtract baseline to sp_median (only pixels with signal above zero)
lok = np.where(sp_median > 0)
sp_median[lok] -= baseline
# compute global offset through periodic correlation
logger.info('Computing global offset')
global_offset, fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=sp_median,
fminmax=None,
naround_zero=50,
plottitle='Median spectrum (cross-correlation)',
pdf=pdf,
debugplot=debugplot
)
logger.info('Global offset: {} pixels'.format(-global_offset))
missing_slitlets = rectwv_coeff.missing_slitlets
if mode == 1:
# apply computed offset to obtain refined_rectwv_coeff_global
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet not in missing_slitlets:
i = islitlet - 1
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= global_offset*cdelt1
elif mode == 2:
# compute individual offset for each slitlet
logger.info('Computing individual offsets')
median_55sp = median_slitlets_rectified(input_image, mode=1)
offset_array = np.zeros(EMIR_NBARS)
xplot = []
yplot = []
xplot_skipped = []
yplot_skipped = []
cout = '0'
for islitlet in range(1, EMIR_NBARS + 1):
if islitlet in list_useful_slitlets:
i = islitlet - 1
sp_median = median_55sp[0].data[i, :]
lok = np.where(sp_median > 0)
if np.any(lok):
baseline = np.percentile(sp_median[lok], q=10)
sp_median[lok] -= baseline
sp_median /= sp_median.max()
offset_array[i], fpeak = periodic_corr1d(
sp_reference=sp_reference,
sp_offset=median_55sp[0].data[i, :],
fminmax=None,
naround_zero=50,
plottitle='slitlet #{0} (cross-correlation)'.format(
islitlet),
pdf=pdf,
debugplot=debugplot
)
else:
offset_array[i] = 0.0
dumdict = refined_rectwv_coeff.contents[i]
dumdict['wpoly_coeff'][0] -= offset_array[i]*cdelt1
xplot.append(islitlet)
yplot.append(-offset_array[i])
# second correction
wpoly_coeff_refined = check_wlcalib_sp(
sp=median_55sp[0].data[i, :],
crpix1=crpix1,
crval1=crval1-offset_array[i]*cdelt1,
cdelt1=cdelt1,
wv_master=catlines_reference_wave,
coeff_ini=dumdict['wpoly_coeff'],
naxis1_ini=EMIR_NAXIS1,
title='slitlet #{0} (after applying offset)'.format(
islitlet),
ylogscale=False,
pdf=pdf,
debugplot=debugplot
)
dumdict['wpoly_coeff'] = wpoly_coeff_refined
cout += '.'
else:
xplot_skipped.append(islitlet)
yplot_skipped.append(0)
cout += 'i'
if islitlet % 10 == 0:
if cout != 'i':
cout = str(islitlet // 10)
logger.info(cout)
# show offsets with opposite sign
stat_summary = summary(np.array(yplot))
logger.info('Statistics of individual slitlet offsets (pixels):')
logger.info('- npoints....: {0:d}'.format(stat_summary['npoints']))
logger.info('- mean.......: {0:7.3f}'.format(stat_summary['mean']))
logger.info('- median.....: {0:7.3f}'.format(stat_summary['median']))
logger.info('- std........: {0:7.3f}'.format(stat_summary['std']))
logger.info('- robust_std.: {0:7.3f}'.format(stat_summary[
'robust_std']))
if (abs(debugplot) % 10 != 0) or (pdf is not None):
ax = ximplotxy(xplot, yplot,
linestyle='', marker='o', color='C0',
xlabel='slitlet number',
ylabel='-offset (pixels) = offset to be applied',
title='cross-correlation result',
show=False, **{'label': 'individual slitlets'})
if len(xplot_skipped) > 0:
ax.plot(xplot_skipped, yplot_skipped, 'mx')
ax.axhline(-global_offset, linestyle='--', color='C1',
label='global offset')
ax.legend()
if pdf is not None:
pdf.savefig()
else:
pause_debugplot(debugplot=debugplot, pltshow=True)
else:
raise ValueError('Unexpected mode={}'.format(mode))
# close output PDF file
if pdf is not None:
pdf.close()
# return result
return refined_rectwv_coeff, expected_cat_image
def main(args=None):
# parse command-line options
parser = argparse.ArgumentParser(
description='description: compute median spectrum for each slitlet')
# positional arguments
parser.add_argument("fitsfile",
help="Input FITS file name",
type=argparse.FileType('rb'))
parser.add_argument("outfile",
help="Output FITS file name",
type=lambda x: arg_file_is_new(parser, x, mode='wb'))
# optional arguments
parser.add_argument("--mode",
help="Output type: 0 -> full frame (default), "
"1 -> individual slitlets, "
"2 -> collapsed single spectrum)",
default=0,
type=int,
choices=[0, 1, 2])
parser.add_argument("--minimum_slitlet_width_mm",
help="Minimum slitlet width (mm) for --mode 2 "
"(default=0)",
default=EMIR_MINIMUM_SLITLET_WIDTH_MM,
type=float)
parser.add_argument("--maximum_slitlet_width_mm",
help="Maximum slitlet width (mm) for --mode 2 "
"(default=" + str(EMIR_MAXIMUM_SLITLET_WIDTH_MM) + ")",
default=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
type=float)
parser.add_argument("--debugplot",
help="Integer indicating plotting/debugging" +
" (default=0)",
default=0,
type=int,
choices=DEBUGPLOT_CODES)
parser.add_argument("--echo",
help="Display full command line",
action="store_true")
args = parser.parse_args(args=args)
if args.echo:
print('\033[1m\033[31mExecuting: ' + ' '.join(sys.argv) + '\033[0m\n')
# read input FITS file
hdulist = fits.open(args.fitsfile)
# determine useful slitlets
main_header = hdulist[0].header
csu_config = CsuConfiguration.define_from_header(main_header)
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=args.minimum_slitlet_width_mm,
maxwidth=args.maximum_slitlet_width_mm)
image_median = median_slitlets_rectified(
hdulist,
mode=args.mode,
list_useful_slitlets=list_useful_slitlets,
debugplot=args.debugplot)
# save result
image_median.writeto(args.outfile, overwrite=True)
def median_slitlets_rectified(
input_image,
mode=0,
minimum_slitlet_width_mm=EMIR_MINIMUM_SLITLET_WIDTH_MM,
maximum_slitlet_width_mm=EMIR_MAXIMUM_SLITLET_WIDTH_MM,
debugplot=0
):
"""Compute median spectrum for each slitlet.
Parameters
----------
input_image : HDUList object
Input 2D image.
mode : int
Indicate desired result:
0 : image with the same size as the input image, with the
median spectrum of each slitlet spanning all the spectra
of the corresponding slitlet
1 : image with 55 spectra, containing the median spectra of
each slitlet
2 : single collapsed median spectrum, using exclusively the
useful slitlets from the input image
minimum_slitlet_width_mm : float
Minimum slitlet width (mm) for a valid slitlet.
maximum_slitlet_width_mm : float
Maximum slitlet width (mm) for a valid slitlet.
debugplot : int
Determines whether intermediate computations and/or plots
are displayed. The valid codes are defined in
numina.array.display.pause_debugplot.
Returns
-------
image_median : HDUList object
Output image.
"""
image_header = input_image[0].header
image2d = input_image[0].data
# check image dimensions
naxis2_expected = EMIR_NBARS * EMIR_NPIXPERSLIT_RECTIFIED
naxis2, naxis1 = image2d.shape
if naxis2 != naxis2_expected:
raise ValueError("NAXIS2={0} should be {1}".format(
naxis2, naxis2_expected
))
# check that the FITS file has been obtained with EMIR
instrument = image_header['instrume']
if instrument != 'EMIR':
raise ValueError("INSTRUME keyword is not 'EMIR'!")
# initialize output image
if mode == 0:
image2d_median = np.zeros((naxis2, naxis1))
else:
image2d_median = np.zeros((EMIR_NBARS, naxis1))
# main loop
for i in range(EMIR_NBARS):
ns1 = i * EMIR_NPIXPERSLIT_RECTIFIED + 1
ns2 = ns1 + EMIR_NPIXPERSLIT_RECTIFIED - 1
sp_median = np.median(image2d[(ns1-1):ns2, :], axis=0)
if mode == 0:
image2d_median[(ns1-1):ns2, :] = np.tile(
sp_median, (EMIR_NPIXPERSLIT_RECTIFIED, 1)
)
else:
image2d_median[i] = np.copy(sp_median)
if mode == 2:
# get CSU configuration from FITS header
csu_config = CsuConfiguration.define_from_header(image_header)
# define wavelength calibration parameters
crpix1 = image_header['crpix1']
crval1 = image_header['crval1']
cdelt1 = image_header['cdelt1']
# segregate slitlets
list_useful_slitlets = csu_config.widths_in_range_mm(
minwidth=minimum_slitlet_width_mm,
maxwidth=maximum_slitlet_width_mm
)
list_not_useful_slitlets = [i for i in list(range(1, EMIR_NBARS + 1))
if i not in list_useful_slitlets]
if abs(debugplot) != 0:
print('>>> list_useful_slitlets....:', list_useful_slitlets)
print('>>> list_not_useful_slitlets:', list_not_useful_slitlets)
# define mask from array data
mask2d, borders = define_mask_borders(image2d_median, sought_value=0)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int), z1z2=(-.2, 1.2), crpix1=crpix1,
crval1=crval1, cdelt1=cdelt1, debugplot=debugplot)
# update mask with unused slitlets
for islitlet in list_not_useful_slitlets:
mask2d[islitlet - 1, :] = np.array([True] * naxis1)
if abs(debugplot) % 10 != 0:
ximshow(mask2d.astype(int), z1z2=(-.2, 1.2), crpix1=crpix1,
crval1=crval1, cdelt1=cdelt1, debugplot=debugplot)
# useful image pixels
image2d_masked = image2d_median * (1 - mask2d.astype(int))
if abs(debugplot) % 10 != 0:
ximshow(image2d_masked, crpix1=crpix1, crval1=crval1,
cdelt1=cdelt1, debugplot=debugplot)
# masked image
image2d_masked = np.ma.masked_array(image2d_median, mask=mask2d)
# median spectrum
image1d_median = np.ma.median(image2d_masked, axis=0).data
image_median = fits.PrimaryHDU(data=image1d_median,
header=image_header)
else:
image_median = fits.PrimaryHDU(data=image2d_median,
header=image_header)
return fits.HDUList([image_median])