def build_traces(self, show_peaks=False, debug=False):
"""
Main routine to generate the align profile traces in all slits
Args:
show_peaks (bool, optional):
Generate QA showing peaks identified by alignment profile tracing
show_trace (bool, optional):
Generate QA showing traces identified. Requires an open ginga RC modules window
debug (bool, optional):
Debug the alignment tracing algorithm
Returns:
dict: self.align_dict
"""
# Generate slits
slitid_img_init = self.slits.slit_img(initial=True)
left, right, _ = self.slits.select_edges(initial=True)
align_prof = dict({})
# Go through the slits
for slit_idx, slit_spat in enumerate(self.slits.spat_id):
specobj_dict = {
'SLITID': slit_idx,
'DET': self.det,
'OBJTYPE': "align_profile",
'PYPELINE': self.spectrograph.pypeline
}
msgs.info(
"Fitting alignment traces in slit {0:d}".format(slit_idx))
align_traces, _ = extract.objfind(
self.rawalignimg.image,
slitid_img_init == slit_spat,
left[:, slit_idx],
right[:, slit_idx],
ir_redux=False,
ncoeff=self.alignpar['trace_npoly'],
specobj_dict=specobj_dict,
sig_thresh=self.alignpar['sig_thresh'],
show_peaks=show_peaks,
show_fits=False,
trim_edg=self.alignpar['trim_edge'],
cont_fit=False,
npoly_cont=0,
nperslit=len(self.alignpar['locations']))
if len(align_traces) != len(self.alignpar['locations']):
# Align tracing has failed for this slit
msgs.error("Alignment tracing has failed on slit {0:d}".format(
slit_idx))
align_prof['{0:d}'.format(slit_idx)] = align_traces.copy()
# Steps
self.steps.append(inspect.stack()[0][3])
# Return
return align_prof
def find_objects_pypeline(self,
image,
std_trace=None,
manual_extract_dict=None,
show_peaks=False,
show_fits=False,
show_trace=False,
show=False,
debug=False):
"""
Pipeline specific find objects routine
Args:
image (np.ndarray):
std_trace (np.ndarray, optional):
manual_extract_dict (dict, optional):
show_peaks (bool, optional):
Generate QA showing peaks identified by object finding
show_fits (bool, optional):
Generate QA showing fits to traces
show_trace (bool, optional):
Generate QA showing traces identified. Requires an open ginga RC modules window
show (bool, optional):
debug (bool, optional):
Returns:
tuple:
specobjs : Specobjs object
Container holding Specobj objects
nobj (int):
Number of objects identified
skymask : ndarray
Boolean image indicating which pixels are useful for global sky subtraction
"""
gdslits = np.where(np.invert(self.reduce_bpm))[0]
# create the ouptut image for skymask
skymask = np.zeros_like(image, dtype=bool)
# Instantiate the specobjs container
sobjs = specobjs.SpecObjs()
# Loop on slits
for slit_idx in gdslits:
slit_spat = self.slits.spat_id[slit_idx]
qa_title = "Finding objects on slit # {:d}".format(slit_spat)
msgs.info(qa_title)
thismask = self.slitmask == slit_spat
inmask = (self.sciImg.fullmask == 0) & thismask
# Find objects
specobj_dict = {
'SLITID': slit_spat,
'DET': self.det,
'OBJTYPE': self.objtype,
'PYPELINE': self.pypeline
}
# TODO we need to add QA paths and QA hooks. QA should be
# done through objfind where all the relevant information
# is. This will be a png file(s) per slit.
sobjs_slit, skymask[thismask] = \
extract.objfind(image, thismask,
self.slits_left[:,slit_idx],
self.slits_right[:,slit_idx],
inmask=inmask, ir_redux=self.ir_redux,
ncoeff=self.par['reduce']['findobj']['trace_npoly'],
std_trace=std_trace,
sig_thresh=self.par['reduce']['findobj']['sig_thresh'],
hand_extract_dict=manual_extract_dict,
specobj_dict=specobj_dict, show_peaks=show_peaks,
show_fits=show_fits, show_trace=show_trace,
trim_edg=self.par['reduce']['findobj']['find_trim_edge'],
cont_fit=self.par['reduce']['findobj']['find_cont_fit'],
npoly_cont=self.par['reduce']['findobj']['find_npoly_cont'],
fwhm=self.par['reduce']['findobj']['find_fwhm'],
maxdev=self.par['reduce']['findobj']['find_maxdev'],
qa_title=qa_title, nperslit=self.par['reduce']['findobj']['maxnumber'],
debug_all=debug)
sobjs.add_sobj(sobjs_slit)
# Steps
self.steps.append(inspect.stack()[0][3])
if show:
self.show('image',
image=image * (self.sciImg.fullmask == 0),
chname='objfind',
sobjs=sobjs,
slits=True)
# Return
return sobjs, len(sobjs), skymask
def ech_objfind(image,
ivar,
ordermask,
slit_left,
slit_righ,
inmask=None,
plate_scale=0.2,
npca=2,
ncoeff=5,
min_snr=0.0,
nabove_min_snr=0,
pca_percentile=20.0,
snr_pca=3.0,
box_radius=2.0,
show_peaks=False,
show_fits=False,
show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale, (float, int)):
plate_scale_ord = np.full(
norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale, (np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error(
'Invalid size for plate_scale. It must either have one element or norders elements'
)
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec / 2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec,
slit_left[:, iord]),
np.interp(slit_spec_pos, spec_vec,
slit_righ[:, iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks = True
show_fits = True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {
'setup': 'HIRES',
'slitid': iord + 1,
'scidx': 0,
'det': 1,
'objtype': 'science'
}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0 * fracpos
if nfound > 1:
(ingroup, multgroup, firstgroup,
nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound == 1:
group = np.zeros(1, dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx
== iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape,
slit_spat_pos[iord, :],
slit_spec_pos,
det=sobjs_align[0].det,
setup=sobjs_align[0].setup,
slitid=(iord + 1),
scidx=sobjs_align[0].scidx,
objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,
iord] + slit_width[:, iord] * uni_frac[
iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(
np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group
== uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius / plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar * 0 + 1.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec) != pixtot)
flux_box[:, iord, iobj] = flux_tmp * mask_tmp
ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0)
mask_box[:, iord, iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(
flux_box[mask_tmp, iord, iobj] *
np.sqrt(ivar_box[mask_tmp, iord, iobj]),
sigma_lower=5.0,
sigma_upper=5.0)
SNR_arr[iord, iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj, dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([], dtype=int)
uni_frac_trim = np.array([], dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info(
'Purging object #{:d}'.format(iobj) +
' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) +
' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:, keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:, iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) &
(SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(
iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T,
usepca=None,
npca=npca,
npoly_cen=npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:, :, iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image,
xinit_fweight,
ncoeff,
inmask=inmask_now,
show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image,
xinit_gweight,
ncoeff,
inmask=inmask_now,
gweight=True,
show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:, iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky * (ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer,
ch,
pca_fits[:, iord, iobj],
str(uni_frac[iobj]),
color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer,
ch,
spec.trace_spat,
spec.idx,
color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
def ech_objfind(image, ivar, ordermask, slit_left, slit_righ,inmask=None,plate_scale=0.2,npca=2,ncoeff = 5,min_snr=0.0,nabove_min_snr=0,
pca_percentile=20.0,snr_pca=3.0,box_radius=2.0,show_peaks=False,show_fits=False,show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale,(float, int)):
plate_scale_ord = np.full(norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale,(np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error('Invalid size for plate_scale. It must either have one element or norders elements')
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec/2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec, slit_left[:,iord]), np.interp(slit_spec_pos, spec_vec, slit_righ[:,iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks=True
show_fits=True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {'setup': 'HIRES', 'slitid': iord + 1, 'scidx': 0,'det': 1, 'objtype': 'science'}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP/(np.median(slit_width)*np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos/1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0*fracpos
if nfound>1:
(ingroup, multgroup, firstgroup, nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac/1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound==1:
group = np.zeros(1,dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx == iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape, slit_spat_pos[iord,:], slit_spec_pos, det = sobjs_align[0].det,
setup = sobjs_align[0].setup, slitid = (iord + 1),
scidx = sobjs_align[0].scidx, objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,iord] + slit_width[:,iord]*uni_frac[iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group == uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius/plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg*inmask_iord, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar*0 + 1.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar*inmask_iord == 0.0, spec.trace_spat,box_rad_pix, ycen = spec.trace_spec) != pixtot)
flux_box[:,iord,iobj] = flux_tmp*mask_tmp
ivar_box[:,iord,iobj] = np.fmax(ivar_tmp*mask_tmp,0.0)
mask_box[:,iord,iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(flux_box[mask_tmp,iord,iobj]*np.sqrt(ivar_box[mask_tmp,iord,iobj]),
sigma_lower=5.0,sigma_upper=5.0)
SNR_arr[iord,iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj,dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([],dtype=int)
uni_frac_trim = np.array([],dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:,iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info('Purging object #{:d}'.format(iobj) + ' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) + ' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:,keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:,iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) & (SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:,:,iobj] = pca_trace((sobjs_final[igroup].trace_spat).T, usepca = None, npca = npca, npoly_cen = npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:,:,iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image, xinit_fweight, ncoeff, inmask = inmask_now, show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image, xinit_gweight, ncoeff, inmask = inmask_now, gweight=True,show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:,iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky*(ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer, ch, pca_fits[:,iord, iobj], str(uni_frac[iobj]), color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
def find_objects_pypeline(self, image, ivar, std=False, std_trace = None, maskslits=None,
manual_extract_dict=None,
show_peaks=False, show_fits=False, show_trace=False, show=False, debug=False):
"""
Find objects in the slits. This is currently setup only for ARMS
Wrapper to extract.objfind
Parameters
----------
tslits_dict: dict
Dictionary containing information on the slits traced for this image
Optional Parameters
-------------------
SHOW_PEAKS: bool
Generate QA showing peaks identified by object finding
SHOW_FITS: bool
Generate QA showing fits to traces
SHOW_TRACE: bool
Generate QA showing traces identified. Requires an open ginga RC modules window
Returns
-------
specobjs : Specobjs object
Container holding Specobj objects
nobj:
Number of objects identified
self.skymask : ndarray
Boolean image indicating which pixels are useful for global sky subtraction
"""
self.maskslits = self.maskslits if maskslits is None else maskslits
gdslits = np.where(np.invert(self.maskslits))[0]
# create the ouptut image for skymask
skymask = np.zeros_like(image, dtype=bool)
# Instantiate the specobjs container
sobjs = specobjs.SpecObjs()
# Loop on slits
for slit in gdslits:
qa_title ="Finding objects on slit # {:d}".format(slit)
msgs.info(qa_title)
thismask = (self.slitmask == slit)
inmask = (self.mask == 0) & thismask
# Find objects
specobj_dict = {'setup': self.setup, 'slitid': slit, 'orderindx': 999,
'det': self.det, 'objtype': self.objtype, 'pypeline': self.pypeline}
# TODO we need to add QA paths and QA hooks. QA should be
# done through objfind where all the relevant information
# is. This will be a png file(s) per slit.
sig_thresh = 30.0 if std else self.redux_par['sig_thresh']
#
sobjs_slit, skymask[thismask] = \
extract.objfind(image, thismask, self.tslits_dict['slit_left'][:,slit],self.tslits_dict['slit_righ'][:,slit],
inmask=inmask, ncoeff=self.redux_par['trace_npoly'],
std_trace=std_trace, sig_thresh=sig_thresh, hand_extract_dict=manual_extract_dict, #self.redux_par['manual'],
specobj_dict=specobj_dict, show_peaks=show_peaks,show_fits=show_fits, show_trace=show_trace,
qa_title=qa_title, nperslit=self.redux_par['maxnumber'])
sobjs.add_sobj(sobjs_slit)
# Steps
self.steps.append(inspect.stack()[0][3])
if show:
self.show('image', image=image*(self.mask == 0), chname = 'objfind',
sobjs=sobjs, slits=True)
# Return
return sobjs, len(sobjs), skymask
def compute_offsets(self):
objid_bri, slitidx_bri, spatid_bri, snr_bar_bri = self.get_brightest_obj(self.stack_dict['specobjs_list'],
self.spat_ids)
msgs.info('Determining offsets using brightest object on slit: {:d} with avg SNR={:5.2f}'.format(spatid_bri,np.mean(snr_bar_bri)))
thismask_stack = self.stack_dict['slitmask_stack'] == spatid_bri
trace_stack_bri = np.zeros((self.nspec, self.nexp))
# TODO Need to think abbout whether we have multiple tslits_dict for each exposure or a single one
for iexp in range(self.nexp):
trace_stack_bri[:,iexp] = self.stack_dict['slits_list'][iexp].center[:,slitidx_bri]
# trace_stack_bri[:,iexp] = (self.stack_dict['tslits_dict_list'][iexp]['slit_left'][:,slitid_bri] +
# self.stack_dict['tslits_dict_list'][iexp]['slit_righ'][:,slitid_bri])/2.0
# Determine the wavelength grid that we will use for the current slit/order
wave_bins = coadd.get_wave_bins(thismask_stack, self.stack_dict['waveimg_stack'], self.wave_grid)
dspat_bins, dspat_stack = coadd.get_spat_bins(thismask_stack, trace_stack_bri)
sci_list = [self.stack_dict['sciimg_stack'] - self.stack_dict['skymodel_stack']]
var_list = []
msgs.info('Rebinning Images')
sci_list_rebin, var_list_rebin, norm_rebin_stack, nsmp_rebin_stack = coadd.rebin2d(
wave_bins, dspat_bins, self.stack_dict['waveimg_stack'], dspat_stack, thismask_stack,
(self.stack_dict['mask_stack'] == 0), sci_list, var_list)
thismask = np.ones_like(sci_list_rebin[0][0,:,:],dtype=bool)
nspec_pseudo, nspat_pseudo = thismask.shape
slit_left = np.full(nspec_pseudo, 0.0)
slit_righ = np.full(nspec_pseudo, nspat_pseudo)
inmask = norm_rebin_stack > 0
traces_rect = np.zeros((nspec_pseudo, self.nexp))
sobjs = specobjs.SpecObjs()
#specobj_dict = {'setup': 'unknown', 'slitid': 999, 'orderindx': 999, 'det': self.det, 'objtype': 'unknown',
# 'pypeline': 'MultiSLit' + '_coadd_2d'}
for iexp in range(self.nexp):
sobjs_exp, _ = extract.objfind(sci_list_rebin[0][iexp,:,:], thismask, slit_left, slit_righ,
inmask=inmask[iexp,:,:], ir_redux=self.ir_redux,
fwhm=self.par['reduce']['findobj']['find_fwhm'],
trim_edg=self.par['reduce']['findobj']['find_trim_edge'],
npoly_cont=self.par['reduce']['findobj']['find_npoly_cont'],
maxdev=self.par['reduce']['findobj']['find_maxdev'],
ncoeff=3, sig_thresh=self.par['reduce']['findobj']['sig_thresh'], nperslit=1,
find_min_max=self.par['reduce']['findobj']['find_min_max'],
show_trace=self.debug_offsets, show_peaks=self.debug_offsets)
sobjs.add_sobj(sobjs_exp)
traces_rect[:, iexp] = sobjs_exp.TRACE_SPAT
# Now deterimine the offsets. Arbitrarily set the zeroth trace to the reference
med_traces_rect = np.median(traces_rect,axis=0)
offsets = med_traces_rect[0] - med_traces_rect
# Print out a report on the offsets
msg_string = msgs.newline() + '---------------------------------------------'
msg_string += msgs.newline() + ' Summary of offsets for highest S/N object '
msg_string += msgs.newline() + ' found on slitid = {:d} '.format(spatid_bri)
msg_string += msgs.newline() + '---------------------------------------------'
msg_string += msgs.newline() + ' exp# offset '
for iexp, off in enumerate(offsets):
msg_string += msgs.newline() + ' {:d} {:5.2f}'.format(iexp, off)
msg_string += msgs.newline() + '-----------------------------------------------'
msgs.info(msg_string)
if self.debug_offsets:
for iexp in range(self.nexp):
plt.plot(traces_rect[:, iexp], linestyle='--', label='original trace')
plt.plot(traces_rect[:, iexp] + offsets[iexp], label='shifted traces')
plt.legend()
plt.show()
return objid_bri, spatid_bri, snr_bar_bri, offsets
thismask,
lcen[:,islit-1],
rcen[:,islit-1],
inmask=((edgmask == False) & (mask_AB == True)),
bsp=bsp,
pos_mask=False,
show_fit=False)
image = diff_AB - residual_img
# Extract negative trace
specobj_slit_neg, skymask_neg, objmask_neg = extract.objfind(-image,
#ivar_AB,
thismask,
lcen[:,islit-1],
rcen[:,islit-1],
sig_thresh=3.0,
inmask=mask_AB,
FWHM=FWHM,
nperslit=1,
trim_edg=(3, 3),
show_trace=False,
show_peaks=False,
show_fits =False)
if specobj_slit_neg is not None:
specobjs_neg.add_sobj(specobj_slit_neg.specobjs.tolist())
# Extract positive trace
specobj_slit_pos, skymask_pos, objmask_pos = extract.objfind(image,
#ivar_AB,
thismask,
lcen[:,islit-1],
rcen[:,islit-1],
sig_thresh=3.0,
def build_traces(self, show_peaks=False, show_trace=False, debug=False):
"""
Main routine to generate the align profile traces in all slits
Args:
show_peaks (bool, optional):
Generate QA showing peaks identified by object finding
show_trace (bool, optional):
Generate QA showing traces identified. Requires an open ginga RC modules window
debug (bool, optional):
Returns:
dict: self.align_dict
"""
align_prof = dict({})
nslits = self.tslits_dict['slit_left'].shape[1]
# Prepare the plotting canvas
if show_trace:
self.show('image',
image=self.msalign.image,
chname='align_traces',
slits=True)
# Go through the slits
for sl in range(nslits):
specobj_dict = {
'setup': "unknown",
'slitid': sl,
'det': self.det,
'objtype': "align_profile",
'pypeline': self.spectrograph.pypeline
}
msgs.info("Fitting alignment traces in slit {0:d}".format(sl))
align_traces, _ = extract.objfind(
self.msalign.image,
self.slitmask == sl,
self.tslits_dict['slit_left'][:, sl],
self.tslits_dict['slit_righ'][:, sl],
ir_redux=False,
ncoeff=self.par['trace_npoly'],
specobj_dict=specobj_dict,
sig_thresh=self.par['sig_thresh'],
show_peaks=show_peaks,
show_fits=False,
trim_edg=self.par['trim_edge'],
cont_fit=False,
npoly_cont=0,
nperslit=len(self.par['locations']))
if len(align_traces) != len(self.par['locations']):
# Align tracing has failed for this slit
msgs.warn(
"Alignment tracing has failed on slit {0:d}".format(sl))
if show_trace:
self.show('overplot',
chname='align_traces',
align_traces=align_traces,
slits=False)
align_prof['{0:d}'.format(sl)] = align_traces.copy()
align_dict = self.generate_dict(align_prof)
# Steps
self.steps.append(inspect.stack()[0][3])
# Return
return align_dict