def add_object_sobj(self):
"""Add an object to specobjs
"""
if self._trcmthd == 'manual' and len(
self._mantrace['spat_a']) <= self._mantrace['polyorder']:
self.update_infobox(
message=
"You need to select more trace points before manually adding\n"
+ "a manual object trace. To do this, use the 'm' key",
yesno=False)
return
# Add an object trace
spec_vec = self._mantrace['spec_trc']
if self._trcmthd == 'manual':
trace_model = self._mantrace['spat_trc'].copy()
# Now empty the manual tracing
self.empty_mantrace()
else:
trace_model = self._trcdict["trace_model"][
self._trcmthd]["trace_model"].copy()
spat_0 = np.interp(self.mmy, spec_vec, trace_model)
shift = self.mmx - spat_0
trace_model += shift
xsize = self._trcdict["slit_righ"] - self._trcdict["slit_left"]
nsamp = np.ceil(xsize.max())
# Extract the SpecObj parameters
par = self._trcdict['sobj_par']
# Create a SpecObj
thisobj = specobjs.SpecObj(par['frameshape'],
par['slit_spat_pos'],
par['slit_spec_pos'],
det=par['det'],
setup=par['setup'],
slitid=par['slitid'],
orderindx=par['orderindx'],
objtype=par['objtype'])
thisobj.hand_extract_spat = self.mmx
thisobj.hand_extract_spec = self.mmy
thisobj.hand_extract_det = par['det']
thisobj.hand_extract_fwhm = None
thisobj.hand_extract_flag = True
f_ximg = RectBivariateSpline(spec_vec, np.arange(self.nspat),
par["ximg"])
thisobj.spat_fracpos = f_ximg(thisobj.hand_extract_spec,
thisobj.hand_extract_spat,
grid=False) # interpolate from ximg
thisobj.smash_peakflux = np.interp(
thisobj.spat_fracpos * nsamp, np.arange(nsamp),
self._trcdict['profile']) # interpolate from fluxconv
# assign the trace
thisobj.trace_spat = trace_model
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[self.nspec // 2]
thisobj.set_idx()
if self._object_traces.nobj != 0:
thisobj.fwhm = self._object_traces._fwhm[0]
else: # Otherwise just use the fwhm parameter input to the code (or the default value)
thisobj.fwhm = 2
# Finally, add new object
self.specobjs.add_sobj(thisobj)
def mk_specobj(flux=5, objid=500):
# specobj
npix = 100
specobj = specobjs.SpecObj((100,100), 0, (0.4,0.6), objtype='science',
spat_pixpos=300)
specobj.boxcar = dict(wave=np.arange(npix)*units.AA, counts=np.ones(npix)*flux)
specobj.optimal = dict(wave=np.arange(npix)*units.AA, counts=np.ones(npix)*flux-0.5)
specobj.objid = objid
specobj.trace_spat = np.arange(npix) / npix
specobj.fwhmfit = np.arange(npix) / npix
# Return
return specobj
"""
import os
import numpy as np
import pytest
from pypeit import msgs
from pypeit import specobjs
msgs.reset(verbosity=2)
#def data_path(filename):
# data_dir = os.path.join(os.path.dirname(__file__), 'files')
# return os.path.join(data_dir, filename)
shape = (1024,2048)
sobj1 = specobjs.SpecObj(shape, 1240., (800., 810.), spat_pixpos=900)
sobj2 = specobjs.SpecObj(shape, 1240., (400., 430.), spat_pixpos=900)
sobj3 = specobjs.SpecObj(shape, 1240., (200., 220.), spat_pixpos=900)
def test_init():
""" Run the parameter setup script
"""
# Null
sobjs1 = specobjs.SpecObjs()
# With a few objs
sobjs2 = specobjs.SpecObjs([sobj1,sobj2])
def test_access():
def ech_objfind(image,
ivar,
ordermask,
slit_left,
slit_righ,
inmask=None,
plate_scale=0.2,
npca=2,
ncoeff=5,
min_snr=0.0,
nabove_min_snr=0,
pca_percentile=20.0,
snr_pca=3.0,
box_radius=2.0,
show_peaks=False,
show_fits=False,
show_trace=False):
if inmask is None:
inmask = (ordermask > 0)
frameshape = image.shape
nspec = frameshape[0]
norders = slit_left.shape[1]
if isinstance(plate_scale, (float, int)):
plate_scale_ord = np.full(
norders, plate_scale) # 0.12 binned by 3 spatially for HIRES
elif isinstance(plate_scale, (np.ndarray, list, tuple)):
if len(plate_scale) == norders:
plate_scale_ord = plate_scale
elif len(plate_scale) == 1:
plate_scale_ord = np.full(norders, plate_scale[0])
else:
msgs.error(
'Invalid size for plate_scale. It must either have one element or norders elements'
)
else:
msgs.error('Invalid type for plate scale')
specmid = nspec // 2
slit_width = slit_righ - slit_left
spec_vec = np.arange(nspec)
slit_spec_pos = nspec / 2.0
slit_spat_pos = np.zeros((norders, 2))
for iord in range(norders):
slit_spat_pos[iord, :] = (np.interp(slit_spec_pos, spec_vec,
slit_left[:, iord]),
np.interp(slit_spec_pos, spec_vec,
slit_righ[:, iord]))
# Loop over orders and find objects
sobjs = specobjs.SpecObjs()
show_peaks = True
show_fits = True
# ToDo replace orderindx with the true order number here? Maybe not. Clean up slitid and orderindx!
for iord in range(norders):
msgs.info('Finding objects on slit # {:d}'.format(iord + 1))
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
specobj_dict = {
'setup': 'HIRES',
'slitid': iord + 1,
'scidx': 0,
'det': 1,
'objtype': 'science'
}
sobjs_slit, skymask[thismask], objmask[thismask], proc_list = \
extract.objfind(image, thismask, slit_left[:,iord], slit_righ[:,iord], inmask=inmask_iord,show_peaks=show_peaks,
show_fits=show_fits, show_trace=False, specobj_dict = specobj_dict)#, sig_thresh = 3.0)
# ToDO make the specobjs _set_item_ work with expressions like this spec[:].orderindx = iord
for spec in sobjs_slit:
spec.ech_orderindx = iord
sobjs.add_sobj(sobjs_slit)
nfound = len(sobjs)
# Compute the FOF linking length based on the instrument place scale and matching length FOFSEP = 1.0"
FOFSEP = 1.0 # separation of FOF algorithm in arcseconds
FOF_frac = FOFSEP / (np.median(slit_width) * np.median(plate_scale_ord))
# Feige: made the code also works for only one object found in one order
# Run the FOF. We use fake coordinaes
fracpos = sobjs.spat_fracpos
ra_fake = fracpos / 1000.0 # Divide all angles by 1000 to make geometry euclidian
dec_fake = 0.0 * fracpos
if nfound > 1:
(ingroup, multgroup, firstgroup,
nextgroup) = spheregroup(ra_fake, dec_fake, FOF_frac / 1000.0)
group = ingroup.copy()
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(uni_group)
msgs.info('FOF matching found {:d}'.format(nobj) + ' unique objects')
elif nfound == 1:
group = np.zeros(1, dtype='int')
uni_group, uni_ind = np.unique(group, return_index=True)
nobj = len(group)
msgs.warn('Only find one object no FOF matching is needed')
gfrac = np.zeros(nfound)
for jj in range(nobj):
this_group = group == uni_group[jj]
gfrac[this_group] = np.median(fracpos[this_group])
uni_frac = gfrac[uni_ind]
sobjs_align = sobjs.copy()
# Now fill in the missing objects and their traces
for iobj in range(nobj):
for iord in range(norders):
# Is there an object on this order that grouped into the current group in question?
on_slit = (group == uni_group[iobj]) & (sobjs_align.ech_orderindx
== iord)
if not np.any(on_slit):
# Add this to the sobjs_align, and assign required tags
thisobj = specobjs.SpecObj(frameshape,
slit_spat_pos[iord, :],
slit_spec_pos,
det=sobjs_align[0].det,
setup=sobjs_align[0].setup,
slitid=(iord + 1),
scidx=sobjs_align[0].scidx,
objtype=sobjs_align[0].objtype)
thisobj.ech_orderindx = iord
thisobj.spat_fracpos = uni_frac[iobj]
thisobj.trace_spat = slit_left[:,
iord] + slit_width[:, iord] * uni_frac[
iobj] # new trace
thisobj.trace_spec = spec_vec
thisobj.spat_pixpos = thisobj.trace_spat[specmid]
thisobj.set_idx()
# Use the real detections of this objects for the FWHM
this_group = group == uni_group[iobj]
# Assign to the fwhm of the nearest detected order
imin = np.argmin(
np.abs(sobjs_align[this_group].ech_orderindx - iord))
thisobj.fwhm = sobjs_align[imin].fwhm
thisobj.maskwidth = sobjs_align[imin].maskwidth
thisobj.ech_fracpos = uni_frac[iobj]
thisobj.ech_group = uni_group[iobj]
thisobj.ech_usepca = True
sobjs_align.add_sobj(thisobj)
group = np.append(group, uni_group[iobj])
gfrac = np.append(gfrac, uni_frac[iobj])
else:
# ToDo fix specobjs to get rid of these crappy loops!
for spec in sobjs_align[on_slit]:
spec.ech_fracpos = uni_frac[iobj]
spec.ech_group = uni_group[iobj]
spec.ech_usepca = False
# Some code to ensure that the objects are sorted in the sobjs_align by fractional position on the order and by order
# respectively
sobjs_sort = specobjs.SpecObjs()
for iobj in range(nobj):
this_group = group == uni_group[iobj]
this_sobj = sobjs_align[this_group]
sobjs_sort.add_sobj(this_sobj[np.argsort(this_sobj.ech_orderindx)])
# Loop over the objects and perform a quick and dirty extraction to assess S/N.
varimg = utils.calc_ivar(ivar)
flux_box = np.zeros((nspec, norders, nobj))
ivar_box = np.zeros((nspec, norders, nobj))
mask_box = np.zeros((nspec, norders, nobj))
SNR_arr = np.zeros((norders, nobj))
for iobj in range(nobj):
for iord in range(norders):
indx = (sobjs_sort.ech_group
== uni_group[iobj]) & (sobjs_sort.ech_orderindx == iord)
spec = sobjs_sort[indx]
thismask = ordermask == (iord + 1)
inmask_iord = inmask & thismask
box_rad_pix = box_radius / plate_scale_ord[iord]
flux_tmp = extract.extract_boxcar(image * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
var_tmp = extract.extract_boxcar(varimg * inmask_iord,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
ivar_tmp = utils.calc_ivar(var_tmp)
pixtot = extract.extract_boxcar(ivar * 0 + 1.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec)
mask_tmp = (extract.extract_boxcar(ivar * inmask_iord == 0.0,
spec.trace_spat,
box_rad_pix,
ycen=spec.trace_spec) != pixtot)
flux_box[:, iord, iobj] = flux_tmp * mask_tmp
ivar_box[:, iord, iobj] = np.fmax(ivar_tmp * mask_tmp, 0.0)
mask_box[:, iord, iobj] = mask_tmp
(mean, med_sn, stddev) = sigma_clipped_stats(
flux_box[mask_tmp, iord, iobj] *
np.sqrt(ivar_box[mask_tmp, iord, iobj]),
sigma_lower=5.0,
sigma_upper=5.0)
SNR_arr[iord, iobj] = med_sn
# Purge objects with low SNR and that don't show up in enough orders
keep_obj = np.zeros(nobj, dtype=bool)
sobjs_trim = specobjs.SpecObjs()
uni_group_trim = np.array([], dtype=int)
uni_frac_trim = np.array([], dtype=float)
for iobj in range(nobj):
if (np.sum(SNR_arr[:, iobj] > min_snr) >= nabove_min_snr):
keep_obj[iobj] = True
ikeep = sobjs_sort.ech_group == uni_group[iobj]
sobjs_trim.add_sobj(sobjs_sort[ikeep])
uni_group_trim = np.append(uni_group_trim, uni_group[iobj])
uni_frac_trim = np.append(uni_frac_trim, uni_frac[iobj])
else:
msgs.info(
'Purging object #{:d}'.format(iobj) +
' which does not satisfy min_snr > {:5.2f}'.format(min_snr) +
' on at least nabove_min_snr >= {:d}'.format(nabove_min_snr) +
' orders')
nobj_trim = np.sum(keep_obj)
if nobj_trim == 0:
return specobjs.SpecObjs()
SNR_arr_trim = SNR_arr[:, keep_obj]
# Do a final loop over objects and make the final decision about which orders will be interpolated/extrapolated by the PCA
for iobj in range(nobj_trim):
SNR_now = SNR_arr_trim[:, iobj]
indx = (sobjs_trim.ech_group == uni_group_trim[iobj])
# PCA interp/extrap if:
# (SNR is below pca_percentile of the total SNRs) AND (SNR < snr_pca)
# OR
# (if this order was not originally traced by the object finding, see above)
usepca = ((SNR_now < np.percentile(SNR_now, pca_percentile)) &
(SNR_now < snr_pca)) | sobjs_trim[indx].ech_usepca
# ToDo fix specobjs to get rid of these crappy loops!
for iord, spec in enumerate(sobjs_trim[indx]):
spec.ech_usepca = usepca[iord]
if usepca[iord]:
msgs.info('Using PCA to predict trace for object #{:d}'.format(
iobj) + ' on order #{:d}'.format(iord))
sobjs_final = sobjs_trim.copy()
# Loop over the objects one by one and adjust/predict the traces
npoly_cen = 3
pca_fits = np.zeros((nspec, norders, nobj_trim))
for iobj in range(nobj_trim):
igroup = sobjs_final.ech_group == uni_group_trim[iobj]
# PCA predict the masked orders which were not traced
pca_fits[:, :, iobj] = pca_trace((sobjs_final[igroup].trace_spat).T,
usepca=None,
npca=npca,
npoly_cen=npoly_cen)
# usepca = sobjs_final[igroup].ech_usepca,
# Perform iterative flux weighted centroiding using new PCA predictions
xinit_fweight = pca_fits[:, :, iobj].copy()
inmask_now = inmask & (ordermask > 0)
xfit_fweight = extract.iter_tracefit(image,
xinit_fweight,
ncoeff,
inmask=inmask_now,
show_fits=show_fits)
# Perform iterative Gaussian weighted centroiding
xinit_gweight = xfit_fweight.copy()
xfit_gweight = extract.iter_tracefit(image,
xinit_gweight,
ncoeff,
inmask=inmask_now,
gweight=True,
show_fits=show_fits)
# Assign the new traces
for iord, spec in enumerate(sobjs_final[igroup]):
spec.trace_spat = xfit_gweight[:, iord]
spec.spat_pixpos = spec.trace_spat[specmid]
# Set the IDs
sobjs_final.set_idx()
if show_trace:
viewer, ch = ginga.show_image(objminsky * (ordermask > 0))
for iobj in range(nobj_trim):
for iord in range(norders):
ginga.show_trace(viewer,
ch,
pca_fits[:, iord, iobj],
str(uni_frac[iobj]),
color='yellow')
for spec in sobjs_trim:
color = 'green' if spec.ech_usepca else 'magenta'
ginga.show_trace(viewer,
ch,
spec.trace_spat,
spec.idx,
color=color)
#for spec in sobjs_final:
# color = 'red' if spec.ech_usepca else 'green'
# ginga.show_trace(viewer, ch, spec.trace_spat, spec.idx, color=color)
return sobjs_final
def ech_load_specobj(fname, order=None):
""" Load a spec1d file into a list of SpecObjExp objects
Parameters
----------
fname : str
Returns
-------
specObjs : list of SpecObjExp
head0
"""
#if order is None:
# msgs.warn('You did not specify an order. Return specObjs with all orders.')
# specObjs, head0 = load.load_specobj(fname)
# return specObjs, head0
speckeys = [
'WAVE', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG', 'COUNTS_IVAR',
'COUNTS'
]
#
specObjs = []
hdulist = fits.open(fname)
head0 = hdulist[0].header
for hdu in hdulist:
if hdu.name == 'PRIMARY':
continue
#elif hdu.name[8:17] != 'ORDER'+'{0:04}'.format(order):
# continue
# Parse name
idx = hdu.name
objp = idx.split('-')
if objp[-1][0:3] == 'DET':
det = int(objp[-1][3:])
else:
det = int(objp[-1][1:])
if objp[-2][:5] == 'ORDER':
iord = int(objp[-2][5:])
else:
msgs.warn('Loading longslit data ?')
iord = int(-1)
# if order is not None and iord !=order then do not return this extenction
# if order is None return all extensions
# if order is not None and iord ==order then only return the specific order you want.
if (order is not None) and (iord != order):
continue
# Load data
spec = Table(hdu.data)
shape = (len(spec), 1024) # 2nd number is dummy
# New and wrong
try:
specobj = specobjs.SpecObj(shape, None, None, idx=idx)
except:
debugger.set_trace()
msgs.error("BUG ME")
# Add order number
specobj.ech_orderindx = iord
# ToDo: need to changed to the real order number?
specobj.ech_order = iord
# Add trace
try:
specobj.trace_spat = spec['TRACE']
except:
# KLUDGE!
specobj.trace_spat = np.arange(len(spec['BOX_WAVE']))
# Add spectrum
if 'BOX_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA
if 'OPT_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
# Append
specObjs.append(specobj)
# Return
return specObjs, head0
def load_specobjs(fname, order=None):
""" Load a spec1d file into a list of SpecObjExp objects
Parameters
----------
fname : str
Returns
-------
specObjs : list of SpecObjExp
head0
"""
sobjs = specobjs.SpecObjs()
speckeys = [
'WAVE', 'WAVE_GRID_MASK', 'WAVE_GRID', 'WAVE_GRID_MIN',
'WAVE_GRID_MAX', 'SKY', 'MASK', 'FLAM', 'FLAM_IVAR', 'FLAM_SIG',
'COUNTS_IVAR', 'COUNTS', 'COUNTS_SIG'
]
# sobjs_keys gives correspondence between header cards and sobjs attribute name
sobjs_key = specobjs.SpecObj.sobjs_key()
hdulist = fits.open(fname)
head0 = hdulist[0].header
#pypeline = head0['PYPELINE']
# Is this an Echelle reduction?
#if 'Echelle' in pypeline:
# echelle = True
#else:
# echelle = False
for hdu in hdulist:
if hdu.name == 'PRIMARY':
continue
# Parse name
idx = hdu.name
objp = idx.split('-')
if objp[-2][:5] == 'ORDER':
iord = int(objp[-2][5:])
else:
msgs.warn('Loading longslit data ?')
iord = int(-1)
if (order is not None) and (iord != order):
continue
specobj = specobjs.SpecObj(None, None, None, idx=idx)
# Assign specobj attributes from header cards
for attr, hdrcard in sobjs_key.items():
try:
value = hdu.header[hdrcard]
except:
continue
setattr(specobj, attr, value)
# Load data
spec = Table(hdu.data)
shape = (len(spec), 1024) # 2nd number is dummy
specobj.shape = shape
specobj.trace_spat = spec['TRACE']
# Add spectrum
if 'BOX_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.boxcar[skey] = spec['BOX_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.boxcar['WAVE'] = specobj.boxcar['WAVE'] * units.AA
if 'OPT_COUNTS' in spec.keys():
for skey in speckeys:
try:
specobj.optimal[skey] = spec['OPT_{:s}'.format(skey)].data
except KeyError:
pass
# Add units on wave
specobj.optimal['WAVE'] = specobj.optimal['WAVE'] * units.AA
# Append
sobjs.add_sobj(specobj)
# Return
return sobjs, head0