def maskoutliers(spec, nsig=5, verbose=False):
"""
Mask large positive outliers and negative flux pixels in the spectrum.
Parameters
----------
spec : Spec1D object
Observed spectrum to mask outliers on.
nsig : int, optional
Number of standard deviations to use for the outlier rejection.
Default is 5.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with outliers masked.
Example
-------
.. code-block:: python
spec = maskoutliers(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0))
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True:
print('Masked ' + str(totnbd) + ' outlier or negative pixels')
return spec2
def maskdiscrepant(spec, model, nsig=10, verbose=False):
"""
Mask pixels that are discrepant when compared to a model.
Parameters
----------
spec : Spec1D object
Observed spectrum for which to mask discrepant values.
model : Spec1D object
Reference/model spectrum to use to find discrepant values.
nsig : int, optional
Number of standard deviations to use for the discrepant values.
Default is 10.0.
verbose : boolean, optional
Verbose output. Default is False.
Returns
-------
spec2 : Spec1D object
Spectrum with discrepant values masked.
Example
-------
.. code-block:: python
spec = maskdiscrepant(spec,nsig=5)
"""
print = getprintfunc(
) # Get print function to be used locally, allows for easy logging
spec2 = spec.copy()
wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D
flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D
mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D
mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D
totnbd = 0
for o in range(spec2.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
my = mflux[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
if medy <= 0.0:
medy = 1.0
y /= medy
my /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - my)
bd, nbd = dln.where(np.abs(y - my) > nsig * sig)
totnbd += nbd
if nbd > 0:
flux[bd, o] = dln.poly(x[bd], coef) * medy
err[bd, o] = 1e30
mask[bd, o] = True
# Flatten to 1D if norder=1
if spec2.norder == 1:
flux = flux.flatten()
err = err.flatten()
mask = mask.flatten()
# Stuff back in
spec2.flux = flux
spec2.err = err
spec2.mask = mask
if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels')
return spec2
def meascutout(meas, obj, size=10, outdir='./'):
""" Input the measurements and create cutouts. """
expstr = fits.getdata(
'/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1)
#expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1)
decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt',
format='ascii')
objid = obj['id'][0]
# Sort by MJD
si = np.argsort(meas['mjd'])
meas = meas[si]
ind1, ind2 = dln.match(expstr['base'], meas['exposure'])
nind = len(ind1)
if nind == 0:
print('No matches')
return
# Sort by input meas catalog
si = np.argsort(ind2)
ind1 = ind1[si]
ind2 = ind2[si]
# Create the reference WCS
wref = WCS(naxis=2)
pixscale = 0.26 # DECam, "/pix
npix = round(size / pixscale)
if npix % 2 == 0: # must be odd
npix += 1
hpix = npix // 2 # center of image
wref.wcs.ctype = ['RA---TAN', 'DEC--TAN']
wref.wcs.crval = [obj['ra'][0], obj['dec'][0]]
wref.wcs.crpix = [npix // 2, npix // 2]
wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]])
wref.array_shape = (npix, npix)
refheader = wref.to_header()
refheader['NAXIS'] = 2
refheader['NAXIS1'] = npix
refheader['NAXIS2'] = npix
# Load the data
instrument = expstr['instrument'][ind1]
fluxfile = expstr['file'][ind1]
fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk
maskfile = expstr['maskfile'][ind1]
maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk
ccdnum = meas['ccdnum'][ind2]
figfiles = []
for i in range(nind):
try:
if instrument[i] == 'c4d':
dind, = np.where(decam['CCDNUM'] == ccdnum[i])
extname = decam['NAME'][dind[0]]
im, head = getfitsext(fluxfile[i], extname, header=True)
mim, mhead = getfitsext(maskfile[i], extname, header=True)
#im,head = fits.getdata(fluxfile[i],header=True,extname=extname)
#mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname)
else:
im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True)
mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True)
except:
print('error')
import pdb
pdb.set_trace()
# Get chip-level information
exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz
chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" +
exposure + "' and ccdnum=" + str(ccdnum[i]),
fmt='table')
w = WCS(head)
# RA/DEC correction for the object
lon = obj['ra'][0] - chres['ra'][0]
lat = obj['dec'][0] - chres['dec'][0]
racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[
'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat
deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[
'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat
# apply these offsets to the header WCS CRVAL
w.wcs.crval += [racorr, deccorr]
head['CRVAL1'] += racorr
head['CRVAL2'] += deccorr
# Object X/Y position
xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0)
# Get the cutout
xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes
ycen = meas['y'][ind2[i]] - 1
smim = dln.gsmooth(im, 2)
# use the object coords for centering
#cutim,xr,yr = cutout(smim,xobj,yobj,size)
# Mask the bad pixels
badmask = (mim > 0)
im[badmask] = np.nanmedian(im[~badmask])
# Create a common TAN WCS that each image gets interpoled onto!!!
#hdu1 = fits.open(fluxfile[i],extname=extname)
smim1 = dln.gsmooth(im, 1.5)
hdu = fits.PrimaryHDU(smim1, head)
cutim, footprint = reproject_interp(hdu, refheader,
order='bicubic') # biquadratic
cutim[footprint == 0] = np.nanmedian(
im[~badmask]) # set out-of-bounds to background
#xr = [0,npix-1]
#yr = [0,npix-1]
xr = [-hpix * pixscale, hpix * pixscale]
yr = [-hpix * pixscale, hpix * pixscale]
# exposure_ccdnum, filter, MJD, delta_MJD, mag
print(
str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' +
str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' +
str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]]))
#figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/'
figfile = outdir
figfile += '%s_%04d_%s_%02d.jpg' % (str(
obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i])
figfiles.append(figfile)
matplotlib.use('Agg')
plt.rcParams.update({'font.size': 11})
if os.path.exists(figfile): os.remove(figfile)
fig = plt.gcf() # get current graphics window
fig.clf() # clear
figsize = 8.0 #6.0
ax = fig.subplots() # projection=wcs
#fig.set_figheight(figsize*0.8)
fig.set_figheight(figsize)
fig.set_figwidth(figsize)
med = np.nanmedian(smim)
sig = dln.mad(smim)
bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med)
lmed = np.nanmedian(bigim)
# Get the flux of the object and scale each image to the same height
#meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm
#cmag = mag_auto + 2.5*alog10(exptime) + zpterm
instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10(
chres['exptime'][0]) - chres['zpterm'][0]
#mag = -2.5*log(flux)+25.0
instflux = 10**((25.0 - instmag) / 2.5)
print('flux = ' + str(instflux))
# Get height of object
# flux of 2D Gaussian is ~2*pi*height*sigma^2
pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600
fwhm = chres['fwhm'][0] / pixscale1
instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2)
print('height = ' + str(instheight))
# Scale the images to the flux level of the first image
cutim -= lmed
if i == 0:
instflux0 = instflux.copy()
instheight0 = instheight.copy()
else:
scale = instflux0 / instflux
#scale = instheight0/instheight
cutim *= scale
print('scaling image by ' + str(scale))
#vmin = lmed-8*sig # 3*sig
#vmax = lmed+12*sig # 5*sig
if i == 0:
vmin = -8 * sig # 3*sig
#vmax = 12*sig # 5*sig
vmax = 0.5 * instheight # 0.5
vmin0 = vmin
vmax0 = vmax
else:
vmin = vmin0
vmax = vmax0
print('vmin = ' + str(vmin))
print('vmax = ' + str(vmax))
plt.imshow(cutim,
origin='lower',
aspect='auto',
interpolation='none',
extent=(xr[0], xr[1], yr[0], yr[1]),
vmin=vmin,
vmax=vmax,
cmap='viridis') # viridis, Greys, jet
#plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none',
# vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet
#plt.colorbar()
# show one vertical, one horizontal line pointing to the center but offset
# then a small dot on the meas position
# 13, 8
plt.plot(np.array([0, 0]),
np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
c='white',
alpha=0.7)
plt.plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
np.array([0, 0]),
c='white',
alpha=0.7)
# Meas X/Y position
xmeas, ymeas = wref.all_world2pix(meas['ra'][ind2[i]],
meas['dec'][ind2[i]], 0)
plt.scatter([(xmeas - hpix) * pixscale], [(ymeas - hpix) * pixscale],
c='r',
marker='+',
s=20)
#plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100)
#plt.scatter([xcen],[ycen],c='r',marker='+',s=100)
# Object X/Y position
#xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0)
xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0)
#plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3)
#plt.scatter(xobj,yobj,c='y',marker='+',s=100)
#leg = ax.legend(loc='upper left', frameon=False)
plt.xlabel(r'$\Delta$ RA (arcsec)')
plt.ylabel(r'$\Delta$ DEC (arcsec)')
#plt.xlabel('X')
#plt.ylabel('Y')
#plt.xlim(xr)
#plt.ylim(yr)
#ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center')
co = 'white' #'lightgray' # blue
ax.annotate('%s %02d %s %6.1f ' %
(meas['exposure'][ind2[i]], ccdnum[i],
meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]),
xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05),
ha='center',
color=co)
ax.annotate(
'%10.2f %10.2f ' %
(meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])),
xy=(xr[0] + dln.valrange(xr) * 0.05,
yr[1] - dln.valrange(yr) * 0.05),
ha='left',
color=co)
ax.annotate('%s = %5.2f +/- %4.2f' %
(meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]],
meas['magerr_auto'][ind2[i]]),
xy=(xr[1] - dln.valrange(xr) * 0.05,
yr[1] - dln.valrange(yr) * 0.05),
ha='right',
color=co)
plt.savefig(figfile)
print('Cutout written to ' + figfile)
#import pdb; pdb.set_trace()
# Make a single blank file at the end so you know it looped
figfile = outdir
figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'blank')
figfiles.append(figfile)
matplotlib.use('Agg')
if os.path.exists(figfile): os.remove(figfile)
fig = plt.gcf() # get current graphics window
fig.clf() # clear
figsize = 8.0 #6.0
fig.set_figheight(figsize)
fig.set_figwidth(figsize)
plt.savefig(figfile)
print(figfile)
# Make the animated gif
animfile = outdir + str(objid) + '_cutouts.gif'
print('Creating animated gif ' + animfile)
if os.path.exists(animfile): os.remove(animfile)
#ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True)
ret = subprocess.run('convert -delay 20 ' + ' '.join(figfiles) + ' ' +
animfile,
shell=True)
#import pdb; pdb.set_trace()
dln.remove(figfiles)
def meascutout(meas, obj, size=10, outdir='./', domask=True):
""" Input the measurements and create cutouts. """
expstr = fits.getdata(
'/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1)
#expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1)
decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt',
format='ascii')
objid = obj['id'][0]
# Sort by MJD
si = np.argsort(meas['mjd'])
meas = meas[si]
# Make cut on FWHM
# maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip
sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum"
sql += " where meas.objectid='" + objid + "'"
chip = qc.query(sql=sql, fmt='table')
ind3, ind4 = dln.match(chip['exposure'], meas['exposure'])
si = np.argsort(ind4) # sort by input meas catalog
ind3 = ind3[si]
ind4 = ind4[si]
chip = chip[ind3]
meas = meas[ind4]
gdfwhm, = np.where((meas['fwhm'] > 0.2 * chip['fwhm'])
& (meas['fwhm'] < 2.0 * chip['fwhm']))
if len(gdfwhm) == 0:
print('All measurements have bad FWHM values')
return
if len(gdfwhm) < len(meas):
print('Removing ' + str(len(meas) - len(gdfwhm)) +
' measurements with bad FWHM values')
meas = meas[gdfwhm]
ind1, ind2 = dln.match(expstr['base'], meas['exposure'])
nind = len(ind1)
if nind == 0:
print('No matches')
return
# Sort by input meas catalog
si = np.argsort(ind2)
ind1 = ind1[si]
ind2 = ind2[si]
# Create the reference WCS
wref = WCS(naxis=2)
pixscale = 0.26 # DECam, "/pix
npix = round(size / pixscale)
if npix % 2 == 0: # must be odd
npix += 1
hpix = npix // 2 # center of image
wref.wcs.ctype = ['RA---TAN', 'DEC--TAN']
wref.wcs.crval = [obj['ra'][0], obj['dec'][0]]
wref.wcs.crpix = [npix // 2, npix // 2]
wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]])
wref.array_shape = (npix, npix)
refheader = wref.to_header()
refheader['NAXIS'] = 2
refheader['NAXIS1'] = npix
refheader['NAXIS2'] = npix
# Load the data
instrument = expstr['instrument'][ind1]
plver = expstr['plver'][ind1]
fluxfile = expstr['file'][ind1]
fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk
maskfile = expstr['maskfile'][ind1]
maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk
ccdnum = meas['ccdnum'][ind2]
figfiles = []
xmeas = []
ymeas = []
cutimarr = np.zeros((npix, npix, nind), float)
for i in range(nind):
#for i in range(3):
instcode = instrument[i]
plver1 = plver[i]
try:
if instrument[i] == 'c4d':
dind, = np.where(decam['CCDNUM'] == ccdnum[i])
extname = decam['NAME'][dind[0]]
im, head = getfitsext(fluxfile[i], extname, header=True)
mim, mhead = getfitsext(maskfile[i], extname, header=True)
#im,head = fits.getdata(fluxfile[i],header=True,extname=extname)
#mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname)
else:
im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True)
mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True)
except:
print('error')
import pdb
pdb.set_trace()
# Turn the mask from integer to bitmask
if ((instcode == 'c4d') &
(plver1 >= 'V3.5.0')) | (instcode == 'k4m') | (instcode == 'ksb'):
omim = mim.copy()
mim *= 0
nonzero = (omim > 0)
mim[nonzero] = 2**((omim - 1)[nonzero]) # This takes about 1 sec
# Fix the DECam Pre-V3.5.0 masks
if (instcode == 'c4d') & (plver1 < 'V3.5.0'):
omim = mim.copy()
mim *= 0 # re-initialize
mim += (np.bitwise_and(omim, 1) == 1) * 1 # bad pixels
mim += (np.bitwise_and(omim, 2) == 2) * 4 # saturated
mim += (np.bitwise_and(omim, 4) == 4) * 32 # interpolated
mim += (np.bitwise_and(omim, 16) == 16) * 16 # cosmic ray
mim += (np.bitwise_and(omim, 64) == 64) * 8 # bleed trail
# Get chip-level information
exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz
chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" +
exposure + "' and ccdnum=" + str(ccdnum[i]),
fmt='table')
w = WCS(head)
# RA/DEC correction for the object
lon = obj['ra'][0] - chres['ra'][0]
lat = obj['dec'][0] - chres['dec'][0]
racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[
'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat
deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[
'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat
# apply these offsets to the header WCS CRVAL
#w.wcs.crval += [racorr,deccorr]
#head['CRVAL1'] += racorr
#head['CRVAL2'] += deccorr
print(racorr, deccorr)
# Object X/Y position
xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0)
# Get the cutout
xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes
ycen = meas['y'][ind2[i]] - 1
smim = dln.gsmooth(im, 2)
# use the object coords for centering
#cutim,xr,yr = cutout(smim,xobj,yobj,size)
# Mask the bad pixels
if domask == True:
badmask = (mim > 0)
im[badmask] = np.nanmedian(im[~badmask])
else:
badmask = (im < 0)
# Create a common TAN WCS that each image gets interpoled onto!!!
#hdu1 = fits.open(fluxfile[i],extname=extname)
smim1 = dln.gsmooth(im, 1.5)
hdu = fits.PrimaryHDU(smim1, head)
cutim, footprint = reproject_interp(hdu, refheader,
order='bicubic') # biquadratic
cutim[footprint == 0] = np.nanmedian(
im[~badmask]) # set out-of-bounds to background
#xr = [0,npix-1]
#yr = [0,npix-1]
xr = [-hpix * pixscale, hpix * pixscale]
yr = [-hpix * pixscale, hpix * pixscale]
# exposure_ccdnum, filter, MJD, delta_MJD, mag
print(
str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' +
str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' +
str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]]))
#figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/'
figfile = outdir
figfile += '%s_%04d_%s_%02d.jpg' % (str(
obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i])
figfiles.append(figfile)
matplotlib.use('Agg')
plt.rc('font', size=15)
plt.rc('axes', titlesize=20)
plt.rc('axes', labelsize=20)
plt.rc('xtick', labelsize=20)
plt.rc('ytick', labelsize=20)
#plt.rcParams.update({'font.size': 15})
#plt.rcParams.update({'axes.size': 20})
#plt.rcParams.update({'xtick.size': 20})
#plt.rcParams.update({'ytick.size': 20})
if os.path.exists(figfile): os.remove(figfile)
fig = plt.gcf() # get current graphics window
fig.clf() # clear
gskw = dict(width_ratios=[30, 1])
fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw)
figsize = 8.0 #6.0
figheight = 8.0
figwidth = 9.0
#ax = fig.subplots() # projection=wcs
#fig.set_figheight(figsize*0.8)
fig.set_figheight(figheight)
fig.set_figwidth(figwidth)
med = np.nanmedian(smim)
sig = dln.mad(smim)
bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med)
lmed = np.nanmedian(bigim)
# Get the flux of the object and scale each image to the same height
#meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm
#cmag = mag_auto + 2.5*alog10(exptime) + zpterm
instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10(
chres['exptime'][0]) - chres['zpterm'][0]
#mag = -2.5*log(flux)+25.0
instflux = 10**((25.0 - instmag) / 2.5)
print('flux = ' + str(instflux))
# Get height of object
# flux of 2D Gaussian is ~2*pi*height*sigma^2
pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600
fwhm = chres['fwhm'][0] / pixscale1
instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2)
print('height = ' + str(instheight))
# Scale the images to the flux level of the first image
cutim -= lmed
if i == 0:
instflux0 = instflux.copy()
instheight0 = instheight.copy()
else:
scale = instflux0 / instflux
#scale = instheight0/instheight
cutim *= scale
print('scaling image by ' + str(scale))
#vmin = lmed-8*sig # 3*sig
#vmax = lmed+12*sig # 5*sig
if i == 0:
vmin = -8 * sig # 3*sig
#vmax = 12*sig # 5*sig
vmax = 0.5 * instheight # 0.5
vmin0 = vmin
vmax0 = vmax
else:
vmin = vmin0
vmax = vmax0
print('vmin = ' + str(vmin))
print('vmax = ' + str(vmax))
cutimarr[:, :, i] = cutim.copy()
ax[0].imshow(cutim,
origin='lower',
aspect='auto',
interpolation='none',
extent=(xr[0], xr[1], yr[0], yr[1]),
vmin=vmin,
vmax=vmax,
cmap='viridis') # viridis, Greys, jet
#plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none',
# vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet
#plt.colorbar()
# show one vertical, one horizontal line pointing to the center but offset
# then a small dot on the meas position
# 13, 8
ax[0].plot(np.array([0, 0]),
np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
c='white',
alpha=0.7)
ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
np.array([0, 0]),
c='white',
alpha=0.7)
# Meas X/Y position
xmeas1, ymeas1 = wref.all_world2pix(meas['ra'][ind2[i]],
meas['dec'][ind2[i]], 0)
xmeas.append(xmeas1)
ymeas.append(ymeas1)
ax[0].scatter([(xmeas1 - hpix) * pixscale],
[(ymeas1 - hpix) * pixscale],
c='r',
marker='+',
s=20)
#plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100)
#plt.scatter([xcen],[ycen],c='r',marker='+',s=100)
# Object X/Y position
#xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0)
xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0)
#plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3)
#plt.scatter(xobj,yobj,c='y',marker='+',s=100)
#leg = ax.legend(loc='upper left', frameon=False)
ax[0].set_xlabel(r'$\Delta$ RA (arcsec)')
ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)')
ax[0].set_xlim((xr[1], xr[0])) # sky right
ax[0].set_ylim(yr)
#plt.xlabel('X')
#plt.ylabel('Y')
#plt.xlim(xr)
#plt.ylim(yr)
#ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center')
co = 'white' #'lightgray' # blue
ax[0].annotate('%s %02d %s %6.1f ' %
(meas['exposure'][ind2[i]], ccdnum[i],
meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]),
xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05),
ha='center',
color=co)
ax[0].annotate(
'%10.2f $\Delta$t=%7.2f ' %
(meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])),
xy=(xr[1] - dln.valrange(xr) * 0.05,
yr[1] - dln.valrange(yr) * 0.05),
ha='left',
color=co)
# xy=(xr[0]+dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='left',color=co)
ax[0].annotate('%s = %5.2f +/- %4.2f' %
(meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]],
meas['magerr_auto'][ind2[i]]),
xy=(xr[0] + dln.valrange(xr) * 0.05,
yr[1] - dln.valrange(yr) * 0.05),
ha='right',
color=co)
# xy=(xr[1]-dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='right',color=co)
# Progress bar
frameratio = (i + 1) / float(nind)
timeratio = (meas['mjd'][ind2[i]] -
np.min(meas['mjd'])) / dln.valrange(meas['mjd'])
#ratio = frameratio
ratio = timeratio
print('ratio = ' + str(100 * ratio))
barim = np.zeros((100, 100), int)
ind = dln.limit(int(round(ratio * 100)), 1, 99)
barim[:, 0:ind] = 1
ax[1].imshow(barim.T, origin='lower', aspect='auto', cmap='Greys')
ax[1].set_xlabel('%7.1f \n days' %
(meas['mjd'][ind2[i]] - np.min(meas['mjd'])))
#ax[1].set_xlabel('%d/%d' % (i+1,nind))
ax[1].set_title('%d/%d' % (i + 1, nind))
ax[1].axes.xaxis.set_ticks([])
#ax[1].axes.xaxis.set_visible(False)
ax[1].axes.yaxis.set_visible(False)
#ax[1].axis('off')
right_side = ax[1].spines['right']
right_side.set_visible(False)
left_side = ax[1].spines['left']
left_side.set_visible(False)
top_side = ax[1].spines['top']
top_side.set_visible(False)
plt.savefig(figfile)
print('Cutout written to ' + figfile)
#import pdb; pdb.set_trace()
avgim = np.sum(cutimarr, axis=2) / nind
avgim *= instheight0 / np.max(avgim)
medim = np.median(cutimarr, axis=2)
# Make a single blank file at the end so you know it looped
figfile = outdir
figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'path')
figfiles.append(figfile)
matplotlib.use('Agg')
if os.path.exists(figfile): os.remove(figfile)
fig = plt.gcf() # get current graphics window
fig.clf() # clear
gskw = dict(width_ratios=[30, 1])
fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw)
fig.set_figheight(figheight)
fig.set_figwidth(figwidth)
ax[0].imshow(avgim,
origin='lower',
aspect='auto',
interpolation='none',
extent=(xr[0], xr[1], yr[0], yr[1]),
vmin=vmin,
vmax=vmax,
cmap='viridis') # viridis, Greys, jet
ax[0].plot(np.array([0, 0]),
np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
c='white',
alpha=0.7,
zorder=1)
ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale,
np.array([0, 0]),
c='white',
alpha=0.7,
zorder=1)
xmeas = np.array(xmeas)
ymeas = np.array(ymeas)
ax[0].plot((xmeas - hpix) * pixscale, (ymeas - hpix) * pixscale, c='r')
#plt.scatter((xmeas-hpix)*pixscale,(ymeas-hpix)*pixscale,c='r',marker='+',s=30)
ax[0].set_xlabel(r'$\Delta$ RA (arcsec)')
ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)')
ax[0].set_xlim((xr[1], xr[0])) # sky -right
ax[0].set_ylim(yr)
ax[1].axis('off')
plt.savefig(figfile)
# Make four copies
for j in np.arange(2, 11):
#pathfile = figfile.replace('path1','path'+str(j))
pathfile = figfile.replace('%04d' % (i + 2), '%04d' % (i + 1 + j))
if os.path.exists(pathfile): os.remove(pathfile)
shutil.copyfile(figfile, pathfile)
figfiles.append(pathfile)
# Make the animated gif
animfile = outdir + str(objid) + '_cutouts.gif'
if os.path.exists(animfile): os.remove(animfile)
# put list of files in a separate file
listfile = outdir + str(objid) + '_cutouts.lst'
if os.path.exists(listfile): os.remove(listfile)
dln.writelines(listfile, figfiles)
delay = dln.scale(nind, [20, 1000], [20, 1])
delay = int(np.round(dln.limit(delay, 1, 20)))
print('delay = ' + str(delay))
print('Creating animated gif ' + animfile)
#ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True)
#ret = subprocess.run('convert -delay 20 '+' '.join(figfiles)+' '+animfile,shell=True)
ret = subprocess.run('convert @' + listfile + ' -delay ' + str(delay) +
' ' + animfile,
shell=True)
#import pdb; pdb.set_trace()
dln.remove(figfiles)
goodwav = np.zeros((npix,len(goodap1)),float)
for i in range(len(goodap1)):
wav1 = idlines[goodap1[i]].func(x)
diff = wav1-medwav0
meddiff = np.median(diff)
goodwav[:,i] = wav1-meddiff
medwav = np.median(goodwav,axis=1)
# Second cut at good and bad apertures by comparing to median wave array
sig = np.zeros(128,float)+999999.0
std = np.zeros(128,float)+999999.0
for i in range(128):
if idlines[i].npoints>0:
wav1 = idlines[i].func(x)
std[i] = np.std(wav1-medwav)
sig[i] = dln.mad(wav1-medwav)
#goodap, = np.where((npoints>0) & (sig<1.0) & (std<1.0) & (npoints>0.5*mednpoints))
#badap, = np.where((npoints>0) & ((sig>=1.0) | (std>=1.0) | (npoints<=0.5*mednpoints)))
goodap, = np.where((apinfo['npoints']>0) & (sig<1.0) & (std<1.0) & (apinfo['rms']<0.1) & (apinfo['npoints']>20))
badap, = np.where((apinfo['npoints']>0) & ((sig>=1.0) | (std>=1.0) | (apinfo['rms']>=0.1) | (apinfo['npoints']<=20)))
nbad = len(badap)
print(str(nbad)+' bad apertures to fix')
apinfo['good'][goodap] = True
#goodap, = np.where((npoints>0) & (sig<1.0) & (std<1.0) & (npoints>0.5*mednpoints) & (rms<0.08))
#badap, = np.where(rms > 0.08)
#nbad = len(badap)
#import pdb; pdb.set_trace()
def fix_pms(objectid):
""" Correct the proper motions in the healpix object catalog."""
t00 = time.time()
hostname = socket.gethostname()
host = hostname.split('.')[0]
version = 'v3'
radeg = np.float64(180.00) / np.pi
meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table')
nmeas = len(meas)
print(' '+str(nmeas))
mnra = np.median(meas['ra'].data)
mndec = np.median(meas['dec'].data)
lim = 20.0 # 50.0
gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) &
(np.abs(meas['dec'].data-mndec)*3600 < lim))
ngd = len(gd)
nbd = nmeas-ngd
print('bad measurements '+str(nbd))
#if nbd==0:
# return None
meas = meas[gd]
# Make cut on FWHM
# maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip
sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum"
sql += " where meas.objectid='"+objectid+"'"
chip = qc.query(sql=sql,fmt='table')
ind3,ind4 = dln.match(chip['exposure'],meas['exposure'])
si = np.argsort(ind4) # sort by input meas catalog
ind3 = ind3[si]
ind4 = ind4[si]
chip = chip[ind3]
meas = meas[ind4]
gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm']))
if len(gdfwhm)==0:
print('All measurements have bad FWHM values')
return
if len(gdfwhm) < len(meas):
print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values')
meas = meas[gdfwhm]
raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec
ra = np.array(meas['ra'],np.float64)
ra -= np.mean(ra)
ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec
t = np.array(meas['mjd'].copy())
t -= np.mean(t)
t /= 365.2425 # convert to year
# Calculate robust slope
try:
#pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True)
# LADfit
pmra_ladcoef, absdev = dln.ladfit(t,ra)
pmra_lad = pmra_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), ra)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmra_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True)
#pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True)
#pmra = pmra_coef[0]
#pmraerr = pmra_coeferr[0]
#radiff = ra-dln.poly(t,pmra_coef)
radiff = ra-t*pmra_lad
radiff -= np.median(radiff)
rasig = dln.mad(radiff)
# Reject outliers
gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True)
pmra = pmra_coef[0]
pmraerr = pmra_coeferr[0]
rasig = dln.mad(ra-dln.poly(t,pmra_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec
dec = np.array(meas['dec'],np.float64)
dec -= np.mean(dec)
dec *= 3600*1e3 # convert to milli arcsec
# Calculate robust slope
try:
#pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True)
# LADfit
pmdec_ladcoef, absdev = dln.ladfit(t,dec)
pmdec_lad = pmdec_ladcoef[1]
# Run RANSAC
ransac = linear_model.RANSACRegressor()
ransac.fit(t.reshape(-1,1), dec)
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)
gdmask = inlier_mask
pmdec_ransac = ransac.estimator_.coef_[0]
print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers')
# Robust, weighted linear with with INLIERS
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True)
#pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True)
#pmdec = pmdec_coef[0]
#pmdecerr = pmdec_coeferr[0]
#decdiff = dec-dln.poly(t,pmdec_coef)
decdiff = dec-t*pmdec_lad
decdiff -= np.median(decdiff)
decsig = dln.mad(decdiff)
# Reject outliers
gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr)
print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected')
#if np.sum(gdsig) < nmeas:
pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True)
pmdec = pmdec_coef[0]
pmdecerr = pmdec_coeferr[0]
decsig = dln.mad(dec-dln.poly(t,pmdec_coef))
except:
print('problem')
#import pdb; pdb.set_trace()
return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int))
deltamjd = np.max(meas['mjd'])-np.min(meas['mjd'])
out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd])
#print(out[[0,2,3]])
#print(out[[5,7,8]])
#import pdb; pdb.set_trace()
return out
def mkempirical(cube,order=0,coords=None,shape=None,rect=False,lookup=False):
"""
Take a star cube and collapse it to make an empirical PSF using median
and outlier rejection.
Parameters
----------
cube : numpy array
Three-dimensional cube of star images (or residual images) of shape
(Npix,Npix,Nstars).
order : int, optional
The order of the variations. 0-constant, 1-linear terms. If order=1,
Then coords and shape must be input.
coords : tuple, optional
Two-element tuple of the X/Y coordinates of the stars. This is needed
to generate the linear empirical model (order=1).
shape : tuple, optional
Two-element tuple giving the shape (Ny,Nx) of the image. This is
needed to generate the linear empirical model (order=1).
rect : boolean, optional
Return a list of RectBivariateSpline functions rather than a numpy array.
lookup : boolean, optional
Parameter to indicate if this is a lookup table. If lookup=False, then
the constant term is constrained to be non-negative. Default is False.
Returns
-------
epsf : numpy array
The empirical PSF model If order=0, then this is just a 2D image. If
order=1, then it will be a 3D cube (Npix,Npix,4) where the four terms
are [constant, X-term, Y-term, X*Y-term]. If rect=True, then a list
of RectBivariateSpline functions are returned.
Example
-------
epsf = mkempirical(cube,order=0)
or
epsf = mkempirical(cube,order=1,coords=coords,shape=im.shape)
"""
ny,nx,nstar = cube.shape
npix = ny
nhpix = ny//2
# Do outlier rejection in each pixel
med = np.nanmedian(cube,axis=2)
bad = ~np.isfinite(med)
if np.sum(bad)>0:
med[bad] = np.nanmedian(med)
sig = dln.mad(cube,axis=2)
bad = ~np.isfinite(sig)
if np.sum(bad)>0:
sig[bad] = np.nanmedian(sig)
# Mask outlier points
outliers = ((np.abs(cube-med.reshape((med.shape)+(-1,)))>3*sig.reshape((med.shape)+(-1,)))
& np.isfinite(cube))
nbadstar = np.sum(outliers,axis=(0,1))
goodmask = ((np.abs(cube-med.reshape((med.shape)+(-1,)))<3*sig.reshape((med.shape)+(-1,)))
& np.isfinite(cube))
# Now take the mean of the unmasked pixels
macube = np.ma.array(cube,mask=~goodmask)
medim = macube.mean(axis=2)
medim = medim.data
# Check how well each star fits the median
goodpix = macube.count(axis=(0,1))
rms = np.sqrt(np.nansum((cube-medim.reshape((medim.shape)+(-1,)))**2,axis=(0,1))/goodpix)
xx,yy = np.meshgrid(np.arange(npix)-nhpix,np.arange(npix)-nhpix)
rr = np.sqrt(xx**2+yy**2)
x = xx[0,:]
y = yy[:,0]
mask = (rr<=nhpix)
# Constant
if order==0:
# Make sure it goes to zero at large radius
medim *= mask # mask corners
# Make sure values are positive
if lookup==False:
medim = np.maximum(medim,0.0)
if rect:
fpars = [RectBivariateSpline(y,x,medim)]
else:
fpars = medim
# Linear
elif order==1:
if coords is None or shape is None:
raise ValueError('Need coords and shape with order=1')
pars = np.zeros((ny,nx,4),float)
# scale coordinates to -1 to +1
xcen,ycen = coords
relx,rely = models.relcoord(xcen,ycen,shape)
# Loop over pixels and fit line to x/y
for i in range(ny):
for j in range(nx):
data1 = cube[i,j,:]
# maybe use a small maxiter
pars1,perror1 = utils.poly2dfit(relx,rely,data1)
pars[i,j,:] = pars1
# Make sure it goes to zero at large radius
if rect:
fpars = []
for i in range(4):
# Make sure edges are zero on average for higher-order terms
outer = np.median(pars[rr>nhpix*0.8,i])
pars[:,:,i] -= outer
# Mask corners
pars[:,:,i] *= mask
# Each higher-order term must have ZERO total volume
# Set up the spline function that we can use to do
# the interpolation
fpars.append(RectBivariateSpline(y,x,pars[:,:,i]))
else:
fpars = pars
return fpars,nbadstar,rms
def mad(self):
""" Calculate the MAD of the image data. Uses only unmasked data."""
if self.mask is not None:
return dln.mad(self.data[~self.mask])
else:
return dln.mad(self.data)
def getpsf(psf,image,cat,fitradius=None,lookup=False,lorder=0,method='qr',subnei=False,
allcat=None,maxiter=10,minpercdiff=1.0,reject=False,maxrejiter=3,verbose=False):
"""
Fit PSF model to stars in an image with outlier rejection of badly-fit stars.
Parameters
----------
psf : PSF object
PSF object with initial parameters to use.
image : CCDData object
Image to use to fit PSF model to stars.
cat : table
Catalog with initial amp/x/y values for the stars to use to fit the PSF.
fitradius : float, table
The fitting radius. If none is input then the initial PSF FWHM will be used.
lookup : boolean, optional
Use an empirical lookup table. Default is False.
lorder : int, optional
The order of the spatial variations (0=constant, 1=linear). Default is 0.
method : str, optional
Method to use for solving the non-linear least squares problem: "qr",
"svd", "cholesky", and "curve_fit". Default is "qr".
subnei : boolean, optional
Subtract stars neighboring the PSF stars. Default is False.
allcat : table, optional
Catalog of all objects in the image. This is needed for bad PSF star
rejection.
maxiter : int, optional
Maximum number of iterations to allow. Only for methods "qr", "svd", and "cholesky".
Default is 10.
minpercdiff : float, optional
Minimum percent change in the parameters to allow until the solution is
considered converged and the iteration loop is stopped. Only for methods
"qr" and "svd". Default is 1.0.
reject : boolean, optional
Reject PSF stars with high RMS values. Default is False.
maxrejiter : int, boolean
Maximum number of PSF star rejection iterations. Default is 3.
verbose : boolean, optional
Verbose output.
Returns
-------
newpsf : PSF object
New PSF object with the best-fit model parameters.
pars : numpy array
Array of best-fit model parameters
perror : numpy array
Uncertainties in "pars".
psfcat : table
Table of best-fitting amp/xcen/ycen values for the PSF stars.
Example
-------
newpsf,pars,perror,psfcat = getpsf(psf,image,cat)
"""
t0 = time.time()
print = utils.getprintfunc() # Get print function to be used locally, allows for easy logging
# Fitting radius
if fitradius is None:
if type(psf)==models.PSFPenny:
fitradius = psf.fwhm()*1.5
else:
fitradius = psf.fwhm()
# subnei but no allcat input
if subnei and allcat is None:
raise ValueError('allcat is needed for PSF neighbor star subtraction')
if 'id' not in cat.colnames:
cat['id'] = np.arange(len(cat))+1
psfcat = cat.copy()
# Initializing output PSF star catalog
dt = np.dtype([('id',int),('amp',float),('x',float),('y',float),('npix',int),('rms',float),
('chisq',float),('ixmin',int),('ixmax',int),('iymin',int),('iymax',int),('reject',int)])
outcat = np.zeros(len(cat),dtype=dt)
outcat = Table(outcat)
for n in ['id','x','y']:
outcat[n] = cat[n]
# Remove stars that are too close to the edge
ny,nx = image.shape
bd = (psfcat['x']<fitradius) | (psfcat['x']>(nx-1-fitradius)) | \
(psfcat['y']<fitradius) | (psfcat['y']>(ny-1-fitradius))
nbd = np.sum(bd)
if nbd > 0:
if verbose:
print('Removing '+str(nbd)+' stars near the edge')
psfcat = psfcat[~bd]
# Generate an empirical image of the stars
# and fit a model to it to get initial estimates
if type(psf)!=models.PSFEmpirical:
cube = starcube(psfcat,image,npix=psf.npix,fillvalue=np.nan)
epsf,nbadstar,rms = mkempirical(cube,order=0)
epsfim = CCDData(epsf,error=epsf.copy()*0+1,mask=~np.isfinite(epsf))
pars,perror,mparams = psf.fit(epsfim,pars=[1.0,psf.npix/2,psf.npix//2],allpars=True)
initpar = mparams.copy()
curpsf = psf.copy()
curpsf.params = initpar
if verbose:
print('Initial estimate from empirical PSF fit = '+str(mparams))
else:
curpsf = psf.copy()
initpar = psf.params.copy()
# Outlier rejection iterations
nrejiter = 0
flag = 0
nrejstar = 100
fitrad = fitradius
useimage = image.copy()
while (flag==0):
if verbose:
print('--- Iteration '+str(nrejiter+1)+' ---')
# Update the fitting radius
if nrejiter>0:
fitrad = curpsf.fwhm()
if verbose:
print(' Fitting radius = %5.3f' % (fitrad))
# Reject outliers
if reject and nrejiter>0:
medrms = np.median(pcat['rms'])
sigrms = dln.mad(pcat['rms'].data)
gd, = np.where(pcat['rms'] < medrms+3*sigrms)
nrejstar = len(psfcat)-len(gd)
if verbose:
print(' RMS = %6.4f +/- %6.4f' % (medrms,sigrms))
print(' Threshold RMS = '+str(medrms+3*sigrms))
print(' Rejecting '+str(nrejstar)+' stars')
if nrejstar>0:
psfcat = psfcat[gd]
# Subtract neighbors
if nrejiter>0 and subnei:
if verbose:
print('Subtracting neighbors')
# Find the neighbors in allcat
# Fit the neighbors and PSF stars
# Subtract neighbors from the image
useimage = image.copy() # start with original image
useimage = subtractnei(useimage,allcat,cat,curpsf)
# Fitting the PSF to the stars
#-----------------------------
newpsf,pars,perror,pcat,pf = fitpsf(curpsf,useimage,psfcat,fitradius=fitrad,method=method,
maxiter=maxiter,minpercdiff=minpercdiff,verbose=verbose)
# Add information into the output catalog
ind1,ind2 = dln.match(outcat['id'],pcat['id'])
outcat['reject'] = 1
for n in pcat.columns:
outcat[n][ind1] = pcat[n][ind2]
outcat['reject'][ind1] = 0
# Compare PSF parameters
if type(newpsf)!=models.PSFEmpirical:
pardiff = newpsf.params-curpsf.params
else:
pardiff = newpsf._data-curpsf._data
sumpardiff = np.sum(np.abs(pardiff))
curpsf = newpsf.copy()
# Stopping criteria
if reject is False or sumpardiff<0.05 or nrejiter>=maxrejiter or nrejstar==0: flag=1
if subnei is True and nrejiter==0: flag=0 # iterate at least once with neighbor subtraction
nrejiter += 1
# Generate an empirical look-up table of corrections
if lookup:
if verbose:
print('Making empirical lookup table with order='+str(lorder))
pf.mklookup(lorder)
# Fit the stars again and get new RMS values
xdata = np.arange(pf.ntotpix)
out = pf.model(xdata,*pf.psf.params)
newpsf = pf.psf.copy()
# Update information in the output catalog
ind1,ind2 = dln.match(outcat['id'],pcat['id'])
outcat['reject'] = 1
outcat['reject'][ind1] = 0
outcat['amp'][ind1] = pf.staramp[ind2]
outcat['x'][ind1] = pf.starxcen[ind2]
outcat['y'][ind1] = pf.starycen[ind2]
outcat['rms'][ind1] = pf.starrms[ind2]
outcat['chisq'][ind1] = pf.starchisq[ind2]
if verbose:
print('Median RMS: '+str(np.median(pf.starrms)))
if verbose:
print('dt = %.2f sec' % (time.time()-t0))
return newpsf, pars, perror, outcat
def gausspeakfit(spec, pix0=None, estsig=5, sigma=None, func=gaussbin):
""" Return integrated-Gaussian centers near input pixel center
Parameters
----------
spec : float array
Data spectrum array.
pix0 : float or integer (scalar or array)
Initial pixel guess.
estsig : float
Initial guess for window width=5*estsig (default=5).
sigma : float array, optional
Uncertainty array (default=None).
func : function, optional
User-supplied function to use to fit (default=gaussbin).
Returns
-------
pars : float array
Array of best-fitting parameters (height, center, sigma, yoffset)
perr : float array
Array of uncertainties of the parameters.
Examples
--------
pars,perr = gausspeakfit(spec,10,sigma=specerr)
"""
# No initial guess input, use maximum value
if pix0 is None:
pix0 = spec.argmax()
medspec = np.median(spec)
sigspec = dln.mad(spec - medspec)
dx = 1
x = np.arange(len(spec))
xwid = 5
xlo0 = pix0 - xwid
xhi0 = pix0 + xwid + 1
xx0 = x[xlo0:xhi0]
# Get quantitative estimates of height, center, sigma
flux = spec[xlo0:xhi0] - medspec
flux -= np.median(flux) # put the median at zero
flux = np.maximum(0, flux) # don't want negative pixels
ht0 = np.max(flux)
totflux = np.sum(flux)
# Gaussian area is A = ht*wid*sqrt(2*pi)
sigma0 = np.maximum((totflux * dx) / (ht0 * np.sqrt(2 * np.pi)), 0.01)
cen0 = np.sum(flux * xx0) / totflux
cen0 = np.minimum(np.maximum((pix0 - dx * 0.5), cen0),
(pix0 + dx * 0.5)) # constrain the center
# Use linear-least squares to calculate height and sigma
psf1 = np.exp(-0.5 * (xx0 - cen0)**2 / sigma0**2) # normalized Gaussian
wtht1 = np.sum(flux * psf1) / np.sum(psf1 * psf1) # linear least squares
# Second iteration
sigma1 = (totflux * dx) / (wtht1 * np.sqrt(2 * np.pi))
psf2 = np.exp(-0.5 * (xx0 - cen0)**2 / sigma1**2) # normalized Gaussian
wtht2 = np.sum(flux * psf2) / np.sum(psf2 * psf2)
# Now get more pixels to fit if necessary
npixwide = int(np.maximum(np.ceil((2 * sigma1) / dx), 5))
xlo = int(np.round(cen0)) - npixwide
xhi = int(np.round(cen0)) + npixwide + 1
xx = x[xlo:xhi]
y = spec[xlo:xhi]
yerr = sigma[xlo:xhi]
# Bounds
initpar = [wtht2, cen0, sigma1, medspec]
lbounds = [
0.5 * ht0, initpar[1] - 1, 0.2,
initpar[3] - np.maximum(3 * sigspec, 0.3 * np.abs(initpar[3]))
]
ubounds = [
3.0 * ht0, initpar[1] + 1, 5.0,
initpar[3] + np.maximum(3 * sigspec, 0.3 * np.abs(initpar[3]))
]
bounds = (lbounds, ubounds)
# Sometimes curve_fit hits the maximum number fo function evaluations and crashes
try:
pars, cov = curve_fit(func,
xx,
y,
p0=initpar,
sigma=yerr,
bounds=bounds,
maxfev=1000)
perr = np.sqrt(np.diag(cov))
except:
return None, None
return pars, perr
def peakfit(spec, sigma=None, pix0=None):
"""
Find lines in a spectrum fit Gaussians to them.
Parameters
----------
spec : float array
Data spectrum array.
sigma : float array, optional
Uncertainty array (default=None).
pix0 : float or integer (scalar or array), optional
Initial pixel guess.
Returns
-------
pars : numpy structured array
Table of best-fitting parameters and uncertainties.
Examples
--------
pars = peakfit(spec,sigma=specerr)
"""
if np.ndim(spec) > 1:
raise ValueError('Spec must be 1-D')
# X array
npix = len(spec)
x = np.arange(npix)
smspec = median_filter(spec, 101, mode='nearest')
sigspec = dln.mad(spec - smspec)
# Find the peaks
if pix0 is None:
maxind, = argrelextrema(spec - smspec, np.greater) # maxima
# sigma cut on the flux
gd, = np.where((spec - smspec)[maxind] > 4 * sigspec)
if len(gd) == 0:
print('No peaks found')
return
pix0 = maxind[gd]
pix0 = np.atleast_1d(pix0)
npeaks = len(pix0)
# Initialize the output table
dtype = np.dtype([('num', int), ('pix0', int), ('pars', np.float64, 4),
('perr', np.float64, 4), ('success', bool)])
out = np.zeros(npeaks, dtype=dtype)
out['num'] = np.arange(npeaks)
out['pix0'] = pix0
# Initialize the residuals spectrum
resid = spec.copy()
# Loop over the peaks and fit them
for i in range(npeaks):
# Run gausspeakfit() on the residual
pars, perr = gausspeakfit(resid, pix0=pix0[i], sigma=sigma)
if pars is not None:
# Get model and subtract from residuals
xlo = np.maximum(0, int(pars[1] - 5 * pars[2]))
xhi = np.minimum(npix, int(pars[1] + 5 * pars[2]))
peakmodel = gaussbin(x[xlo:xhi], pars[0], pars[1],
pars[2]) # leave yoffset in
resid[xlo:xhi] -= peakmodel
# Stuff results in output table
out['pars'][i] = pars
out['perr'][i] = perr
out['success'][i] = True
else:
out['pars'][i] = np.nan
out['perr'][i] = np.nan
return out
def orig_salariscna(abund, mc=False):
'''
Calculate the Salaris corrected [Fe/H] according to Salaris et al. 1993 with Piersanti et al. 2007 and
Asplund et al. 2009. Also C and N have been added to the alpha elements and Ne has been excluded.
Inputs:
------
abund: [9x2 array] first column is [Fe/H],[C/Fe],[N/Fe],[O/Fe],[Mg/Fe],[Si/Fe],[S/Fe],[Ca/Fe],[Ti/Fe]
and second column is the errors
Output:
------
calc_salfeh: Salaris corrected metallicity
calc_salfeh_err: error in corrected metallicity
'''
### Salaris coefficients
# (atomic_wgts/hydrogen_wgt) = (C,N,O,Mg,Si,S,Ca,Ti)/H
asplund = np.array([8.43, 7.83, 8.69, 7.60, 7.51, 7.12, 6.34, 4.95])
mass_ratio = np.array([
12.011, 14.007, 15.999, 24.305, 28.085, 32.06, 40.078, 47.867
]) / 1.008 #IUPAC
# with Ne
# (atomic_wgts/hydrogen_wgt) = (C,N,O,Ne,Mg,Si,S,Ca,Ti)/H
#asplund = np.array([8.43,7.83,8.69,7.84,7.60,7.51,7.12,6.34,4.95])
#mass_ratio = np.array([12.011,14.007,15.999,20.1797,24.305,28.085,32.06,40.078,47.867])/1.008 #IUPAC
ZX_sol = 0.0181 # (Z/X) Asplund et al. 2009
XZ_k = np.multiply(10**(asplund - 12.0), mass_ratio / ZX_sol)
sal_a = np.sum(XZ_k)
sal_b = 1 - sal_a
### Alpha+C+N
wgts = asplund / np.sum(asplund)
### Replace bad values with solar
for i in range(len(abund[:, 0])):
if abund[i, 0] < -10. or abund[i, 0] > 10. or np.isfinite(
abund[i, 0]) == False:
abund[i, 0] = 0.0
if abund[i, 1] < -10. or abund[i, 1] > 10. or np.isfinite(
abund[i, 1]) == False:
abund[i, 1] = 0.0
feh = abund[0, 0]
feh_err = abund[0, 1]
cnalpha = abund[1:, 0]
cnafe = np.log10(np.sum(np.multiply(10**cnalpha, wgts)))
salfeh = feh + np.log10(sal_a * 10**(cnafe) + sal_b)
### MC for Salaris Correction
if mc:
cnalpha_err = abund[1:, 1]
nsamples = 1000
salfehdist = 999999.0 * np.ones(nsamples)
noisyfeh = np.random.normal(feh, feh_err, nsamples)
for i in range(nsamples):
noisycnalpha = 999999.0 * np.ones(len(cnalpha))
for j in range(len(cnalpha)):
noisycnalpha[j] = np.random.normal(cnalpha[j], cnalpha_err[j])
cnafe = np.log10(np.sum(np.multiply(10**noisycnalpha, wgts)))
salfehdist[i] = noisyfeh[i] + np.log10(sal_a * 10**(cnafe) + sal_b)
calc_salfeh_err = dln.mad(salfehdist)
return salfeh, calc_salfeh_err
return salfeh
def normalize(self, ncorder=6, perclevel=0.95):
"""
Normalize the spectrum.
Parameters
----------
ncorder : float, optional
Polynomial order to use for the continuum fitting.
The default is 6.
perclevel : float, optional
Percent level (1.0 for 100%) to use for the continuum value
in large bins. Default is 0.95.
Returns
-------
The flux and err arrays will normalized (divided) by the continuum
and the continuum saved in cont. The normalized property is set to
True.
Examples
--------
spec.normalize()
"""
self._flux = self.flux # Save the original
#nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel)
binsize = 0.10
perclevel = 90.0
wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D
flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D
err = self.err.copy().reshape(self.npix, self.norder) # make 2D
mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D
cont = err.copy() * 0.0 + 1
for o in range(self.norder):
w = wave[:, o].copy()
x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5)
) # -1 to +1
y = flux[:, o].copy()
m = mask[:, o].copy()
# Divide by median
medy = np.nanmedian(y)
y /= medy
# Perform sigma clipping out large positive outliers
coef = dln.poly_fit(x, y, 2, robust=True)
sig = dln.mad(y - dln.poly(x, coef))
bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig)
if nbd > 0: m[bd] = True
gdmask = (y > 0) & (m == False
) # need positive fluxes and no mask set
# Bin the data points
xr = [np.nanmin(x), np.nanmax(x)]
bins = np.ceil((xr[1] - xr[0]) / binsize) + 1
ybin, bin_edges, binnumber = bindata.binned_statistic(
x[gdmask],
y[gdmask],
statistic='percentile',
percentile=perclevel,
bins=bins,
range=None)
xbin = bin_edges[0:-1] + 0.5 * binsize
# Interpolate to full grid
fnt = np.isfinite(ybin)
cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True)
cont1 *= medy
flux[:, o] /= cont1
err[:, o] /= cont1
cont[:, o] = cont1
# Flatten to 1D if norder=1
if self.norder == 1:
flux = flux.flatten()
err = err.flatten()
cont = cont.flatten()
# Stuff back in
self.flux = flux
self.err = err
self.cont = cont
self.normalized = True
return
