def maskoutliers(spec, nsig=5, verbose=False): """ Mask large positive outliers and negative flux pixels in the spectrum. Parameters ---------- spec : Spec1D object Observed spectrum to mask outliers on. nsig : int, optional Number of standard deviations to use for the outlier rejection. Default is 5.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with outliers masked. Example ------- .. code-block:: python spec = maskoutliers(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0)) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' outlier or negative pixels') return spec2
def maskdiscrepant(spec, model, nsig=10, verbose=False): """ Mask pixels that are discrepant when compared to a model. Parameters ---------- spec : Spec1D object Observed spectrum for which to mask discrepant values. model : Spec1D object Reference/model spectrum to use to find discrepant values. nsig : int, optional Number of standard deviations to use for the discrepant values. Default is 10.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with discrepant values masked. Example ------- .. code-block:: python spec = maskdiscrepant(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() my = mflux[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - my) bd, nbd = dln.where(np.abs(y - my) > nsig * sig) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels') return spec2
def meascutout(meas, obj, size=10, outdir='./'): """ Input the measurements and create cutouts. """ expstr = fits.getdata( '/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1) #expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1) decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt', format='ascii') objid = obj['id'][0] # Sort by MJD si = np.argsort(meas['mjd']) meas = meas[si] ind1, ind2 = dln.match(expstr['base'], meas['exposure']) nind = len(ind1) if nind == 0: print('No matches') return # Sort by input meas catalog si = np.argsort(ind2) ind1 = ind1[si] ind2 = ind2[si] # Create the reference WCS wref = WCS(naxis=2) pixscale = 0.26 # DECam, "/pix npix = round(size / pixscale) if npix % 2 == 0: # must be odd npix += 1 hpix = npix // 2 # center of image wref.wcs.ctype = ['RA---TAN', 'DEC--TAN'] wref.wcs.crval = [obj['ra'][0], obj['dec'][0]] wref.wcs.crpix = [npix // 2, npix // 2] wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]]) wref.array_shape = (npix, npix) refheader = wref.to_header() refheader['NAXIS'] = 2 refheader['NAXIS1'] = npix refheader['NAXIS2'] = npix # Load the data instrument = expstr['instrument'][ind1] fluxfile = expstr['file'][ind1] fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk maskfile = expstr['maskfile'][ind1] maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk ccdnum = meas['ccdnum'][ind2] figfiles = [] for i in range(nind): try: if instrument[i] == 'c4d': dind, = np.where(decam['CCDNUM'] == ccdnum[i]) extname = decam['NAME'][dind[0]] im, head = getfitsext(fluxfile[i], extname, header=True) mim, mhead = getfitsext(maskfile[i], extname, header=True) #im,head = fits.getdata(fluxfile[i],header=True,extname=extname) #mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname) else: im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True) mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True) except: print('error') import pdb pdb.set_trace() # Get chip-level information exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" + exposure + "' and ccdnum=" + str(ccdnum[i]), fmt='table') w = WCS(head) # RA/DEC correction for the object lon = obj['ra'][0] - chres['ra'][0] lat = obj['dec'][0] - chres['dec'][0] racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[ 'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[ 'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat # apply these offsets to the header WCS CRVAL w.wcs.crval += [racorr, deccorr] head['CRVAL1'] += racorr head['CRVAL2'] += deccorr # Object X/Y position xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0) # Get the cutout xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes ycen = meas['y'][ind2[i]] - 1 smim = dln.gsmooth(im, 2) # use the object coords for centering #cutim,xr,yr = cutout(smim,xobj,yobj,size) # Mask the bad pixels badmask = (mim > 0) im[badmask] = np.nanmedian(im[~badmask]) # Create a common TAN WCS that each image gets interpoled onto!!! #hdu1 = fits.open(fluxfile[i],extname=extname) smim1 = dln.gsmooth(im, 1.5) hdu = fits.PrimaryHDU(smim1, head) cutim, footprint = reproject_interp(hdu, refheader, order='bicubic') # biquadratic cutim[footprint == 0] = np.nanmedian( im[~badmask]) # set out-of-bounds to background #xr = [0,npix-1] #yr = [0,npix-1] xr = [-hpix * pixscale, hpix * pixscale] yr = [-hpix * pixscale, hpix * pixscale] # exposure_ccdnum, filter, MJD, delta_MJD, mag print( str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' + str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' + str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]])) #figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/' figfile = outdir figfile += '%s_%04d_%s_%02d.jpg' % (str( obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i]) figfiles.append(figfile) matplotlib.use('Agg') plt.rcParams.update({'font.size': 11}) if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear figsize = 8.0 #6.0 ax = fig.subplots() # projection=wcs #fig.set_figheight(figsize*0.8) fig.set_figheight(figsize) fig.set_figwidth(figsize) med = np.nanmedian(smim) sig = dln.mad(smim) bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med) lmed = np.nanmedian(bigim) # Get the flux of the object and scale each image to the same height #meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm #cmag = mag_auto + 2.5*alog10(exptime) + zpterm instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10( chres['exptime'][0]) - chres['zpterm'][0] #mag = -2.5*log(flux)+25.0 instflux = 10**((25.0 - instmag) / 2.5) print('flux = ' + str(instflux)) # Get height of object # flux of 2D Gaussian is ~2*pi*height*sigma^2 pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600 fwhm = chres['fwhm'][0] / pixscale1 instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2) print('height = ' + str(instheight)) # Scale the images to the flux level of the first image cutim -= lmed if i == 0: instflux0 = instflux.copy() instheight0 = instheight.copy() else: scale = instflux0 / instflux #scale = instheight0/instheight cutim *= scale print('scaling image by ' + str(scale)) #vmin = lmed-8*sig # 3*sig #vmax = lmed+12*sig # 5*sig if i == 0: vmin = -8 * sig # 3*sig #vmax = 12*sig # 5*sig vmax = 0.5 * instheight # 0.5 vmin0 = vmin vmax0 = vmax else: vmin = vmin0 vmax = vmax0 print('vmin = ' + str(vmin)) print('vmax = ' + str(vmax)) plt.imshow(cutim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet #plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none', # vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet #plt.colorbar() # show one vertical, one horizontal line pointing to the center but offset # then a small dot on the meas position # 13, 8 plt.plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7) plt.plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7) # Meas X/Y position xmeas, ymeas = wref.all_world2pix(meas['ra'][ind2[i]], meas['dec'][ind2[i]], 0) plt.scatter([(xmeas - hpix) * pixscale], [(ymeas - hpix) * pixscale], c='r', marker='+', s=20) #plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100) #plt.scatter([xcen],[ycen],c='r',marker='+',s=100) # Object X/Y position #xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0) xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0) #plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3) #plt.scatter(xobj,yobj,c='y',marker='+',s=100) #leg = ax.legend(loc='upper left', frameon=False) plt.xlabel(r'$\Delta$ RA (arcsec)') plt.ylabel(r'$\Delta$ DEC (arcsec)') #plt.xlabel('X') #plt.ylabel('Y') #plt.xlim(xr) #plt.ylim(yr) #ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center') co = 'white' #'lightgray' # blue ax.annotate('%s %02d %s %6.1f ' % (meas['exposure'][ind2[i]], ccdnum[i], meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]), xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05), ha='center', color=co) ax.annotate( '%10.2f %10.2f ' % (meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])), xy=(xr[0] + dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='left', color=co) ax.annotate('%s = %5.2f +/- %4.2f' % (meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]], meas['magerr_auto'][ind2[i]]), xy=(xr[1] - dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='right', color=co) plt.savefig(figfile) print('Cutout written to ' + figfile) #import pdb; pdb.set_trace() # Make a single blank file at the end so you know it looped figfile = outdir figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'blank') figfiles.append(figfile) matplotlib.use('Agg') if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear figsize = 8.0 #6.0 fig.set_figheight(figsize) fig.set_figwidth(figsize) plt.savefig(figfile) print(figfile) # Make the animated gif animfile = outdir + str(objid) + '_cutouts.gif' print('Creating animated gif ' + animfile) if os.path.exists(animfile): os.remove(animfile) #ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True) ret = subprocess.run('convert -delay 20 ' + ' '.join(figfiles) + ' ' + animfile, shell=True) #import pdb; pdb.set_trace() dln.remove(figfiles)
def meascutout(meas, obj, size=10, outdir='./', domask=True): """ Input the measurements and create cutouts. """ expstr = fits.getdata( '/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposures.fits.gz', 1) #expstr = fits.getdata('/net/dl2/dnidever/nsc/instcal/v3/lists/nsc_v3_exposure.fits.gz',1) decam = Table.read('/home/dnidever/projects/delvered/data/decam.txt', format='ascii') objid = obj['id'][0] # Sort by MJD si = np.argsort(meas['mjd']) meas = meas[si] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='" + objid + "'" chip = qc.query(sql=sql, fmt='table') ind3, ind4 = dln.match(chip['exposure'], meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2 * chip['fwhm']) & (meas['fwhm'] < 2.0 * chip['fwhm'])) if len(gdfwhm) == 0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing ' + str(len(meas) - len(gdfwhm)) + ' measurements with bad FWHM values') meas = meas[gdfwhm] ind1, ind2 = dln.match(expstr['base'], meas['exposure']) nind = len(ind1) if nind == 0: print('No matches') return # Sort by input meas catalog si = np.argsort(ind2) ind1 = ind1[si] ind2 = ind2[si] # Create the reference WCS wref = WCS(naxis=2) pixscale = 0.26 # DECam, "/pix npix = round(size / pixscale) if npix % 2 == 0: # must be odd npix += 1 hpix = npix // 2 # center of image wref.wcs.ctype = ['RA---TAN', 'DEC--TAN'] wref.wcs.crval = [obj['ra'][0], obj['dec'][0]] wref.wcs.crpix = [npix // 2, npix // 2] wref.wcs.cd = np.array([[pixscale / 3600.0, 0.0], [0.0, pixscale / 3600]]) wref.array_shape = (npix, npix) refheader = wref.to_header() refheader['NAXIS'] = 2 refheader['NAXIS1'] = npix refheader['NAXIS2'] = npix # Load the data instrument = expstr['instrument'][ind1] plver = expstr['plver'][ind1] fluxfile = expstr['file'][ind1] fluxfile = fluxfile.replace('/net/mss1/', '/mss1/') # for thing/hulk maskfile = expstr['maskfile'][ind1] maskfile = maskfile.replace('/net/mss1/', '/mss1/') # for thing/hulk ccdnum = meas['ccdnum'][ind2] figfiles = [] xmeas = [] ymeas = [] cutimarr = np.zeros((npix, npix, nind), float) for i in range(nind): #for i in range(3): instcode = instrument[i] plver1 = plver[i] try: if instrument[i] == 'c4d': dind, = np.where(decam['CCDNUM'] == ccdnum[i]) extname = decam['NAME'][dind[0]] im, head = getfitsext(fluxfile[i], extname, header=True) mim, mhead = getfitsext(maskfile[i], extname, header=True) #im,head = fits.getdata(fluxfile[i],header=True,extname=extname) #mim,mhead = fits.getdata(maskfile[i],header=True,extname=extname) else: im, head = fits.getdata(fluxfile[i], ccdnum[i], header=True) mim, mhead = fits.getdata(maskfile[i], ccdnum[i], header=True) except: print('error') import pdb pdb.set_trace() # Turn the mask from integer to bitmask if ((instcode == 'c4d') & (plver1 >= 'V3.5.0')) | (instcode == 'k4m') | (instcode == 'ksb'): omim = mim.copy() mim *= 0 nonzero = (omim > 0) mim[nonzero] = 2**((omim - 1)[nonzero]) # This takes about 1 sec # Fix the DECam Pre-V3.5.0 masks if (instcode == 'c4d') & (plver1 < 'V3.5.0'): omim = mim.copy() mim *= 0 # re-initialize mim += (np.bitwise_and(omim, 1) == 1) * 1 # bad pixels mim += (np.bitwise_and(omim, 2) == 2) * 4 # saturated mim += (np.bitwise_and(omim, 4) == 4) * 32 # interpolated mim += (np.bitwise_and(omim, 16) == 16) * 16 # cosmic ray mim += (np.bitwise_and(omim, 64) == 64) * 8 # bleed trail # Get chip-level information exposure = os.path.basename(fluxfile[i])[0:-8] # remove fits.fz chres = qc.query(sql="select * from nsc_dr2.chip where exposure='" + exposure + "' and ccdnum=" + str(ccdnum[i]), fmt='table') w = WCS(head) # RA/DEC correction for the object lon = obj['ra'][0] - chres['ra'][0] lat = obj['dec'][0] - chres['dec'][0] racorr = chres['ra_coef1'][0] + chres['ra_coef2'][0] * lon + chres[ 'ra_coef3'][0] * lon * lat + chres['ra_coef4'][0] * lat deccorr = chres['dec_coef1'][0] + chres['dec_coef2'][0] * lon + chres[ 'dec_coef3'][0] * lon * lat + chres['dec_coef4'][0] * lat # apply these offsets to the header WCS CRVAL #w.wcs.crval += [racorr,deccorr] #head['CRVAL1'] += racorr #head['CRVAL2'] += deccorr print(racorr, deccorr) # Object X/Y position xobj, yobj = w.all_world2pix(obj['ra'], obj['dec'], 0) # Get the cutout xcen = meas['x'][ind2[i]] - 1 # convert to 0-indexes ycen = meas['y'][ind2[i]] - 1 smim = dln.gsmooth(im, 2) # use the object coords for centering #cutim,xr,yr = cutout(smim,xobj,yobj,size) # Mask the bad pixels if domask == True: badmask = (mim > 0) im[badmask] = np.nanmedian(im[~badmask]) else: badmask = (im < 0) # Create a common TAN WCS that each image gets interpoled onto!!! #hdu1 = fits.open(fluxfile[i],extname=extname) smim1 = dln.gsmooth(im, 1.5) hdu = fits.PrimaryHDU(smim1, head) cutim, footprint = reproject_interp(hdu, refheader, order='bicubic') # biquadratic cutim[footprint == 0] = np.nanmedian( im[~badmask]) # set out-of-bounds to background #xr = [0,npix-1] #yr = [0,npix-1] xr = [-hpix * pixscale, hpix * pixscale] yr = [-hpix * pixscale, hpix * pixscale] # exposure_ccdnum, filter, MJD, delta_MJD, mag print( str(i + 1) + ' ' + meas['exposure'][ind2[i]] + ' ' + str(ccdnum[i]) + ' ' + str(meas['x'][ind2[i]]) + ' ' + str(meas['y'][ind2[i]]) + ' ' + str(meas['mag_auto'][ind2[i]])) #figdir = '/net/dl2/dnidever/nsc/instcal/v3/hpm2/cutouts/' figfile = outdir figfile += '%s_%04d_%s_%02d.jpg' % (str( obj['id'][0]), i + 1, meas['exposure'][ind2[i]], ccdnum[i]) figfiles.append(figfile) matplotlib.use('Agg') plt.rc('font', size=15) plt.rc('axes', titlesize=20) plt.rc('axes', labelsize=20) plt.rc('xtick', labelsize=20) plt.rc('ytick', labelsize=20) #plt.rcParams.update({'font.size': 15}) #plt.rcParams.update({'axes.size': 20}) #plt.rcParams.update({'xtick.size': 20}) #plt.rcParams.update({'ytick.size': 20}) if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear gskw = dict(width_ratios=[30, 1]) fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw) figsize = 8.0 #6.0 figheight = 8.0 figwidth = 9.0 #ax = fig.subplots() # projection=wcs #fig.set_figheight(figsize*0.8) fig.set_figheight(figheight) fig.set_figwidth(figwidth) med = np.nanmedian(smim) sig = dln.mad(smim) bigim, xr2, yr2 = cutout(smim, xcen, ycen, 151, missing=med) lmed = np.nanmedian(bigim) # Get the flux of the object and scale each image to the same height #meas.mag_aper1 = cat1.mag_aper[0] + 2.5*alog10(exptime) + chstr[i].zpterm #cmag = mag_auto + 2.5*alog10(exptime) + zpterm instmag = meas['mag_auto'][ind2[i]] - 2.5 * np.log10( chres['exptime'][0]) - chres['zpterm'][0] #mag = -2.5*log(flux)+25.0 instflux = 10**((25.0 - instmag) / 2.5) print('flux = ' + str(instflux)) # Get height of object # flux of 2D Gaussian is ~2*pi*height*sigma^2 pixscale1 = np.max(np.abs(w.wcs.cd)) * 3600 fwhm = chres['fwhm'][0] / pixscale1 instheight = instflux / (2 * 3.14 * (fwhm / 2.35)**2) print('height = ' + str(instheight)) # Scale the images to the flux level of the first image cutim -= lmed if i == 0: instflux0 = instflux.copy() instheight0 = instheight.copy() else: scale = instflux0 / instflux #scale = instheight0/instheight cutim *= scale print('scaling image by ' + str(scale)) #vmin = lmed-8*sig # 3*sig #vmax = lmed+12*sig # 5*sig if i == 0: vmin = -8 * sig # 3*sig #vmax = 12*sig # 5*sig vmax = 0.5 * instheight # 0.5 vmin0 = vmin vmax0 = vmax else: vmin = vmin0 vmax = vmax0 print('vmin = ' + str(vmin)) print('vmax = ' + str(vmax)) cutimarr[:, :, i] = cutim.copy() ax[0].imshow(cutim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet #plt.imshow(cutim,origin='lower',aspect='auto',interpolation='none', # vmin=vmin,vmax=vmax,cmap='viridis') # viridis, Greys, jet #plt.colorbar() # show one vertical, one horizontal line pointing to the center but offset # then a small dot on the meas position # 13, 8 ax[0].plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7) ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7) # Meas X/Y position xmeas1, ymeas1 = wref.all_world2pix(meas['ra'][ind2[i]], meas['dec'][ind2[i]], 0) xmeas.append(xmeas1) ymeas.append(ymeas1) ax[0].scatter([(xmeas1 - hpix) * pixscale], [(ymeas1 - hpix) * pixscale], c='r', marker='+', s=20) #plt.scatter([xmeas],[ymeas],c='r',marker='+',s=100) #plt.scatter([xcen],[ycen],c='r',marker='+',s=100) # Object X/Y position #xobj,yobj = w.all_world2pix(obj['ra'],obj['dec'],0) xobj, yobj = wref.all_world2pix(obj['ra'], obj['dec'], 0) #plt.scatter(xobj,yobj,marker='o',s=200,facecolors='none',edgecolors='y',linewidth=3) #plt.scatter(xobj,yobj,c='y',marker='+',s=100) #leg = ax.legend(loc='upper left', frameon=False) ax[0].set_xlabel(r'$\Delta$ RA (arcsec)') ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)') ax[0].set_xlim((xr[1], xr[0])) # sky right ax[0].set_ylim(yr) #plt.xlabel('X') #plt.ylabel('Y') #plt.xlim(xr) #plt.ylim(yr) #ax.annotate(r'S/N=%5.1f',xy=(np.mean(xr), yr[0]+dln.valrange(yr)*0.05),ha='center') co = 'white' #'lightgray' # blue ax[0].annotate('%s %02d %s %6.1f ' % (meas['exposure'][ind2[i]], ccdnum[i], meas['filter'][ind2[i]], expstr['exptime'][ind1[i]]), xy=(np.mean(xr), yr[0] + dln.valrange(yr) * 0.05), ha='center', color=co) ax[0].annotate( '%10.2f $\Delta$t=%7.2f ' % (meas['mjd'][ind2[i]], meas['mjd'][ind2[i]] - np.min(meas['mjd'])), xy=(xr[1] - dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='left', color=co) # xy=(xr[0]+dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='left',color=co) ax[0].annotate('%s = %5.2f +/- %4.2f' % (meas['filter'][ind2[i]], meas['mag_auto'][ind2[i]], meas['magerr_auto'][ind2[i]]), xy=(xr[0] + dln.valrange(xr) * 0.05, yr[1] - dln.valrange(yr) * 0.05), ha='right', color=co) # xy=(xr[1]-dln.valrange(xr)*0.05, yr[1]-dln.valrange(yr)*0.05),ha='right',color=co) # Progress bar frameratio = (i + 1) / float(nind) timeratio = (meas['mjd'][ind2[i]] - np.min(meas['mjd'])) / dln.valrange(meas['mjd']) #ratio = frameratio ratio = timeratio print('ratio = ' + str(100 * ratio)) barim = np.zeros((100, 100), int) ind = dln.limit(int(round(ratio * 100)), 1, 99) barim[:, 0:ind] = 1 ax[1].imshow(barim.T, origin='lower', aspect='auto', cmap='Greys') ax[1].set_xlabel('%7.1f \n days' % (meas['mjd'][ind2[i]] - np.min(meas['mjd']))) #ax[1].set_xlabel('%d/%d' % (i+1,nind)) ax[1].set_title('%d/%d' % (i + 1, nind)) ax[1].axes.xaxis.set_ticks([]) #ax[1].axes.xaxis.set_visible(False) ax[1].axes.yaxis.set_visible(False) #ax[1].axis('off') right_side = ax[1].spines['right'] right_side.set_visible(False) left_side = ax[1].spines['left'] left_side.set_visible(False) top_side = ax[1].spines['top'] top_side.set_visible(False) plt.savefig(figfile) print('Cutout written to ' + figfile) #import pdb; pdb.set_trace() avgim = np.sum(cutimarr, axis=2) / nind avgim *= instheight0 / np.max(avgim) medim = np.median(cutimarr, axis=2) # Make a single blank file at the end so you know it looped figfile = outdir figfile += '%s_%04d_%s.jpg' % (str(obj['id'][0]), i + 2, 'path') figfiles.append(figfile) matplotlib.use('Agg') if os.path.exists(figfile): os.remove(figfile) fig = plt.gcf() # get current graphics window fig.clf() # clear gskw = dict(width_ratios=[30, 1]) fig, ax = plt.subplots(ncols=2, nrows=1, gridspec_kw=gskw) fig.set_figheight(figheight) fig.set_figwidth(figwidth) ax[0].imshow(avgim, origin='lower', aspect='auto', interpolation='none', extent=(xr[0], xr[1], yr[0], yr[1]), vmin=vmin, vmax=vmax, cmap='viridis') # viridis, Greys, jet ax[0].plot(np.array([0, 0]), np.array([-0.066 * npix, 0.066 * npix]) * pixscale, c='white', alpha=0.7, zorder=1) ax[0].plot(np.array([-0.066 * npix, 0.066 * npix]) * pixscale, np.array([0, 0]), c='white', alpha=0.7, zorder=1) xmeas = np.array(xmeas) ymeas = np.array(ymeas) ax[0].plot((xmeas - hpix) * pixscale, (ymeas - hpix) * pixscale, c='r') #plt.scatter((xmeas-hpix)*pixscale,(ymeas-hpix)*pixscale,c='r',marker='+',s=30) ax[0].set_xlabel(r'$\Delta$ RA (arcsec)') ax[0].set_ylabel(r'$\Delta$ DEC (arcsec)') ax[0].set_xlim((xr[1], xr[0])) # sky -right ax[0].set_ylim(yr) ax[1].axis('off') plt.savefig(figfile) # Make four copies for j in np.arange(2, 11): #pathfile = figfile.replace('path1','path'+str(j)) pathfile = figfile.replace('%04d' % (i + 2), '%04d' % (i + 1 + j)) if os.path.exists(pathfile): os.remove(pathfile) shutil.copyfile(figfile, pathfile) figfiles.append(pathfile) # Make the animated gif animfile = outdir + str(objid) + '_cutouts.gif' if os.path.exists(animfile): os.remove(animfile) # put list of files in a separate file listfile = outdir + str(objid) + '_cutouts.lst' if os.path.exists(listfile): os.remove(listfile) dln.writelines(listfile, figfiles) delay = dln.scale(nind, [20, 1000], [20, 1]) delay = int(np.round(dln.limit(delay, 1, 20))) print('delay = ' + str(delay)) print('Creating animated gif ' + animfile) #ret = subprocess.run('convert -delay 100 '+figdir+str(objid)+'_*.jpg '+animfile,shell=True) #ret = subprocess.run('convert -delay 20 '+' '.join(figfiles)+' '+animfile,shell=True) ret = subprocess.run('convert @' + listfile + ' -delay ' + str(delay) + ' ' + animfile, shell=True) #import pdb; pdb.set_trace() dln.remove(figfiles)
goodwav = np.zeros((npix,len(goodap1)),float) for i in range(len(goodap1)): wav1 = idlines[goodap1[i]].func(x) diff = wav1-medwav0 meddiff = np.median(diff) goodwav[:,i] = wav1-meddiff medwav = np.median(goodwav,axis=1) # Second cut at good and bad apertures by comparing to median wave array sig = np.zeros(128,float)+999999.0 std = np.zeros(128,float)+999999.0 for i in range(128): if idlines[i].npoints>0: wav1 = idlines[i].func(x) std[i] = np.std(wav1-medwav) sig[i] = dln.mad(wav1-medwav) #goodap, = np.where((npoints>0) & (sig<1.0) & (std<1.0) & (npoints>0.5*mednpoints)) #badap, = np.where((npoints>0) & ((sig>=1.0) | (std>=1.0) | (npoints<=0.5*mednpoints))) goodap, = np.where((apinfo['npoints']>0) & (sig<1.0) & (std<1.0) & (apinfo['rms']<0.1) & (apinfo['npoints']>20)) badap, = np.where((apinfo['npoints']>0) & ((sig>=1.0) | (std>=1.0) | (apinfo['rms']>=0.1) | (apinfo['npoints']<=20))) nbad = len(badap) print(str(nbad)+' bad apertures to fix') apinfo['good'][goodap] = True #goodap, = np.where((npoints>0) & (sig<1.0) & (std<1.0) & (npoints>0.5*mednpoints) & (rms<0.08)) #badap, = np.where(rms > 0.08) #nbad = len(badap) #import pdb; pdb.set_trace()
def fix_pms(objectid): """ Correct the proper motions in the healpix object catalog.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' radeg = np.float64(180.00) / np.pi meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table') nmeas = len(meas) print(' '+str(nmeas)) mnra = np.median(meas['ra'].data) mndec = np.median(meas['dec'].data) lim = 20.0 # 50.0 gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) & (np.abs(meas['dec'].data-mndec)*3600 < lim)) ngd = len(gd) nbd = nmeas-ngd print('bad measurements '+str(nbd)) #if nbd==0: # return None meas = meas[gd] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='"+objectid+"'" chip = qc.query(sql=sql,fmt='table') ind3,ind4 = dln.match(chip['exposure'],meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm'])) if len(gdfwhm)==0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values') meas = meas[gdfwhm] raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec ra = np.array(meas['ra'],np.float64) ra -= np.mean(ra) ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec t = np.array(meas['mjd'].copy()) t -= np.mean(t) t /= 365.2425 # convert to year # Calculate robust slope try: #pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True) # LADfit pmra_ladcoef, absdev = dln.ladfit(t,ra) pmra_lad = pmra_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), ra) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmra_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True) #pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True) #pmra = pmra_coef[0] #pmraerr = pmra_coeferr[0] #radiff = ra-dln.poly(t,pmra_coef) radiff = ra-t*pmra_lad radiff -= np.median(radiff) rasig = dln.mad(radiff) # Reject outliers gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True) pmra = pmra_coef[0] pmraerr = pmra_coeferr[0] rasig = dln.mad(ra-dln.poly(t,pmra_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec dec = np.array(meas['dec'],np.float64) dec -= np.mean(dec) dec *= 3600*1e3 # convert to milli arcsec # Calculate robust slope try: #pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True) # LADfit pmdec_ladcoef, absdev = dln.ladfit(t,dec) pmdec_lad = pmdec_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), dec) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmdec_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True) #pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True) #pmdec = pmdec_coef[0] #pmdecerr = pmdec_coeferr[0] #decdiff = dec-dln.poly(t,pmdec_coef) decdiff = dec-t*pmdec_lad decdiff -= np.median(decdiff) decsig = dln.mad(decdiff) # Reject outliers gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True) pmdec = pmdec_coef[0] pmdecerr = pmdec_coeferr[0] decsig = dln.mad(dec-dln.poly(t,pmdec_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) deltamjd = np.max(meas['mjd'])-np.min(meas['mjd']) out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd]) #print(out[[0,2,3]]) #print(out[[5,7,8]]) #import pdb; pdb.set_trace() return out
def mkempirical(cube,order=0,coords=None,shape=None,rect=False,lookup=False): """ Take a star cube and collapse it to make an empirical PSF using median and outlier rejection. Parameters ---------- cube : numpy array Three-dimensional cube of star images (or residual images) of shape (Npix,Npix,Nstars). order : int, optional The order of the variations. 0-constant, 1-linear terms. If order=1, Then coords and shape must be input. coords : tuple, optional Two-element tuple of the X/Y coordinates of the stars. This is needed to generate the linear empirical model (order=1). shape : tuple, optional Two-element tuple giving the shape (Ny,Nx) of the image. This is needed to generate the linear empirical model (order=1). rect : boolean, optional Return a list of RectBivariateSpline functions rather than a numpy array. lookup : boolean, optional Parameter to indicate if this is a lookup table. If lookup=False, then the constant term is constrained to be non-negative. Default is False. Returns ------- epsf : numpy array The empirical PSF model If order=0, then this is just a 2D image. If order=1, then it will be a 3D cube (Npix,Npix,4) where the four terms are [constant, X-term, Y-term, X*Y-term]. If rect=True, then a list of RectBivariateSpline functions are returned. Example ------- epsf = mkempirical(cube,order=0) or epsf = mkempirical(cube,order=1,coords=coords,shape=im.shape) """ ny,nx,nstar = cube.shape npix = ny nhpix = ny//2 # Do outlier rejection in each pixel med = np.nanmedian(cube,axis=2) bad = ~np.isfinite(med) if np.sum(bad)>0: med[bad] = np.nanmedian(med) sig = dln.mad(cube,axis=2) bad = ~np.isfinite(sig) if np.sum(bad)>0: sig[bad] = np.nanmedian(sig) # Mask outlier points outliers = ((np.abs(cube-med.reshape((med.shape)+(-1,)))>3*sig.reshape((med.shape)+(-1,))) & np.isfinite(cube)) nbadstar = np.sum(outliers,axis=(0,1)) goodmask = ((np.abs(cube-med.reshape((med.shape)+(-1,)))<3*sig.reshape((med.shape)+(-1,))) & np.isfinite(cube)) # Now take the mean of the unmasked pixels macube = np.ma.array(cube,mask=~goodmask) medim = macube.mean(axis=2) medim = medim.data # Check how well each star fits the median goodpix = macube.count(axis=(0,1)) rms = np.sqrt(np.nansum((cube-medim.reshape((medim.shape)+(-1,)))**2,axis=(0,1))/goodpix) xx,yy = np.meshgrid(np.arange(npix)-nhpix,np.arange(npix)-nhpix) rr = np.sqrt(xx**2+yy**2) x = xx[0,:] y = yy[:,0] mask = (rr<=nhpix) # Constant if order==0: # Make sure it goes to zero at large radius medim *= mask # mask corners # Make sure values are positive if lookup==False: medim = np.maximum(medim,0.0) if rect: fpars = [RectBivariateSpline(y,x,medim)] else: fpars = medim # Linear elif order==1: if coords is None or shape is None: raise ValueError('Need coords and shape with order=1') pars = np.zeros((ny,nx,4),float) # scale coordinates to -1 to +1 xcen,ycen = coords relx,rely = models.relcoord(xcen,ycen,shape) # Loop over pixels and fit line to x/y for i in range(ny): for j in range(nx): data1 = cube[i,j,:] # maybe use a small maxiter pars1,perror1 = utils.poly2dfit(relx,rely,data1) pars[i,j,:] = pars1 # Make sure it goes to zero at large radius if rect: fpars = [] for i in range(4): # Make sure edges are zero on average for higher-order terms outer = np.median(pars[rr>nhpix*0.8,i]) pars[:,:,i] -= outer # Mask corners pars[:,:,i] *= mask # Each higher-order term must have ZERO total volume # Set up the spline function that we can use to do # the interpolation fpars.append(RectBivariateSpline(y,x,pars[:,:,i])) else: fpars = pars return fpars,nbadstar,rms
def mad(self): """ Calculate the MAD of the image data. Uses only unmasked data.""" if self.mask is not None: return dln.mad(self.data[~self.mask]) else: return dln.mad(self.data)
def getpsf(psf,image,cat,fitradius=None,lookup=False,lorder=0,method='qr',subnei=False, allcat=None,maxiter=10,minpercdiff=1.0,reject=False,maxrejiter=3,verbose=False): """ Fit PSF model to stars in an image with outlier rejection of badly-fit stars. Parameters ---------- psf : PSF object PSF object with initial parameters to use. image : CCDData object Image to use to fit PSF model to stars. cat : table Catalog with initial amp/x/y values for the stars to use to fit the PSF. fitradius : float, table The fitting radius. If none is input then the initial PSF FWHM will be used. lookup : boolean, optional Use an empirical lookup table. Default is False. lorder : int, optional The order of the spatial variations (0=constant, 1=linear). Default is 0. method : str, optional Method to use for solving the non-linear least squares problem: "qr", "svd", "cholesky", and "curve_fit". Default is "qr". subnei : boolean, optional Subtract stars neighboring the PSF stars. Default is False. allcat : table, optional Catalog of all objects in the image. This is needed for bad PSF star rejection. maxiter : int, optional Maximum number of iterations to allow. Only for methods "qr", "svd", and "cholesky". Default is 10. minpercdiff : float, optional Minimum percent change in the parameters to allow until the solution is considered converged and the iteration loop is stopped. Only for methods "qr" and "svd". Default is 1.0. reject : boolean, optional Reject PSF stars with high RMS values. Default is False. maxrejiter : int, boolean Maximum number of PSF star rejection iterations. Default is 3. verbose : boolean, optional Verbose output. Returns ------- newpsf : PSF object New PSF object with the best-fit model parameters. pars : numpy array Array of best-fit model parameters perror : numpy array Uncertainties in "pars". psfcat : table Table of best-fitting amp/xcen/ycen values for the PSF stars. Example ------- newpsf,pars,perror,psfcat = getpsf(psf,image,cat) """ t0 = time.time() print = utils.getprintfunc() # Get print function to be used locally, allows for easy logging # Fitting radius if fitradius is None: if type(psf)==models.PSFPenny: fitradius = psf.fwhm()*1.5 else: fitradius = psf.fwhm() # subnei but no allcat input if subnei and allcat is None: raise ValueError('allcat is needed for PSF neighbor star subtraction') if 'id' not in cat.colnames: cat['id'] = np.arange(len(cat))+1 psfcat = cat.copy() # Initializing output PSF star catalog dt = np.dtype([('id',int),('amp',float),('x',float),('y',float),('npix',int),('rms',float), ('chisq',float),('ixmin',int),('ixmax',int),('iymin',int),('iymax',int),('reject',int)]) outcat = np.zeros(len(cat),dtype=dt) outcat = Table(outcat) for n in ['id','x','y']: outcat[n] = cat[n] # Remove stars that are too close to the edge ny,nx = image.shape bd = (psfcat['x']<fitradius) | (psfcat['x']>(nx-1-fitradius)) | \ (psfcat['y']<fitradius) | (psfcat['y']>(ny-1-fitradius)) nbd = np.sum(bd) if nbd > 0: if verbose: print('Removing '+str(nbd)+' stars near the edge') psfcat = psfcat[~bd] # Generate an empirical image of the stars # and fit a model to it to get initial estimates if type(psf)!=models.PSFEmpirical: cube = starcube(psfcat,image,npix=psf.npix,fillvalue=np.nan) epsf,nbadstar,rms = mkempirical(cube,order=0) epsfim = CCDData(epsf,error=epsf.copy()*0+1,mask=~np.isfinite(epsf)) pars,perror,mparams = psf.fit(epsfim,pars=[1.0,psf.npix/2,psf.npix//2],allpars=True) initpar = mparams.copy() curpsf = psf.copy() curpsf.params = initpar if verbose: print('Initial estimate from empirical PSF fit = '+str(mparams)) else: curpsf = psf.copy() initpar = psf.params.copy() # Outlier rejection iterations nrejiter = 0 flag = 0 nrejstar = 100 fitrad = fitradius useimage = image.copy() while (flag==0): if verbose: print('--- Iteration '+str(nrejiter+1)+' ---') # Update the fitting radius if nrejiter>0: fitrad = curpsf.fwhm() if verbose: print(' Fitting radius = %5.3f' % (fitrad)) # Reject outliers if reject and nrejiter>0: medrms = np.median(pcat['rms']) sigrms = dln.mad(pcat['rms'].data) gd, = np.where(pcat['rms'] < medrms+3*sigrms) nrejstar = len(psfcat)-len(gd) if verbose: print(' RMS = %6.4f +/- %6.4f' % (medrms,sigrms)) print(' Threshold RMS = '+str(medrms+3*sigrms)) print(' Rejecting '+str(nrejstar)+' stars') if nrejstar>0: psfcat = psfcat[gd] # Subtract neighbors if nrejiter>0 and subnei: if verbose: print('Subtracting neighbors') # Find the neighbors in allcat # Fit the neighbors and PSF stars # Subtract neighbors from the image useimage = image.copy() # start with original image useimage = subtractnei(useimage,allcat,cat,curpsf) # Fitting the PSF to the stars #----------------------------- newpsf,pars,perror,pcat,pf = fitpsf(curpsf,useimage,psfcat,fitradius=fitrad,method=method, maxiter=maxiter,minpercdiff=minpercdiff,verbose=verbose) # Add information into the output catalog ind1,ind2 = dln.match(outcat['id'],pcat['id']) outcat['reject'] = 1 for n in pcat.columns: outcat[n][ind1] = pcat[n][ind2] outcat['reject'][ind1] = 0 # Compare PSF parameters if type(newpsf)!=models.PSFEmpirical: pardiff = newpsf.params-curpsf.params else: pardiff = newpsf._data-curpsf._data sumpardiff = np.sum(np.abs(pardiff)) curpsf = newpsf.copy() # Stopping criteria if reject is False or sumpardiff<0.05 or nrejiter>=maxrejiter or nrejstar==0: flag=1 if subnei is True and nrejiter==0: flag=0 # iterate at least once with neighbor subtraction nrejiter += 1 # Generate an empirical look-up table of corrections if lookup: if verbose: print('Making empirical lookup table with order='+str(lorder)) pf.mklookup(lorder) # Fit the stars again and get new RMS values xdata = np.arange(pf.ntotpix) out = pf.model(xdata,*pf.psf.params) newpsf = pf.psf.copy() # Update information in the output catalog ind1,ind2 = dln.match(outcat['id'],pcat['id']) outcat['reject'] = 1 outcat['reject'][ind1] = 0 outcat['amp'][ind1] = pf.staramp[ind2] outcat['x'][ind1] = pf.starxcen[ind2] outcat['y'][ind1] = pf.starycen[ind2] outcat['rms'][ind1] = pf.starrms[ind2] outcat['chisq'][ind1] = pf.starchisq[ind2] if verbose: print('Median RMS: '+str(np.median(pf.starrms))) if verbose: print('dt = %.2f sec' % (time.time()-t0)) return newpsf, pars, perror, outcat
def gausspeakfit(spec, pix0=None, estsig=5, sigma=None, func=gaussbin): """ Return integrated-Gaussian centers near input pixel center Parameters ---------- spec : float array Data spectrum array. pix0 : float or integer (scalar or array) Initial pixel guess. estsig : float Initial guess for window width=5*estsig (default=5). sigma : float array, optional Uncertainty array (default=None). func : function, optional User-supplied function to use to fit (default=gaussbin). Returns ------- pars : float array Array of best-fitting parameters (height, center, sigma, yoffset) perr : float array Array of uncertainties of the parameters. Examples -------- pars,perr = gausspeakfit(spec,10,sigma=specerr) """ # No initial guess input, use maximum value if pix0 is None: pix0 = spec.argmax() medspec = np.median(spec) sigspec = dln.mad(spec - medspec) dx = 1 x = np.arange(len(spec)) xwid = 5 xlo0 = pix0 - xwid xhi0 = pix0 + xwid + 1 xx0 = x[xlo0:xhi0] # Get quantitative estimates of height, center, sigma flux = spec[xlo0:xhi0] - medspec flux -= np.median(flux) # put the median at zero flux = np.maximum(0, flux) # don't want negative pixels ht0 = np.max(flux) totflux = np.sum(flux) # Gaussian area is A = ht*wid*sqrt(2*pi) sigma0 = np.maximum((totflux * dx) / (ht0 * np.sqrt(2 * np.pi)), 0.01) cen0 = np.sum(flux * xx0) / totflux cen0 = np.minimum(np.maximum((pix0 - dx * 0.5), cen0), (pix0 + dx * 0.5)) # constrain the center # Use linear-least squares to calculate height and sigma psf1 = np.exp(-0.5 * (xx0 - cen0)**2 / sigma0**2) # normalized Gaussian wtht1 = np.sum(flux * psf1) / np.sum(psf1 * psf1) # linear least squares # Second iteration sigma1 = (totflux * dx) / (wtht1 * np.sqrt(2 * np.pi)) psf2 = np.exp(-0.5 * (xx0 - cen0)**2 / sigma1**2) # normalized Gaussian wtht2 = np.sum(flux * psf2) / np.sum(psf2 * psf2) # Now get more pixels to fit if necessary npixwide = int(np.maximum(np.ceil((2 * sigma1) / dx), 5)) xlo = int(np.round(cen0)) - npixwide xhi = int(np.round(cen0)) + npixwide + 1 xx = x[xlo:xhi] y = spec[xlo:xhi] yerr = sigma[xlo:xhi] # Bounds initpar = [wtht2, cen0, sigma1, medspec] lbounds = [ 0.5 * ht0, initpar[1] - 1, 0.2, initpar[3] - np.maximum(3 * sigspec, 0.3 * np.abs(initpar[3])) ] ubounds = [ 3.0 * ht0, initpar[1] + 1, 5.0, initpar[3] + np.maximum(3 * sigspec, 0.3 * np.abs(initpar[3])) ] bounds = (lbounds, ubounds) # Sometimes curve_fit hits the maximum number fo function evaluations and crashes try: pars, cov = curve_fit(func, xx, y, p0=initpar, sigma=yerr, bounds=bounds, maxfev=1000) perr = np.sqrt(np.diag(cov)) except: return None, None return pars, perr
def peakfit(spec, sigma=None, pix0=None): """ Find lines in a spectrum fit Gaussians to them. Parameters ---------- spec : float array Data spectrum array. sigma : float array, optional Uncertainty array (default=None). pix0 : float or integer (scalar or array), optional Initial pixel guess. Returns ------- pars : numpy structured array Table of best-fitting parameters and uncertainties. Examples -------- pars = peakfit(spec,sigma=specerr) """ if np.ndim(spec) > 1: raise ValueError('Spec must be 1-D') # X array npix = len(spec) x = np.arange(npix) smspec = median_filter(spec, 101, mode='nearest') sigspec = dln.mad(spec - smspec) # Find the peaks if pix0 is None: maxind, = argrelextrema(spec - smspec, np.greater) # maxima # sigma cut on the flux gd, = np.where((spec - smspec)[maxind] > 4 * sigspec) if len(gd) == 0: print('No peaks found') return pix0 = maxind[gd] pix0 = np.atleast_1d(pix0) npeaks = len(pix0) # Initialize the output table dtype = np.dtype([('num', int), ('pix0', int), ('pars', np.float64, 4), ('perr', np.float64, 4), ('success', bool)]) out = np.zeros(npeaks, dtype=dtype) out['num'] = np.arange(npeaks) out['pix0'] = pix0 # Initialize the residuals spectrum resid = spec.copy() # Loop over the peaks and fit them for i in range(npeaks): # Run gausspeakfit() on the residual pars, perr = gausspeakfit(resid, pix0=pix0[i], sigma=sigma) if pars is not None: # Get model and subtract from residuals xlo = np.maximum(0, int(pars[1] - 5 * pars[2])) xhi = np.minimum(npix, int(pars[1] + 5 * pars[2])) peakmodel = gaussbin(x[xlo:xhi], pars[0], pars[1], pars[2]) # leave yoffset in resid[xlo:xhi] -= peakmodel # Stuff results in output table out['pars'][i] = pars out['perr'][i] = perr out['success'][i] = True else: out['pars'][i] = np.nan out['perr'][i] = np.nan return out
def orig_salariscna(abund, mc=False): ''' Calculate the Salaris corrected [Fe/H] according to Salaris et al. 1993 with Piersanti et al. 2007 and Asplund et al. 2009. Also C and N have been added to the alpha elements and Ne has been excluded. Inputs: ------ abund: [9x2 array] first column is [Fe/H],[C/Fe],[N/Fe],[O/Fe],[Mg/Fe],[Si/Fe],[S/Fe],[Ca/Fe],[Ti/Fe] and second column is the errors Output: ------ calc_salfeh: Salaris corrected metallicity calc_salfeh_err: error in corrected metallicity ''' ### Salaris coefficients # (atomic_wgts/hydrogen_wgt) = (C,N,O,Mg,Si,S,Ca,Ti)/H asplund = np.array([8.43, 7.83, 8.69, 7.60, 7.51, 7.12, 6.34, 4.95]) mass_ratio = np.array([ 12.011, 14.007, 15.999, 24.305, 28.085, 32.06, 40.078, 47.867 ]) / 1.008 #IUPAC # with Ne # (atomic_wgts/hydrogen_wgt) = (C,N,O,Ne,Mg,Si,S,Ca,Ti)/H #asplund = np.array([8.43,7.83,8.69,7.84,7.60,7.51,7.12,6.34,4.95]) #mass_ratio = np.array([12.011,14.007,15.999,20.1797,24.305,28.085,32.06,40.078,47.867])/1.008 #IUPAC ZX_sol = 0.0181 # (Z/X) Asplund et al. 2009 XZ_k = np.multiply(10**(asplund - 12.0), mass_ratio / ZX_sol) sal_a = np.sum(XZ_k) sal_b = 1 - sal_a ### Alpha+C+N wgts = asplund / np.sum(asplund) ### Replace bad values with solar for i in range(len(abund[:, 0])): if abund[i, 0] < -10. or abund[i, 0] > 10. or np.isfinite( abund[i, 0]) == False: abund[i, 0] = 0.0 if abund[i, 1] < -10. or abund[i, 1] > 10. or np.isfinite( abund[i, 1]) == False: abund[i, 1] = 0.0 feh = abund[0, 0] feh_err = abund[0, 1] cnalpha = abund[1:, 0] cnafe = np.log10(np.sum(np.multiply(10**cnalpha, wgts))) salfeh = feh + np.log10(sal_a * 10**(cnafe) + sal_b) ### MC for Salaris Correction if mc: cnalpha_err = abund[1:, 1] nsamples = 1000 salfehdist = 999999.0 * np.ones(nsamples) noisyfeh = np.random.normal(feh, feh_err, nsamples) for i in range(nsamples): noisycnalpha = 999999.0 * np.ones(len(cnalpha)) for j in range(len(cnalpha)): noisycnalpha[j] = np.random.normal(cnalpha[j], cnalpha_err[j]) cnafe = np.log10(np.sum(np.multiply(10**noisycnalpha, wgts))) salfehdist[i] = noisyfeh[i] + np.log10(sal_a * 10**(cnafe) + sal_b) calc_salfeh_err = dln.mad(salfehdist) return salfeh, calc_salfeh_err return salfeh
def normalize(self, ncorder=6, perclevel=0.95): """ Normalize the spectrum. Parameters ---------- ncorder : float, optional Polynomial order to use for the continuum fitting. The default is 6. perclevel : float, optional Percent level (1.0 for 100%) to use for the continuum value in large bins. Default is 0.95. Returns ------- The flux and err arrays will normalized (divided) by the continuum and the continuum saved in cont. The normalized property is set to True. Examples -------- spec.normalize() """ self._flux = self.flux # Save the original #nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel) binsize = 0.10 perclevel = 90.0 wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D err = self.err.copy().reshape(self.npix, self.norder) # make 2D mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D cont = err.copy() * 0.0 + 1 for o in range(self.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig) if nbd > 0: m[bd] = True gdmask = (y > 0) & (m == False ) # need positive fluxes and no mask set # Bin the data points xr = [np.nanmin(x), np.nanmax(x)] bins = np.ceil((xr[1] - xr[0]) / binsize) + 1 ybin, bin_edges, binnumber = bindata.binned_statistic( x[gdmask], y[gdmask], statistic='percentile', percentile=perclevel, bins=bins, range=None) xbin = bin_edges[0:-1] + 0.5 * binsize # Interpolate to full grid fnt = np.isfinite(ybin) cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True) cont1 *= medy flux[:, o] /= cont1 err[:, o] /= cont1 cont[:, o] = cont1 # Flatten to 1D if norder=1 if self.norder == 1: flux = flux.flatten() err = err.flatten() cont = cont.flatten() # Stuff back in self.flux = flux self.err = err self.cont = cont self.normalized = True return