def w2p(dispersion,w,extrapolate=True): """ Convert wavelength values to pixels for a "dispersion solution". Parameters ---------- dispersion : 1D array The dispersion solution. This is basically just a 1D array of monotonically increasing (or decreasing) wavelengths. w : array Array of wavelength values to convert to pixels. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. This is True by default. Returns ------- x : array Array of converted pixel values. Examples -------- x = w2p(disp,w) """ x = interp1d(dispersion,np.arange(len(dispersion)),kind='cubic',bounds_error=False,fill_value=(np.nan,np.nan),assume_sorted=False)(w) # Need to extrapolate if ((np.min(w)<np.min(dispersion)) | (np.max(w)>np.max(dispersion))) & (extrapolate is True): win = dispersion xin = np.arange(len(dispersion)) si = np.argsort(win) win = win[si] xin = xin[si] npix = len(win) # At the beginning if (np.min(w)<np.min(dispersion)): coef1 = dln.poly_fit(win[0:10], xin[0:10], 2) bd1, nbd1 = dln.where(w < np.min(dispersion)) x[bd1] = dln.poly(w[bd1],coef1) # At the end if (np.max(w)>np.max(dispersion)): coef2 = dln.poly_fit(win[npix-10:], xin[npix-10:], 2) bd2, nbd2 = dln.where(w > np.max(dispersion)) x[bd2] = dln.poly(w[bd2],coef2) return x
def p2w(dispersion,x,extrapolate=True): """ Convert pixel values to wavelengths for a "dispersion solution". Parameters ---------- dispersion : 1D array The dispersion solution. This is basically just a 1D array of monotonically increasing (or decreasing) wavelengths. x : array Array of pixel values to convert to wavelengths. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. This is True by default. Returns ------- w : array Array of converted wavelengths. Examples -------- w = p2w(disp,x) """ npix = len(dispersion) w = interp1d(np.arange(len(dispersion)),dispersion,kind='cubic',bounds_error=False,fill_value=(np.nan,np.nan),assume_sorted=False)(x) # Need to extrapolate if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True): xin = np.arange(npix) win = dispersion # At the beginning if (np.min(x)<0): coef1 = dln.poly_fit(xin[0:10], win[0:10], 2) bd1, nbd1 = dln.where(x < 0) w[bd1] = dln.poly(x[bd1],coef1) # At the end if (np.max(x)>(npix-1)): coef2 = dln.poly_fit(xin[npix-10:], win[npix-10:], 2) bd2, nbd2 = dln.where(x > (npix-1)) w[bd2] = dln.poly(x[bd2],coef2) return w
def maskdiscrepant(spec,model,nsig=10,verbose=False,logger=None): """ Mask pixels that are discrepant when compared to a model. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() my = mflux[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-my) bd,nbd = dln.where( np.abs(y-my) > nsig*sig ) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' discrepant pixels') return spec2
def maskoutliers(spec,nsig=5,verbose=False,logger=None): """ Mask large positive outliers and negative flux pixels in the spectrum. """ if logger is None: logger = dln.basiclogger() spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix,spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix,spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix,spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix,spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:,o].copy() x = (w-np.median(w))/(np.max(w*0.5)-np.min(w*0.5)) # -1 to +1 y = flux[:,o].copy() m = mask[:,o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x,y,2,robust=True) sig = dln.mad(y-dln.poly(x,coef)) bd,nbd = dln.where( ((y-dln.poly(x,coef)) > nsig*sig) | (y<0)) totnbd += nbd if nbd>0: flux[bd,o] = dln.poly(x[bd],coef)*medy err[bd,o] = 1e30 mask[bd,o] = True # Flatten to 1D if norder=1 if spec2.norder==1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: logger.info('Masked '+str(totnbd)+' outlier or negative pixels') return spec2
def continuum(spec, norder=6, perclevel=90.0, binsize=0.1, interp=True): """ Measure the continuum of a spectrum. Parameters ---------- spec : Spec1D object A spectrum object. This at least needs to have a FLUX and WAVE attribute. norder : float, optional Polynomial order to use for the continuum fitting. The default is 6. perclevel : float, optional Percent level to use for the continuum value in large bins. Default is 90. binsize : float, optional Fraction of the wavelength range (scaled to -1 to +1) to bin. Default is 0.1. interp : bool, optional Use interpolation of the binned values instead of a polynomial fit. Default is True. Returns ------- cont : numpy array The continuum array, in the same shape as the input flux. Examples -------- .. code-block:: python cont = continuum(spec) """ wave = spec.wave.copy().reshape(spec.npix, spec.norder) # make 2D flux = spec.flux.copy().reshape(spec.npix, spec.norder) # make 2D err = spec.err.copy().reshape(spec.npix, spec.norder) # make 2D mask = spec.mask.copy().reshape(spec.npix, spec.norder) # make 2D cont = err.copy() * 0.0 + 1 for o in range(spec.norder): w = wave[:, o].copy() wr = [np.min(w), np.max(w)] x = (w - np.mean(wr)) / dln.valrange(wr) * 2 # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x[~m], y[~m], 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig) if nbd > 0: m[bd] = True gdmask = (y > 0) & (m == False) # need positive fluxes and no mask set if np.sum(gdmask) == 0: continue # Bin the data points xr = [np.nanmin(x), np.nanmax(x)] bins = int(np.ceil((xr[1] - xr[0]) / binsize)) ybin, bin_edges, binnumber = bindata.binned_statistic( x[gdmask], y[gdmask], statistic='percentile', percentile=perclevel, bins=bins, range=None) xbin = bin_edges[0:-1] + 0.5 * binsize # Interpolate to full grid if interp is True: fnt = np.isfinite(ybin) cont1 = dln.interp(xbin[fnt], ybin[fnt], x, kind='quadratic', extrapolate=True, exporder=1) else: coef = dln.poly_fit(xbin, ybin, norder) cont1 = dln.poly(x, coef) cont1 *= medy cont[:, o] = cont1 # Flatten to 1D if norder=1 if spec.norder == 1: cont = cont.flatten() return cont
def apstar(filename,badval=20735): """ Read an SDSS APOGEE apStar spectrum. Parameters ---------- filename : string The name of the spectrum file to load. Returns ------- spec : Spec1D object The spectrum as a Spec1D object. Examples -------- spec = apstar('spec.fits') """ base, ext = os.path.splitext(os.path.basename(filename)) # APOGEE apStar, combined spectrum if (base.find("apStar") > -1) | (base.find("asStar") > -1): # HISTORY APSTAR: HDU0 = Header only # HISTORY APSTAR: All image extensions have: # HISTORY APSTAR: row 1: combined spectrum with individual pixel weighting # HISTORY APSTAR: row 2: combined spectrum with global weighting # HISTORY APSTAR: row 3-nvisits+2: individual resampled visit spectra # HISTORY APSTAR: unless nvisits=1, which only have a single row # HISTORY APSTAR: All spectra shifted to rest (vacuum) wavelength scale # HISTORY APSTAR: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU2 - Error (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU3 - Flag mask: # HISTORY APSTAR: row 1: bitwise OR of all visits # HISTORY APSTAR: row 2: bitwise AND of all visits # HISTORY APSTAR: row 3-nvisits+2: individual visit masks # HISTORY APSTAR: HDU4 - Sky (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU5 - Sky Error (10^-17 ergs/s/cm^2/Ang) # HISTORY APSTAR: HDU6 - Telluric # HISTORY APSTAR: HDU7 - Telluric Error # HISTORY APSTAR: HDU8 - LSF coefficients # HISTORY APSTAR: HDU9 - RV and CCF structure # Get number of extensions hdulist = fits.open(filename) nhdu = len(hdulist) hdulist.close() # Spectrum, error, sky, skyerr are in units of 1e-17 # these are 2D arrays with [Nvisit+2,Npix] # the first two are combined and the rest are the individual spectra head1 = fits.getheader(filename,1) w0 = np.float64(head1["CRVAL1"]) dw = np.float64(head1["CDELT1"]) nw = head1["NAXIS1"] wave = 10**(np.arange(nw)*dw+w0) # flux, err, sky, skyerr are in units of 1e-17 flux = fits.getdata(filename,1).T * 1e-17 lsfcoef = fits.getdata(filename,8).T spec = Spec1D(flux,wave=wave,lsfpars=lsfcoef,lsftype='Gauss-Hermite',lsfxtype='Pixels') spec.filename = filename spec.sptype = "apStar" spec.waveregime = "NIR" spec.instrument = "APOGEE" spec.head = fits.getheader(filename,0) spec.err = fits.getdata(filename,2).T * 1e-17 #bad = (spec.err<=0) # fix bad error values #if np.sum(bad) > 0: # spec.err[bad] = 1e30 spec.bitmask = fits.getdata(filename,3) spec.sky = fits.getdata(filename,4).T * 1e-17 spec.skyerr = fits.getdata(filename,5).T * 1e-17 spec.telluric = fits.getdata(filename,6).T spec.telerr = fits.getdata(filename,7).T spec.lsf = fits.getdata(filename,8).T # Create the bad pixel mask # "bad" pixels: # flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY', # 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF',''] # badflag = [1,1,1,1,1,1,1,1, # 0,0,0,0,0,0,1,0] mask = (np.bitwise_and(spec.bitmask,badval)!=0) | (np.isfinite(spec.flux)==False) # Extra masking for bright skylines x = np.arange(spec.npix) nsky = 4 medsky = median_filter(spec.sky,201,mode='reflect') medcoef = dln.poly_fit(x,medsky/np.median(medsky),2) medsky2 = dln.poly(x,medcoef)*np.median(medsky) skymask1 = (sky>nsky*medsky2) # pixels Nsig above median sky mask[:,i] = np.logical_or(mask[:,i],skymask1) # OR combine spec.mask = mask # Fix NaN or bad pixels pixels bd,nbd = dln.where( (np.isfinite(spec.flux[:,i])==False) | (spec.err[:,i] <= 0.0) ) if nbd>0: spec.flux[bd] = 0.0 spec.err[bd] = 1e30 spec.mask[bd] = True if nhdu>=9: spec.meta = fits.getdata(filename,9) # meta-data spec.snr = spec.head["SNR"] if base.find("apStar") > -1: spec.observatory = 'apo' else: spec.observatory = 'lco' spec.wavevac = True return spec
def normalize(self, ncorder=6, perclevel=0.95): """ Normalize the spectrum. Parameters ---------- ncorder : float, optional Polynomial order to use for the continuum fitting. The default is 6. perclevel : float, optional Percent level (1.0 for 100%) to use for the continuum value in large bins. Default is 0.95. Returns ------- The flux and err arrays will normalized (divided) by the continuum and the continuum saved in cont. The normalized property is set to True. Examples -------- spec.normalize() """ self._flux = self.flux # Save the original #nspec, cont, masked = normspec(self,ncorder=ncorder,perclevel=perclevel) binsize = 0.10 perclevel = 90.0 wave = self.wave.copy().reshape(self.npix, self.norder) # make 2D flux = self.flux.copy().reshape(self.npix, self.norder) # make 2D err = self.err.copy().reshape(self.npix, self.norder) # make 2D mask = self.mask.copy().reshape(self.npix, self.norder) # make 2D cont = err.copy() * 0.0 + 1 for o in range(self.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where((y - dln.poly(x, coef)) > 5 * sig) if nbd > 0: m[bd] = True gdmask = (y > 0) & (m == False ) # need positive fluxes and no mask set # Bin the data points xr = [np.nanmin(x), np.nanmax(x)] bins = np.ceil((xr[1] - xr[0]) / binsize) + 1 ybin, bin_edges, binnumber = bindata.binned_statistic( x[gdmask], y[gdmask], statistic='percentile', percentile=perclevel, bins=bins, range=None) xbin = bin_edges[0:-1] + 0.5 * binsize # Interpolate to full grid fnt = np.isfinite(ybin) cont1 = dln.interp(xbin[fnt], ybin[fnt], x, extrapolate=True) cont1 *= medy flux[:, o] /= cont1 err[:, o] /= cont1 cont[:, o] = cont1 # Flatten to 1D if norder=1 if self.norder == 1: flux = flux.flatten() err = err.flatten() cont = cont.flatten() # Stuff back in self.flux = flux self.err = err self.cont = cont self.normalized = True return
for j in range(ntlines): bestline = np.argmin(np.abs(tlinecat['wave'][j] - linecat['wave0'])) maxdiff = 0.5 #1.0 if np.abs(tlinecat['wave'][j] - linecat['wave0'][bestline]) < maxdiff: match[j] = bestline dwave[j] = tlinecat['wave'][j] - linecat['wave0'][bestline] gmatch, = np.where(match > -1) linecat['wave'][match[gmatch]] = tlinecat['wave'][gmatch] linecat['match'][match[gmatch]] = 1 # Perform an initial fit gline1, = np.where(linecat['match'] == 1) meanwave = np.mean(linecat['wave'][gline1]) coef1 = dln.poly_fit(linecat['center'][gline1], linecat['wave'][gline1], 3) dwave = dln.poly(linecat['center'][gline1], coef1) - linecat['wave'][gline1] sigwave = dln.mad(dwave) if sigwave > 0.1: print('Sigma high. Trying higher order') coef1 = dln.poly_fit(linecat['center'][gline1], linecat['wave'][gline1], 4) dwave = dln.poly(linecat['center'][gline1], coef1) - linecat['wave'][gline1] sigwave = dln.mad(dwave) print('Sigma of initial fit %6.4f A' % sigwave) bdline, = np.where(np.abs(dwave) > 3 * sigwave) if len(bdline) > 0: linecat['outlier'][gline1[bdline]] = True print('throwing out ' + str(len(bdline)) + ' outlier lines')
def apvisit(filename): """ Read a SDSS APOGEE apVisit spectrum. Parameters ---------- filename : string The name of the spectrum file to load. Returns ------- spec : Spec1D object The spectrum as a Spec1D object. Examples -------- spec = apvisit('spec.fits') """ base, ext = os.path.splitext(os.path.basename(filename)) # Get number of extensions hdulist = fits.open(filename) nhdu = len(hdulist) hdulist.close() # Check that this has the right format validfile = False if (base.find("apVisit") == -1) | (base.find("asVisit") == -1): if nhdu >= 10: gd, = np.where( np.char.array(hdulist[0].header['HISTORY']).astype(str).find( 'AP1DVISIT') > -1) if len(gd) > 0: validfile = True if validfile is False: return None # APOGEE apVisit, visit-level spectrum # HISTORY AP1DVISIT: HDU0 = Header only # HISTORY AP1DVISIT: HDU1 - Flux (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU2 - Error (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU3 - Flag mask (bitwise OR combined) # HISTORY AP1DVISIT: HDU4 - Wavelength (Ang) # HISTORY AP1DVISIT: HDU5 - Sky (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU6 - Sky Error (10^-17 ergs/s/cm^2/Ang) # HISTORY AP1DVISIT: HDU7 - Telluric # HISTORY AP1DVISIT: HDU8 - Telluric Error # HISTORY AP1DVISIT: HDU9 - Wavelength coefficients # HISTORY AP1DVISIT: HDU10 - LSF coefficients # HISTORY AP1DVISIT: HDU11 - RV catalog # flux, err, sky, skyerr are in units of 1e-17 flux = fits.getdata(filename, 1).T * 1e-17 # [Npix,Norder] wave = fits.getdata(filename, 4).T lsfcoef = fits.getdata(filename, 10).T spec = Spec1D(flux, wave=wave, lsfpars=lsfcoef, lsftype='Gauss-Hermite', lsfxtype='Pixels') spec.reader = 'apvisit' spec.filename = filename spec.sptype = "apVisit" spec.waveregime = "NIR" spec.instrument = "APOGEE" spec.head = fits.getheader(filename, 0) spec.err = fits.getdata(filename, 2).T * 1e-17 # [Npix,Norder] #bad = (spec.err<=0) # fix bad error values #if np.sum(bad) > 0: # spec.err[bad] = 1e30 spec.bitmask = fits.getdata(filename, 3).T spec.sky = fits.getdata(filename, 5).T * 1e-17 spec.skyerr = fits.getdata(filename, 6).T * 1e-17 spec.telluric = fits.getdata(filename, 7).T spec.telerr = fits.getdata(filename, 8).T spec.wcoef = fits.getdata(filename, 9).T # Create the bad pixel mask # "bad" pixels: # flag = ['BADPIX','CRPIX','SATPIX','UNFIXABLE','BADDARK','BADFLAT','BADERR','NOSKY', # 'LITTROW_GHOST','PERSIST_HIGH','PERSIST_MED','PERSIST_LOW','SIG_SKYLINE','SIG_TELLURIC','NOT_ENOUGH_PSF',''] # badflag = [1,1,1,1,1,1,1,1, # 0,0,0,0,0,0,1,0] mask = (np.bitwise_and(spec.bitmask, 16639) != 0) | (np.isfinite(spec.flux) == False) # Extra masking for bright skylines x = np.arange(spec.npix) nsky = 4 for i in range(spec.norder): sky = spec.sky[:, i] medsky = median_filter(sky, 201, mode='reflect') medcoef = dln.poly_fit(x, medsky / np.median(medsky), 2) medsky2 = dln.poly(x, medcoef) * np.median(medsky) skymask1 = (sky > nsky * medsky2) # pixels Nsig above median sky mask[:, i] = np.logical_or(mask[:, i], skymask1) # OR combine spec.mask = mask # Fix NaN pixels for i in range(spec.norder): bd, nbd = dln.where((np.isfinite(spec.flux[:, i]) == False) | (spec.err[:, i] <= 0)) if nbd > 0: spec.flux[bd, i] = 0.0 spec.err[bd, i] = 1e30 spec.mask[bd, i] = True if (nhdu >= 11): spec.meta = fits.getdata(filename, 11) # catalog of RV and other meta-data # Spectrum, error, sky, skyerr are in units of 1e-17 spec.snr = spec.head["SNR"] if base.find("apVisit") > -1: spec.observatory = 'apo' else: spec.observatory = 'lco' spec.wavevac = True return spec
def sigma(self,x=None,xtype='pixels',order=0,extrapolate=True): """ Return the Gaussian sigma at specified locations. The sigma will be in units of lsf.xtype. Parameters ---------- x : array, optional The x-values for which to return the Gaussian sigma values. xtype : string, optional The type of x-value input, either 'wave' or 'pixels'. Default is 'pixels'. order : int, optional The order to use if there are multiple orders. The default is 0. extrapolate : bool, optional Extrapolate beyond the dispersion solution, if necessary. True by default. Returns ------- sigma : array The array of Gaussian sigma values. Examples -------- sigma = lsf.sigma([100,200]) """ # The sigma will be returned in units given in lsf.xtype if self._sigma is not None: _sigma = self._sigma if self.ndim==2: _sigma = self._sigma[:,order] if x is None: return _sigma else: # Wavelength input if xtype.lower().find('wave') > -1: x0 = np.array(x).copy() # backup x = self.wave2pix(x0,order=order) # convert to pixels # Integer, just return the values if( type(x)==int) | (np.array(x).dtype.kind=='i'): return _sigma[x] # Floats, interpolate else: sig = interp1d(np.arange(len(_sigma)),_sigma,kind='cubic',bounds_error=False, fill_value=(np.nan,np.nan),assume_sorted=True)(x) # Extrapolate npix = self.npix if ((np.min(x)<0) | (np.max(x)>(npix-1))) & (extrapolate is True): xin = np.arange(npix) # At the beginning if (np.min(x)<0): coef1 = dln.poly_fit(xin[0:10], _sigma[0:10], 2) bd1, nbd1 = dln.where(x <0) sig[bd1] = dln.poly(x[bd1],coef1) # At the end if (np.max(x)>(npix-1)): coef2 = dln.poly_fit(xin[npix-10:], _sigma[npix-10:], 2) bd2, nbd2 = dln.where(x > (npix-1)) sig[bd2] = dln.poly(x[bd2],coef2) return sig # Need to calculate else: if x is None: x = np.arange(self.npix) if self.pars is None: raise Exception("No LSF parameters") # Get parameters pars = self.pars if self.ndim==2: pars=self.pars[:,order] # Pixels input if xtype.lower().find('pix') > -1: # Pixel LSF parameters if self.xtype.lower().find('pix') > -1: return np.polyval(pars[::-1],x) # Wave LSF parameters else: w = self.pix2wave(x,order=order) return np.polyval(pars[::-1],w) # Wavelengths input else: # Wavelength LSF parameters if self.xtype.lower().find('wave') > -1: return np.polyval(pars[::-1],x) # Pixel LSF parameters else: x0 = np.array(x).copy() x = self.wave2pix(x0,order=order) return np.polyval(pars[::-1],x)
def maskdiscrepant(spec, model, nsig=10, verbose=False): """ Mask pixels that are discrepant when compared to a model. Parameters ---------- spec : Spec1D object Observed spectrum for which to mask discrepant values. model : Spec1D object Reference/model spectrum to use to find discrepant values. nsig : int, optional Number of standard deviations to use for the discrepant values. Default is 10.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with discrepant values masked. Example ------- .. code-block:: python spec = maskdiscrepant(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D mflux = model.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() my = mflux[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy my /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - my) bd, nbd = dln.where(np.abs(y - my) > nsig * sig) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' discrepant pixels') return spec2
def maskoutliers(spec, nsig=5, verbose=False): """ Mask large positive outliers and negative flux pixels in the spectrum. Parameters ---------- spec : Spec1D object Observed spectrum to mask outliers on. nsig : int, optional Number of standard deviations to use for the outlier rejection. Default is 5.0. verbose : boolean, optional Verbose output. Default is False. Returns ------- spec2 : Spec1D object Spectrum with outliers masked. Example ------- .. code-block:: python spec = maskoutliers(spec,nsig=5) """ print = getprintfunc( ) # Get print function to be used locally, allows for easy logging spec2 = spec.copy() wave = spec2.wave.copy().reshape(spec2.npix, spec2.norder) # make 2D flux = spec2.flux.copy().reshape(spec2.npix, spec2.norder) # make 2D err = spec2.err.copy().reshape(spec2.npix, spec2.norder) # make 2D mask = spec2.mask.copy().reshape(spec2.npix, spec2.norder) # make 2D totnbd = 0 for o in range(spec2.norder): w = wave[:, o].copy() x = (w - np.median(w)) / (np.max(w * 0.5) - np.min(w * 0.5) ) # -1 to +1 y = flux[:, o].copy() m = mask[:, o].copy() # Divide by median medy = np.nanmedian(y) if medy <= 0.0: medy = 1.0 y /= medy # Perform sigma clipping out large positive outliers coef = dln.poly_fit(x, y, 2, robust=True) sig = dln.mad(y - dln.poly(x, coef)) bd, nbd = dln.where(((y - dln.poly(x, coef)) > nsig * sig) | (y < 0)) totnbd += nbd if nbd > 0: flux[bd, o] = dln.poly(x[bd], coef) * medy err[bd, o] = 1e30 mask[bd, o] = True # Flatten to 1D if norder=1 if spec2.norder == 1: flux = flux.flatten() err = err.flatten() mask = mask.flatten() # Stuff back in spec2.flux = flux spec2.err = err spec2.mask = mask if verbose is True: print('Masked ' + str(totnbd) + ' outlier or negative pixels') return spec2
def fix_pms(objectid): """ Correct the proper motions in the healpix object catalog.""" t00 = time.time() hostname = socket.gethostname() host = hostname.split('.')[0] version = 'v3' radeg = np.float64(180.00) / np.pi meas = qc.query(sql="select * from nsc_dr2.meas where objectid='"+objectid+"'",fmt='table') nmeas = len(meas) print(' '+str(nmeas)) mnra = np.median(meas['ra'].data) mndec = np.median(meas['dec'].data) lim = 20.0 # 50.0 gd, = np.where( (np.abs(meas['ra'].data-mnra)/np.cos(np.deg2rad(mndec))*3600 < lim) & (np.abs(meas['dec'].data-mndec)*3600 < lim)) ngd = len(gd) nbd = nmeas-ngd print('bad measurements '+str(nbd)) #if nbd==0: # return None meas = meas[gd] # Make cut on FWHM # maybe only use values for 0.5*fwhm_chip to 1.5*fwhm_chip sql = "select chip.* from nsc_dr2.chip as chip join nsc_dr2.meas as meas on chip.exposure=meas.exposure and chip.ccdnum=meas.ccdnum" sql += " where meas.objectid='"+objectid+"'" chip = qc.query(sql=sql,fmt='table') ind3,ind4 = dln.match(chip['exposure'],meas['exposure']) si = np.argsort(ind4) # sort by input meas catalog ind3 = ind3[si] ind4 = ind4[si] chip = chip[ind3] meas = meas[ind4] gdfwhm, = np.where((meas['fwhm'] > 0.2*chip['fwhm']) & (meas['fwhm'] < 2.0*chip['fwhm'])) if len(gdfwhm)==0: print('All measurements have bad FWHM values') return if len(gdfwhm) < len(meas): print('Removing '+str(len(meas)-len(gdfwhm))+' measurements with bad FWHM values') meas = meas[gdfwhm] raerr = np.array(meas['raerr']*1e3,np.float64) # milli arcsec ra = np.array(meas['ra'],np.float64) ra -= np.mean(ra) ra *= 3600*1e3 * np.cos(mndec/radeg) # convert to true angle, milli arcsec t = np.array(meas['mjd'].copy()) t -= np.mean(t) t /= 365.2425 # convert to year # Calculate robust slope try: #pmra, pmraerr = dln.robust_slope(t,ra,raerr,reweight=True) # LADfit pmra_ladcoef, absdev = dln.ladfit(t,ra) pmra_lad = pmra_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), ra) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmra_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmra_coef, pmra_coeferr = dln.poly_fit(t[gdmask],ra[gdmask],1,sigma=raerr[gdmask],robust=True,error=True) #pmra_coef, pmra_coeferr = dln.poly_fit(t,ra,1,sigma=raerr,robust=True,error=True) #pmra = pmra_coef[0] #pmraerr = pmra_coeferr[0] #radiff = ra-dln.poly(t,pmra_coef) radiff = ra-t*pmra_lad radiff -= np.median(radiff) rasig = dln.mad(radiff) # Reject outliers gdsig = (np.abs(radiff) < 2.5*rasig) | (np.abs(radiff) < 2.5*raerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmra_coef, pmra_coeferr = dln.poly_fit(t[gdsig],ra[gdsig],1,sigma=raerr[gdsig],robust=True,error=True) pmra = pmra_coef[0] pmraerr = pmra_coeferr[0] rasig = dln.mad(ra-dln.poly(t,pmra_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) decerr = np.array(meas['decerr']*1e3,np.float64) # milli arcsec dec = np.array(meas['dec'],np.float64) dec -= np.mean(dec) dec *= 3600*1e3 # convert to milli arcsec # Calculate robust slope try: #pmdec, pmdecerr = dln.robust_slope(t,dec,decerr,reweight=True) # LADfit pmdec_ladcoef, absdev = dln.ladfit(t,dec) pmdec_lad = pmdec_ladcoef[1] # Run RANSAC ransac = linear_model.RANSACRegressor() ransac.fit(t.reshape(-1,1), dec) inlier_mask = ransac.inlier_mask_ outlier_mask = np.logical_not(inlier_mask) gdmask = inlier_mask pmdec_ransac = ransac.estimator_.coef_[0] print(' ransac '+str(np.sum(inlier_mask))+' inliers '+str(np.sum(outlier_mask))+' outliers') # Robust, weighted linear with with INLIERS #pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdmask],dec[gdmask],1,sigma=decerr[gdmask],robust=True,error=True) #pmdec_coef, pmdec_coeferr = dln.poly_fit(t,dec,1,sigma=decerr,robust=True,error=True) #pmdec = pmdec_coef[0] #pmdecerr = pmdec_coeferr[0] #decdiff = dec-dln.poly(t,pmdec_coef) decdiff = dec-t*pmdec_lad decdiff -= np.median(decdiff) decsig = dln.mad(decdiff) # Reject outliers gdsig = (np.abs(decdiff) < 2.5*decsig) | (np.abs(decdiff) < 2.5*decerr) print(' '+str(nmeas-np.sum(gdsig))+' 2.5*sigma clip outliers rejected') #if np.sum(gdsig) < nmeas: pmdec_coef, pmdec_coeferr = dln.poly_fit(t[gdsig],dec[gdsig],1,sigma=decerr[gdsig],robust=True,error=True) pmdec = pmdec_coef[0] pmdecerr = pmdec_coeferr[0] decsig = dln.mad(dec-dln.poly(t,pmdec_coef)) except: print('problem') #import pdb; pdb.set_trace() return np.append(np.zeros(10,float)+np.nan, np.zeros(2,int)) deltamjd = np.max(meas['mjd'])-np.min(meas['mjd']) out = np.array([pmra,pmraerr,pmra_ransac,pmra_lad,rasig,pmdec,pmdecerr,pmdec_ransac,pmdec_lad,decsig,nmeas,deltamjd]) #print(out[[0,2,3]]) #print(out[[5,7,8]]) #import pdb; pdb.set_trace() return out
def zscaling(im, contrast=0.25, nsample=500000): """ This finds the IRAF display z1,z2 scalings for an image Parameters: im 2D image =contrast Contrast to use (contrast=0.25 by default) =nsample The number of points of the image to use for the sample (nsample=5e4 by default). Returns: [z1,z1] The minimum and maximum value to use for display scaling. Examples: zcals = zscale(im) From IRAF display help If the contrast is not zero the sample pixels are ranked in brightness to form the function I(i) where i is the rank of the pixel and I is its value. Generally the midpoint of this function (the median) is very near the peak of the image histogram and there is a well defined slope about the midpoint which is related to the width of the histogram. At the ends of the I(i) function there are a few very bright and dark pixels due to objects and defects in the field. To determine the slope a linear function is fit with iterative rejection; I(i) = intercept + slope * (i - midpoint) If more than half of the points are rejected then there is no well defined slope and the full range of the sample defines z1 and z2. Otherwise the endpoints of the linear function are used (provided they are within the original range of the sample): z1 = I(midpoint) + (slope / contrast) * (1 - midpoint) z2 = I(midpoint) + (slope / contrast) * (npoints - midpoint) The actual IRAF program is called zsc_zlimits and is in the file /net/astro/iraf2.12/iraf/pkg/images/tv/display/zscale.x By D.Nidever Oct 2007 (using IRAF display zscale algorithm) """ if im.ndim != 2: raise ValueError('The input must be 2 dimensiona') nx, ny = im.shape n = nx * ny nsample = np.minimum(nsample, n) xind = np.round(np.random.random(nsample) * (nx - 1)).astype(int) yind = np.round(np.random.random(nsample) * (ny - 1)).astype(int) f = im[xind, yind] si = np.argsort(f) f2 = f[si] x = np.arange(nsample) midpoint = np.round(nsample * 0.5) med = np.median(f) zmin = np.min(im) zmax = np.max(im) zmed = np.median(im) # Robust fitting program coef = dln.poly_fit(x, f2, 1, robust=True) # y = m*x + b # I = intercept + slope * (i-midpoint) # I = intercept + slope * i - slope*midpoint # I = slope * i + (intercept - slope*midpoint) # b = intercept - slope*midpoint # intercept = b + slope*midpoint slope = coef[0] intercept = coef[1] + slope * midpoint # z1 = I(midpoint) + (slope / contrast) * (1 - midpoint) # z2 = I(midpoint) + (slope / contrast) * (npoints - midpoint) #z1 = f2[midpoint] + (slope/contrast) * (1L - midpoint) #z2 = f2[midpoint] + (slope/contrast) * (nsample - midpoint) z1 = zmed + (slope / contrast) * (1 - midpoint) z2 = zmed + (slope / contrast) * (nsample - midpoint) z1 = np.maximum(z1, zmin) z2 = np.minimum(z2, zmax) return [z1, z2]
print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(nind)+' exposures') if nind>0: calstr1 = calstr[ind] zpterm = calstr1['zpterm'] bdzp,nbdzp = dln.where(~np.isfinite(zpterm)) # fix Infinity/NAN if nbdzp>0:zpterm[bdzp] = 999999.9 am = calstr1['airmass'] mjd = calstr1['mjd'] bdam,nbdam = dln.where(am < 0.9) if nbdam>0: am[bdam] = np.median(am) # I GOT TO HERE IN THE TRANSLATING!!! #glactc,calstr1.ra,calstr1.dec,2000.0,glon,glat,1,/deg # Measure airmass dependence gg0,ngg0 = dln.where((np.abs(zpterm)<50) & (am<2.0)) coef0 = dln.poly_fit(am[gg0],zpterm[gg0],1,robust=True) zpf = dln.poly(am,coef0) sig0 = np.mad(zpterm[gg0]-zpf[gg0]) gg,ngg = dln.where(np.abs(zpterm-zpf) < (np.maximum(3.5*sig0,0.2))) coef = dln.poly_fit(am[gg],zpterm[gg],1,robust=True) print(zpstr['instrument'][i]+'-'+zpstr['filter'][i]+' '+str(coef)) # Trim out bad exposures to determine the correlations and make figures gg,ngg = dln.where(np.abs(zpterm-zpf) < (3.5*sig0 > 0.2) and calstr1.airmass < 2.0 and calstr1.fwhm < 2.0 and calstr1.rarms < 0.15 & calstr1.decrms < 0.15 and calstr1.success == 1 and calstr1.wcscal == 'Successful' and calstr1.zptermerr < 0.05 & calstr1.zptermsig < 0.08 and (calstr1.ngoodchipwcs == calstr1.nchips) & (calstr1.instrument != 'c4d' or calstr1.zpspatialvar_nccd <= 5 or (calstr1.instrument == 'c4d' and calstr1.zpspatialvar_nccd > 5 and calstr1.zpspatialvar_rms < 0.1)) and np.abs(glat) > 10 and calstr1.nrefmatch > 100 and calstr1.exptime >= 30) # Zpterm with airmass dependence removed relzpterm = zpterm + 25 # 25 to get "absolute" zpterm