def plotFluxRatiosDist(predFlux, measFlux, RA, Dec, refRA, refDec, labels, outDir, name1, name2, format, clip=True): """ Makes plot of measured-to-predicted flux ratio vs. distance from center """ import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip try: import matplotlib if matplotlib.get_backend() is not 'Agg': matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter except Exception as e: raise ImportError('PyPlot could not be imported. Plotting is not ' 'available: {0}'.format(e.message)) if name1 is None: name1 = 'Model 1' if name2 is None: name2 = 'Model 2' ratio = measFlux / predFlux separation = np.zeros(len(measFlux)) for i in range(len(measFlux)): separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value fig = plt.figure(figsize=(7.0, 5.0)) ax1 = plt.subplot(1, 1, 1) ax1.plot(separation, ratio, 'o') plt.title('Flux Density Ratios ({0} / {1})'.format(name1, name2)) plt.ylabel('Flux density ratio') plt.xlabel('Distance from center (deg)') # Calculate mean ratio and std dev. if clip: mean_ratio = np.mean(sigma_clip(ratio)) std = np.std(sigma_clip(ratio)) else: mean_ratio = np.mean(ratio) std = np.std(ratio) ax1.set_ylim(0, 2.0*mean_ratio) xmin, xmax, ymin, ymax = plt.axis() ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g') ax1.plot([0.0, xmax], [mean_ratio+std, mean_ratio+std], '-.g') ax1.plot([0.0, xmax], [mean_ratio-std, mean_ratio-std], '-.g') if labels is not None: xls = separation yls = ratio for label, xl, yl in zip(labels, xls, yls): plt.annotate(label, xy = (xl, yl), xytext = (-2, 2), textcoords= 'offset points', ha='right', va='bottom') plt.savefig(outDir+'flux_ratio_vs_distance.{}'.format(format), format=format)
def plotFluxRatiosFlux(predFlux, measFlux, labels, outDir, clip=True): """ Makes plot of measured-to-predicted flux ratio vs. flux """ import os import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip try: import matplotlib if matplotlib.get_backend() is not 'Agg': matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter except Exception as e: raise ImportError('PyPlot could not be imported. Plotting is not ' 'available: {0}'.format(e.message)) ratio = measFlux / predFlux fig = plt.figure(figsize=(7.0, 5.0)) ax1 = plt.subplot(1, 1, 1) ax1.plot(measFlux, ratio, 'o') ax1.set_ylim(0, 2) plt.title('Flux Ratios (Model 1 / Model 2)') plt.ylabel('Flux ratio') plt.xlabel('Model 1 flux (Jy)') # Calculate mean ratio and std dev. if clip: mean_ratio = np.mean(sigma_clip(ratio)) std = np.std(sigma_clip(ratio)) else: mean_ratio = np.mean(ratio) std = np.std(ratio) xmin, xmax, ymin, ymax = plt.axis() ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g') ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], '-.g') ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], '-.g') if labels is not None: xls = measFlux yls = ratio for label, xl, yl in zip(labels, xls, yls): plt.annotate(label, xy=(xl, yl), xytext=(-2, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(outDir + 'flux_ratio_vs_flux.pdf', format='pdf')
def plotFluxRatiosDist(predFlux, measFlux, RA, Dec, refRA, refDec, labels, outDir, clip=True): """ Makes plot of measured-to-predicted flux ratio vs. distance from center """ import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip try: import matplotlib if matplotlib.get_backend() is not "Agg": matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter except Exception as e: raise ImportError("PyPlot could not be imported. Plotting is not " "available: {0}".format(e.message)) ratio = measFlux / predFlux separation = np.zeros(len(measFlux)) for i in range(len(measFlux)): separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value fig = plt.figure(figsize=(7.0, 5.0)) ax1 = plt.subplot(1, 1, 1) ax1.plot(separation, ratio, "o") ax1.set_ylim(0, 2) plt.title("Flux Ratios (Model 1 / Model 2)") plt.ylabel("Flux ratio") plt.xlabel("Distance from center (deg)") # Calculate mean ratio and std dev. if clip: mean_ratio = np.mean(sigma_clip(ratio)) std = np.std(sigma_clip(ratio)) else: mean_ratio = np.mean(ratio) std = np.std(ratio) xmin, xmax, ymin, ymax = plt.axis() ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], "--g") ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], "-.g") ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], "-.g") if labels is not None: xls = separation yls = ratio for label, xl, yl in zip(labels, xls, yls): plt.annotate(label, xy=(xl, yl), xytext=(-2, 2), textcoords="offset points", ha="right", va="bottom") plt.savefig(outDir + "flux_ratio_vs_distance.pdf", format="pdf")
def plotFluxRatiosFlux(predFlux, measFlux, labels, outDir, clip=True): """ Makes plot of measured-to-predicted flux ratio vs. flux """ import os import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip try: import matplotlib if matplotlib.get_backend() is not 'Agg': matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter except Exception as e: raise ImportError('PyPlot could not be imported. Plotting is not ' 'available: {0}'.format(e.message)) ratio = measFlux / predFlux fig = plt.figure(figsize=(7.0, 5.0)) ax1 = plt.subplot(1, 1, 1) ax1.plot(measFlux, ratio, 'o') ax1.set_ylim(0, 2) plt.title('Flux Ratios (Model 1 / Model 2)') plt.ylabel('Flux ratio') plt.xlabel('Model 1 flux (Jy)') # Calculate mean ratio and std dev. if clip: mean_ratio = np.mean(sigma_clip(ratio)) std = np.std(sigma_clip(ratio)) else: mean_ratio = np.mean(ratio) std = np.std(ratio) xmin, xmax, ymin, ymax = plt.axis() ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g') ax1.plot([0.0, xmax], [mean_ratio+std, mean_ratio+std], '-.g') ax1.plot([0.0, xmax], [mean_ratio-std, mean_ratio-std], '-.g') if labels is not None: xls = measFlux yls = ratio for label, xl, yl in zip(labels, xls, yls): plt.annotate(label, xy = (xl, yl), xytext = (-2, 2), textcoords= 'offset points', ha='right', va='bottom') plt.savefig(outDir+'flux_ratio_vs_flux.pdf', format='pdf')
def calculateWeights(self): """ finds illum cal factors by making an image per wavelength bin, then smoothing it and dividing the mean of the smoothedImage over pixels by the smoothedImage """ cubeWeightsList = [] self.averageSpectra = [] deltaWeightsList = [] self.totalCube = np.sum(self.spectralCubes,axis=0) #sum all cubes self.totalFrame = np.sum(self.totalCube,axis=-1)#sum over wvl weights = [] for iWvl in xrange(self.nWvlBins): wvlSlice = self.totalCube[:,:,iWvl] wvlSlice[wvlSlice == 0] = np.nan nanMask = np.isnan(wvlSlice) #do a sigma-clipping on the wvlSlice and insert it back in the cube maskedWvlSlice = np.ma.array(wvlSlice,mask=nanMask) clippedWvlSlice = sigma_clip(wvlSlice,sig=self.nSigmaClip,iters=None,cenfunc=np.ma.median) wvlSlice[clippedWvlSlice.mask] = np.nan self.totalCube[:,:,iWvl] = wvlSlice #do a smoothing over the slice smoothedWvlSlice = nearestNRobustMeanFilter(wvlSlice,n=self.nNearest,nSigmaClip=self.nSigmaClip) wvlIllumWeights = np.mean(smoothedWvlSlice)/smoothedWvlSlice weights.append(wvlIllumWeights) self.weights = np.array(weights) #move the wvl dimension to the end self.weights = np.swapaxes(self.weights,0,1) self.weights = np.swapaxes(self.weights,1,2) self.deltaWeights = np.zeros_like(self.weights) self.flags = np.zeros_like(self.weights)
def sigmas(files, nofiles = 48, plot = False): '''Returns a mean spectrum in the form of a 2d array - one dimension is the wavelengths, the other is the flux files. The input is the start of the filenames; i.e. if you have 1040 data files named "mortenisthebest00001.txt", "mortenisthebest00002.txt" etc., the input is files="mortenisthebest0" (the file format is assumed to be .txt) and nofiles = 1040. ''' all_data = np.zeros((nofiles, 2048, 2)) # First loop over the files to get data from each and put the data # in all_data: for indx in range(nofiles): file1 = files + str(indx).zfill(len(str(nofiles))) + '.txt' data1 = np.genfromtxt(file1, skip_header = 17, skip_footer = 1) all_data[indx] = data1 datamean = np.zeros((2,2048)) # We want to have a mean array datamean that gives us the x-values # and mean flux values for each wavelength in the spectrum. # The x-values are assumed to be the same for every spectrum. datamean[0] = all_data[0,:,0] # Do a sigma clip to take away outliers for i in range(len(all_data[0,:,0])): bool_clip = sigma_clip(all_data[:,i,1], sig = 1.5)[1] datamean[1,i] = np.mean(all_data[:,i,1][bool_clip]) # datamean[1] = np.mean(all_data[:,:,1], axis = 0) if plot == True: plt.clf() plt.plot(datamean[0], datamean[1]) plt.show() plt.semilogy() plt.axis([530,680,15,60000]) plt.xlabel('Wavelength (nm)', fontsize = 20) plt.ylabel('Flux (counts)', fontsize = 20) plt.title('The raw mean of the spectra', fontsize = 25) return datamean
def _compute_diff(arg): """Worker: Computes the DC offset of frame-coadd""" upperKey, upperPath, upperWeightPath, lowerKey, lowerPath, \ lowerWeightPath, overlap = arg # print "diff between", upperKey, lowerKey upper = SliceableImage.makeFromFITS(upperKey, upperPath, upperWeightPath) upper.setRange(overlap.upSlice) lower = SliceableImage.makeFromFITS(lowerKey, lowerPath, lowerWeightPath) lower.setRange(overlap.loSlice) goodPix = np.where((upper.weight > 0.) & (lower.weight > 0.)) nPixels = len(goodPix[0]) if nPixels > 10: diff_pixels = upper.image[goodPix] - lower.image[goodPix] diff_pixels = diff_pixels[np.isfinite(diff_pixels)] clipped = sigma_clip(diff_pixels, sig=5., iters=1, varfunc=np.nanvar) median = np.nanmedian(clipped[~clipped.mask]) sigma = np.nanstd(clipped[~clipped.mask]) diffData = {"diffimage_mean": median, "diffimage_sigma": sigma, "area": nPixels} else: diffData = None return upperKey, lowerKey, diffData
illumImg = imgDict['illumImg'] exptime = imgDict['exptime'] nRows,nCols = np.shape(flatImg) nanMask = np.isnan(rawImg) | np.isnan(flatImg) | np.isnan(illumImg) rawImg = np.ma.array(rawImg,mask=nanMask) flatImg = np.ma.array(flatImg,mask=nanMask) illumImg = np.ma.array(illumImg,mask=nanMask) bClipData = True sig=3. if bClipData: iters=None cenfunc = np.ma.median rawImg = sigma_clip(rawImg,sig=sig,iters=iters,cenfunc=cenfunc) flatImg = sigma_clip(flatImg,sig=sig,iters=iters,cenfunc=cenfunc) illumImg = sigma_clip(illumImg,sig=sig,iters=iters,cenfunc=cenfunc) clipMask = rawImg.mask | flatImg.mask | illumImg.mask rawImg.mask = clipMask flatImg.mask = clipMask illumImg.mask = clipMask print 'removed ',np.sum(clipMask)-np.sum(nanMask),'pixels' rawList = rawImg[~rawImg.mask] flatList = flatImg[~flatImg.mask] illumList = illumImg[~illumImg.mask] #plotArray(title='with flatcal',image=flatImg)
def findStats(predFlux, measFlux, RA, Dec, refRA, refDec, outDir, info0, info1, info2, name1, name2): """ Calculates statistics and saves them to 'stats.txt' """ import os import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip ratio = measFlux / predFlux meanRatio = np.mean(ratio) stdRatio = np.std(ratio) clippedRatio = sigma_clip(ratio) meanClippedRatio = np.mean(clippedRatio) stdClippedRatio = np.std(clippedRatio) RAOffsets = np.zeros(len(RA)) DecOffsets = np.zeros(len(Dec)) for i in range(len(RA)): if RA[i] >= refRA[i]: sign = 1.0 else: sign = -1.0 RAOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], refRA[i], Dec[i]).value # deg if Dec[i] >= refDec[i]: sign = 1.0 else: sign = -1.0 DecOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], RA[i], refDec[i]).value # deg meanRAOffset = np.mean(RAOffsets) stdRAOffset = np.std(RAOffsets) clippedRAOffsets = sigma_clip(RAOffsets) meanClippedRAOffset = np.mean(clippedRAOffsets) stdClippedRAOffset = np.std(clippedRAOffsets) meanDecOffset = np.mean(DecOffsets) stdDecOffset = np.std(DecOffsets) clippedDecOffsets = sigma_clip(DecOffsets) meanClippedDecOffset = np.mean(clippedDecOffsets) stdClippedDecOffset = np.std(clippedDecOffsets) stats = {'meanRatio':meanRatio, 'stdRatio':stdRatio, 'meanClippedRatio':meanClippedRatio, 'stdClippedRatio':stdClippedRatio, 'meanRAOffsetDeg':meanRAOffset, 'stdRAOffsetDeg':stdRAOffset, 'meanClippedRAOffsetDeg':meanClippedRAOffset, 'stdClippedRAOffsetDeg':stdClippedRAOffset, 'meanDecOffsetDeg':meanDecOffset, 'stdDecOffsetDeg':stdDecOffset, 'meanClippedDecOffsetDeg':meanClippedDecOffset, 'stdClippedDecOffsetDeg':stdClippedDecOffset} outLines = ['Statistics from sky model comparison\n'] outLines.append('------------------------------------\n\n') outLines.append('Sky model 1 ({}):\n'.format(name1)) outLines.append(info1+'\n\n') outLines.append('Sky model 2 ({}):\n'.format(name2)) outLines.append(info2+'\n\n') outLines.append(info0+'\n') outLines.append('Number of matches found for comparison: {0}\n\n'.format(len(predFlux))) outLines.append('Mean flux density ratio (1 / 2): {0}\n'.format(meanRatio)) outLines.append('Std. dev. flux density ratio (1 / 2): {0}\n'.format(stdRatio)) outLines.append('Mean 3-sigma-clipped flux density ratio (1 / 2): {0}\n'.format(meanClippedRatio)) outLines.append('Std. dev. 3-sigma-clipped flux density ratio (1 / 2): {0}\n\n'.format(stdClippedRatio)) outLines.append('Mean RA offset (1 - 2): {0} degrees\n'.format(meanRAOffset)) outLines.append('Std. dev. RA offset (1 - 2): {0} degrees\n'.format(stdRAOffset)) outLines.append('Mean 3-sigma-clipped RA offset (1 - 2): {0} degrees\n'.format(meanClippedRAOffset)) outLines.append('Std. dev. 3-sigma-clipped RA offset (1 - 2): {0} degrees\n\n'.format(stdClippedRAOffset)) outLines.append('Mean Dec offset (1 - 2): {0} degrees\n'.format(meanDecOffset)) outLines.append('Std. dev. Dec offset (1 - 2): {0} degrees\n'.format(stdDecOffset)) outLines.append('Mean 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n'.format(meanClippedDecOffset)) outLines.append('Std. dev. 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n\n'.format(stdClippedDecOffset)) fileName = outDir+'stats.txt' statsFile = open(fileName, 'w') statsFile.writelines(outLines) statsFile.close() return stats
def instantiate(self, *args): ''' Clear the old NuisanceSampler, instantiate the regions using the stored residual spectra, and create a new NuisanceSampler. ''' # threshold for sigma clipping sigma = config["sigma_clip"] # array that specifies if a pixel is already covered. # to start, it should be all False covered = np.zeros((self.npoints, ), dtype='bool') #average all of the spectra in the deque together residual_array = np.array(self.resid_deque) if len(self.resid_deque) == 0: raise RuntimeError("No residual spectra stored yet.") else: residuals = np.average(residual_array, axis=0) # run the sigma_clip algorithm until converged, and we've identified the outliers filtered_data = sigma_clip(residuals, sig=sigma, iters=None) mask = filtered_data.mask wl = self.wl sigma0 = config['region_priors']['sigma0'] logAmp = config["region_params"]["logAmp"] sigma = config["region_params"]["sigma"] # Sort in decreasing strength of residual self.nregions = 0 regions = {} region_mus = {} for w, resid in sorted(zip(wl[mask], np.abs(residuals[mask])), key=itemgetter(1), reverse=True): if w in wl[covered]: continue else: # check to make sure region is not *right* at the edge of the echelle order if w <= np.min(wl) or w >= np.max(wl): continue else: # instantiate region and update coverage # Default amp and sigma values regions[self.nregions] = { "logAmp": logAmp, "sigma": sigma, "mu": w } region_mus[ self.nregions] = w # for evaluating the mu prior self.nregions += 1 # determine the stretch of wl covered by this new region ind = (wl >= (w - sigma0)) & (wl <= (w + sigma0)) # update the covered regions covered = covered | ind # Take the current nuisance positions as a starting point, and add the regions starting_dict = self.sampler.params.copy() starting_dict["regions"] = regions region_mus = np.array([region_mus[i] for i in range(self.nregions)]) # Setup the priors region_priors = config["region_priors"] region_priors.update({"mus": region_mus}) prior_params = {"regions": region_priors} # do all this crap again cheb_MH_cov = float(config["cheb_jump"])**2 * np.ones((self.npoly, )) cov_MH_cov = np.array([ float(config["cov_jump"][key]) for key in self.sampler.cov_tup ])**2 region_MH_cov = [ float(config["region_jump"][key])**2 for key in C.cov_region_parameters ] regions_MH_cov = np.array( [region_MH_cov for i in range(self.nregions)]).flatten() nuisance_MH_cov = np.diag( np.concatenate((cheb_MH_cov, cov_MH_cov, regions_MH_cov))) print(starting_dict) print("cov shape {}".format(nuisance_MH_cov.shape)) # Initialize a new sampler, replacing the old one self.sampler = NuisanceSampler(OrderModel=self, starting_param_dict=starting_dict, cov=nuisance_MH_cov, debug=True, outdir=self.noutdir, prior_params=prior_params, order=self.order) self.p0 = self.sampler.p0 # Update the nuisance parameters to the starting values so that we at least have a self.data_mat print("Updating nuisance parameter data products to starting values.") self.update_nuisance(starting_dict) self.lnprob = self.evaluate() # To speed up convergence, try just doing a bunch of nuisance runs before # going into the iteration pattern print("Doing nuisance burn-in for {} samples".format( config["nuisance_burn"])) self.independent_sample(config["nuisance_burn"])
def ubersmoother(self,check=False): def plot_ubersmoother(x, y, idx, idx0, smooth=False, labels=None): # Plot various steps of the smoothing process ax.scatter(x, y, s=60, alpha=0.2) try: idx0_len = len(idx0) except: idx0_len = 0 if idx0_len > 0: ax.scatter(x[idx0], y[idx0], marker='x', c='black', lw=1.5, s=60, label='removed previously') ax.scatter(x[idx], y[idx], marker='x', c='red', lw=2.5, s=60, label='removed this loop') if smooth: ax.plot(x,np.interp(x, x[~idx], y[~idx]), c='orange', lw=2, label='smoothed') # Set axis ax.axis([np.min(x), np.max(x), np.min(y)*0.98, np.max(y)*1.02]) # Set labels ax.legend(loc='best') ax.set_title(labels[0]) ax.set_xlabel(labels[1]) ax.set_ylabel(labels[2]) ####################### # Initializing variables time = self.time phase = self.phase flux = self.flux ferr = self.ferr # Find the number of full periods in lightcurve n_epochs = float(len(flux)) n_epochs_period = float(len(np.where((time >= time[0]) & (time < time[0]+period))[0])) n_full_periods = np.ceil(n_epochs / n_epochs_period) # Append nans to flux so there an integer number of periods n_pad = (n_epochs_period*n_full_periods) - n_epochs ind_pad = np.append(range(0,len(flux)),[-1]*n_pad) ind_reshape = np.reshape(ind_pad,(n_full_periods,n_epochs_period)) flux_smooth = np.array(flux[:]) outlier_mask = np.zeros(flux.shape,dtype=bool) ########################### # SMOOTHING THE LIGHTCURVE: # 1) Slice by period-width slices (starting at 1 period) # 2) Sort the flux by phase # A. if slice width == period: rolling median # B. if slice width > period: median_filtering + sigma_clipping # 3) Shift slice over one period, repeat steps. # 4) Store outliers in mask over which to interpolate later. # width is 1 period, median filtering+sigma_clipping if > 1 period) # Loop up to slices of 10*period OR # if fewer than 10 periods, # of periods if n_full_periods < 10: n_slices = n_full_periods else: n_slices = 10 if check: plt_ct = 1 for p in np.arange(1,n_slices): for s in np.arange(0,n_full_periods-p): indices = [0,p]+s # Start and end indices start_INDX = indices[0].astype("int") end_INDX = indices[1].astype("int") # Select region of interest ind_region = ind_reshape[start_INDX:end_INDX].flatten() msk = ind_region < 0 ind_region = ind_region[~msk] flux_region = flux_smooth[ind_region] time_region = time[ind_region] phase_region = phase[ind_region] # Phase the data phasesort = np.argsort(phase_region) flux_region_phasesort = flux_region[phasesort] time_region_phasesort = time_region[phasesort] phase_region_phasesort = phase_region[phasesort] ind_region_phasesort = ind_region[phasesort] if p == 1: ######################### ### Rolling median method threshold = 3. flux_pandas = rolling_median(flux_region_phasesort, window=5, center=True, freq=period) diff = np.abs(flux_region_phasesort - flux_pandas) outlier_idx = diff > np.std(diff[~np.isnan(diff)]) outlier_mask[ind_region_phasesort] = outlier_idx if check and s==0: fig = plt.figure(figsize=(12,8)) ax = fig.add_subplot(111) labels = ['Rolling Median on 1-folded Period Slices', 'Phase', 'Flux (counts)'] plot_ubersmoother(phase_region_phasesort, flux_region_phasesort, outlier_idx, None, labels=labels) plt.show() else: ################################### # Median_filtering + sigma_clipping. # Median_filtering boxsize = 3% of period (scales with npts) box = round(len(phase_region_phasesort)*0.03) flux_region_filt = median_filter(flux_region_phasesort, size=box, mode="reflect")#, #cval=np.median(flux_region_phasesort)) clip = sigma_clip(flux_region_phasesort - flux_region_filt, 2.0) # Sort clip into time spacing and save outliers in outliermask timesort = np.argsort(time_region_phasesort) clip_timesort = clip[timesort] outlier_mask[ind_region[np.where(clip_timesort.mask == True)[0]]] = True if check and s==0 and p in [2,9]: if plt_ct == 1: fig = plt.figure(figsize=(12,12)) ax = fig.add_subplot(2,1,plt_ct) labels = ['Sigma Clipping on %s-folded Period Slices' %(p), 'Phase', 'Flux (counts)'] plot_ubersmoother(phase_region_phasesort, flux_region_phasesort, clip.mask, outlier_mask[ind_region_phasesort], labels=labels) ax.plot(phase_region_phasesort, flux_region_filt, c='red', lw=1.5) if plt_ct == 3: fig.subplots_adjust(hspace=0.40) plt.show() else: plt_ct+=1 # Interpolate across missing points flux_smooth = np.interp(time, time[~outlier_mask], flux[~outlier_mask]) # If long-cadence data, insert raw transits, apply gaussian filter to # smooth. Gaussian_filter causes us to underestimate the bottoms of # the wd occultations. if "LC" in self.filename: only_transit = np.where((phase >= 0.73) & (phase <= 0.77))[0] flux_smooth[only_transit] = flux[only_transit] flux_smooth_final = gaussian_filter(flux_smooth, 1, order=0) # For the short-cadence data, median_filter (290) entire lightcurve, # then insert median_filtered (23 window) wd occultations elif "SC" in self.filename: only_transit = np.where((phase >= 0.74) & (phase <= 0.78))[0] flux_smooth_no_transit = median_filter(flux_smooth, size=290) flux_smooth_transit = median_filter(flux_smooth, size=23) flux_smooth_no_transit[only_transit] = flux_smooth_transit[only_transit] flux_smooth_final = flux_smooth_no_transit if check: fig = plt.figure(figsize=(12,8)) ax = fig.add_subplot(111) ax.plot(time, flux, '-ko', lw=2, alpha=0.6) ax.plot(time, flux_smooth_final, c="#32cd32", lw=2, alpha=0.7) ax.axis([np.min(time), np.max(time), np.min(flux)*0.98, np.max(flux)*1.02]) ax.set_title('Final Smoothed Flux') ax.set_xlabel('Time (days)') ax.set_ylabel('Flux (counts') plt.show() self.flux_smooth = flux_smooth_final self.outlier_mask = outlier_mask # Use smoothed flux to normalize. Mask out occultations self.normalize()
def correct_overscan(amplifier, bias_slice=None, data_slice=None, clip=3.): """Extract bias-corrected, exposed parts of raw GMOS fits file Bias is determined from selected regions of overscan, clipping outliers Parameters ---------- amplifier: ~fits.ImageHDU bias_slice: list of 2 slices slices in Y, X to use for determining the bias level (default=None: use 'BIASSEC' from header) data_slice: list of 2 slices slices in Y, X that give good parts of the array (default=None: use 'DATASEC' from header) clip: ~float number of standard deviations from the median within which overscan values have to be to be included. Returns ------- ~fits.ImageHDU bias-corrected exposed parts Note ---- The output fits structure will have the following new headers: APPLIED DATASEC: section of initial data array extracted APPLIED BIASSEC: section of initial data array used to determine bias level BIAS: bias value subtracted (ADU) BIASSTD: standard deviation of values going in to bias value (ADU) (should be similar to read noise; this is not checked here) BIASUNC: formal uncertainty in bias (BIASSTD/sqrt(N), in ADU) The following headers are updated CRPIX1: X reference pixel for the world coordinate system CRPIX1: Y reference pixel for the world coordinate system CCDSEC: the part of the chip represented by the data section DETSEC: the part of the overall array represented by the data section """ if bias_slice is None: bias_slice = sec2slice(amplifier.header['BIASSEC']) if data_slice is None: data_slice = sec2slice(amplifier.header['DATASEC']) data_subslice = None else: # ensure we get numbers only, rather than relative indices, # otherwise cannot store it as DATASEC header later data_slice = [slice(*s.indices(shape)) for s, shape in zip(data_slice, amplifier.data.shape)] # get equivalent subslice of 'DATASEC' for updating CCDSEC, DETSEC data_subslice = get_subslices(sec2slice(amplifier.header['DATASEC']), data_slice) overscan = amplifier.data[bias_slice] if clip: clipped = stats.sigma_clip(overscan, clip, 1) else: clipped = overscan bias_estimate, bias_std = clipped.mean(), clipped.std() bias_unc = bias_std/np.sqrt(clipped.size) outamp = fits.ImageHDU(amplifier.data[data_slice]-bias_estimate, amplifier.header) # add new headers with information on what was done outamp.header['APPLIED DATASEC'] \ = slice2sec(data_slice), 'Section considered good' outamp.header['APPLIED BIASSEC'] \ = slice2sec(bias_slice), 'Section used to estimate bias' outamp.header['BIAS'] = bias_estimate, 'ADU' outamp.header['BIASSTD'] = bias_std, 'ADU' outamp.header['BIASUNC'] = bias_unc, 'ADU' # adjust remove/existing headers as appropriate outamp.header.remove('BIASSEC') # no BIAS section left outamp.header.remove('DATASEC') # all of image is now DATA outamp.header['CRPIX1'] += data_slice[1].start outamp.header['CRPIX2'] += data_slice[0].start if data_subslice: binning = np.fromstring(amplifier.header['CCDSUM'], sep=' ', dtype=np.int) ccd_subslice = adjust_subslices(data_subslice, binning) outamp.header['CCDSEC'] = slice2sec(slice_slices( sec2slice(amplifier.header['CCDSEC']), ccd_subslice)) outamp.header['DETSEC'] = slice2sec(slice_slices( sec2slice(amplifier.header['DETSEC']), ccd_subslice)) return outamp
beforeImg[~stableMask] = 0 afterImg[~stableMask] = 0 print 'masked stable sdev<{}%'.format(sdevThreshold) bCoolMask = False if bCoolMask: effIntImg = np.load('stableMask.npz')['effIntImg'] effIntThreshold = .999 mask = effIntImg >= effIntThreshold beforeImg[~mask] = 0 afterImg[~mask] = 0 print 'masked effIntTime fraction >{}'.format(effIntThreshold) nRows,nCols = np.shape(beforeImg) clippedBeforeImg = sigma_clip(beforeImg,sig=3) clipMask = clippedBeforeImg.mask beforeImg[clipMask] = 0 afterImg[clipMask] = 0 deadBeforeImg = (beforeImg == 0) deadAfterImg = (afterImg == 0) beforeList = beforeImg[beforeImg != 0] afterList = afterImg[afterImg != 0] afterImg[deadAfterImg] = np.nan beforeImg[deadBeforeImg] = np.nan plotArray(title='without flatcal',image=beforeImg) #plotArray(title='with flatcal',image=afterImg)
def plotFluxRatiosDist(predFlux, measFlux, RA, Dec, refRA, refDec, labels, outDir, name1, name2, format, clip=True): """ Makes plot of measured-to-predicted flux ratio vs. distance from center """ import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip try: import matplotlib if matplotlib.get_backend() is not 'Agg': matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter except Exception as e: raise ImportError('PyPlot could not be imported. Plotting is not ' 'available: {0}'.format(e.message)) if name1 is None: name1 = 'Model 1' if name2 is None: name2 = 'Model 2' ratio = measFlux / predFlux separation = np.zeros(len(measFlux)) for i in range(len(measFlux)): separation[i] = calculateSeparation(RA[i], Dec[i], refRA, refDec).value fig = plt.figure(figsize=(7.0, 5.0)) ax1 = plt.subplot(1, 1, 1) ax1.plot(separation, ratio, 'o') plt.title('Flux Density Ratios ({0} / {1})'.format(name1, name2)) plt.ylabel('Flux density ratio') plt.xlabel('Distance from center (deg)') # Calculate mean ratio and std dev. if clip: mean_ratio = np.mean(sigma_clip(ratio)) std = np.std(sigma_clip(ratio)) else: mean_ratio = np.mean(ratio) std = np.std(ratio) ax1.set_ylim(0, 2.0 * mean_ratio) xmin, xmax, ymin, ymax = plt.axis() ax1.plot([0.0, xmax], [mean_ratio, mean_ratio], '--g') ax1.plot([0.0, xmax], [mean_ratio + std, mean_ratio + std], '-.g') ax1.plot([0.0, xmax], [mean_ratio - std, mean_ratio - std], '-.g') if labels is not None: xls = separation yls = ratio for label, xl, yl in zip(labels, xls, yls): plt.annotate(label, xy=(xl, yl), xytext=(-2, 2), textcoords='offset points', ha='right', va='bottom') plt.savefig(outDir + 'flux_ratio_vs_distance.{}'.format(format), format=format)
def collapse(array_in, region=None, method='sum', sigma=False): """ Collapse a slice of a datacube, with a given mode, along the wavelength slice. This function handles both numpy.array (with no sigma clipping) and numpy.ma masked arrays (with sigma clipping, from astropy.stast.funcs) Parameters ---------- array_in : numpy.ndarray The input datacube array. region : The region of the datacube to collapse, in the form of [a, b], where a is the 1st slice boundary, and b is the 2nd + 1 slice boundary. Can be omitted to collapse entire z-axis. method : {'median', 'sum', 'mean'} The method of collapse. sigma : bool, optional Flag for wether to perform sigma clipping or not (default is False). Returns ------- collapsedArray : numpy.ndarray Collapsed image. Example usage: 1. >> result = ifu_3d_collapse(cube, region=[1000, 1999], method='median') Median collapse cube from z-axis frames 1000-2000. 2. >> result = ifu_3d_collapsed(cube, method='mean', sigma=2.5) Mean collapse a cube along the entire z-axis, performing a sigma clipping of 2.5. """ # Extract the desired slice... if not region: print('(3d_collapse): Entire z-axis will be collapsed!') slice_array = array_in else: print('(3d_collapse): z-axis collapse limits: ', region) slice_array = array_in[region[0]:region[1], :, :] # ... and perform the desired operation, based on input mode. collapsed_array = _arrayCollapse(slice_array, method=method) if sigma: # Import sigma_clip from astropy from astropy.stats.funcs import sigma_clip # Perform sigma clipping on input cube print('(3d_collapse): Clipping data ...') clipped = sigma_clip(slice_array, sigma, iters=None, axis=0, copy=True) # Compare the clipped sample to determine wether we need to # recalculate the collapse if np.array_equal(clipped.compressed(), slice_array.flatten()): print('Arrays are exactly the same, no clipping required.') else: # Now that we have the sigma mask, recalculate the collapse... collapsed_array = _maskedCollapse(clipped, method) # Returns the collapsed array #print('(3d_collapse): Shape of returned array:', collapsed_array.shape) return collapsed_array
def findStats(predFlux, measFlux, RA, Dec, refRA, refDec, outDir, info0, info1, info2, name1, name2): """ Calculates statistics and saves them to 'stats.txt' """ import os import numpy as np from ..operations_lib import calculateSeparation try: from astropy.stats.funcs import sigma_clip except ImportError: from astropy.stats import sigma_clip ratio = measFlux / predFlux meanRatio = np.mean(ratio) stdRatio = np.std(ratio) clippedRatio = sigma_clip(ratio) meanClippedRatio = np.mean(clippedRatio) stdClippedRatio = np.std(clippedRatio) RAOffsets = np.zeros(len(RA)) DecOffsets = np.zeros(len(Dec)) for i in range(len(RA)): if RA[i] >= refRA[i]: sign = 1.0 else: sign = -1.0 RAOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], refRA[i], Dec[i]).value # deg if Dec[i] >= refDec[i]: sign = 1.0 else: sign = -1.0 DecOffsets[i] = sign * calculateSeparation(RA[i], Dec[i], RA[i], refDec[i]).value # deg meanRAOffset = np.mean(RAOffsets) stdRAOffset = np.std(RAOffsets) clippedRAOffsets = sigma_clip(RAOffsets) meanClippedRAOffset = np.mean(clippedRAOffsets) stdClippedRAOffset = np.std(clippedRAOffsets) meanDecOffset = np.mean(DecOffsets) stdDecOffset = np.std(DecOffsets) clippedDecOffsets = sigma_clip(DecOffsets) meanClippedDecOffset = np.mean(clippedDecOffsets) stdClippedDecOffset = np.std(clippedDecOffsets) stats = { 'meanRatio': meanRatio, 'stdRatio': stdRatio, 'meanClippedRatio': meanClippedRatio, 'stdClippedRatio': stdClippedRatio, 'meanRAOffsetDeg': meanRAOffset, 'stdRAOffsetDeg': stdRAOffset, 'meanClippedRAOffsetDeg': meanClippedRAOffset, 'stdClippedRAOffsetDeg': stdClippedRAOffset, 'meanDecOffsetDeg': meanDecOffset, 'stdDecOffsetDeg': stdDecOffset, 'meanClippedDecOffsetDeg': meanClippedDecOffset, 'stdClippedDecOffsetDeg': stdClippedDecOffset } outLines = ['Statistics from sky model comparison\n'] outLines.append('------------------------------------\n\n') outLines.append('Sky model 1 ({}):\n'.format(name1)) outLines.append(info1 + '\n\n') outLines.append('Sky model 2 ({}):\n'.format(name2)) outLines.append(info2 + '\n\n') outLines.append(info0 + '\n') outLines.append('Number of matches found for comparison: {0}\n\n'.format( len(predFlux))) outLines.append('Mean flux density ratio (1 / 2): {0}\n'.format(meanRatio)) outLines.append( 'Std. dev. flux density ratio (1 / 2): {0}\n'.format(stdRatio)) outLines.append( 'Mean 3-sigma-clipped flux density ratio (1 / 2): {0}\n'.format( meanClippedRatio)) outLines.append( 'Std. dev. 3-sigma-clipped flux density ratio (1 / 2): {0}\n\n'.format( stdClippedRatio)) outLines.append( 'Mean RA offset (1 - 2): {0} degrees\n'.format(meanRAOffset)) outLines.append( 'Std. dev. RA offset (1 - 2): {0} degrees\n'.format(stdRAOffset)) outLines.append( 'Mean 3-sigma-clipped RA offset (1 - 2): {0} degrees\n'.format( meanClippedRAOffset)) outLines.append( 'Std. dev. 3-sigma-clipped RA offset (1 - 2): {0} degrees\n\n'.format( stdClippedRAOffset)) outLines.append( 'Mean Dec offset (1 - 2): {0} degrees\n'.format(meanDecOffset)) outLines.append( 'Std. dev. Dec offset (1 - 2): {0} degrees\n'.format(stdDecOffset)) outLines.append( 'Mean 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n'.format( meanClippedDecOffset)) outLines.append( 'Std. dev. 3-sigma-clipped Dec offset (1 - 2): {0} degrees\n\n'.format( stdClippedDecOffset)) fileName = outDir + 'stats.txt' statsFile = open(fileName, 'w') statsFile.writelines(outLines) statsFile.close() return stats
def instantiate(self, *args): ''' Clear the old NuisanceSampler, instantiate the regions using the stored residual spectra, and create a new NuisanceSampler. ''' # threshold for sigma clipping sigma=config["sigma_clip"] # array that specifies if a pixel is already covered. # to start, it should be all False covered = np.zeros((self.npoints,), dtype='bool') #average all of the spectra in the deque together residual_array = np.array(self.resid_deque) if len(self.resid_deque) == 0: raise RuntimeError("No residual spectra stored yet.") else: residuals = np.average(residual_array, axis=0) # run the sigma_clip algorithm until converged, and we've identified the outliers filtered_data = sigma_clip(residuals, sig=sigma, iters=None) mask = filtered_data.mask wl = self.wl sigma0 = config['region_priors']['sigma0'] logAmp = config["region_params"]["logAmp"] sigma = config["region_params"]["sigma"] # Sort in decreasing strength of residual self.nregions = 0 regions = {} region_mus = {} for w, resid in sorted(zip(wl[mask], np.abs(residuals[mask])), key=itemgetter(1), reverse=True): if w in wl[covered]: continue else: # check to make sure region is not *right* at the edge of the echelle order if w <= np.min(wl) or w >= np.max(wl): continue else: # instantiate region and update coverage # Default amp and sigma values regions[self.nregions] = {"logAmp":logAmp, "sigma":sigma, "mu":w} region_mus[self.nregions] = w # for evaluating the mu prior self.nregions += 1 # determine the stretch of wl covered by this new region ind = (wl >= (w - sigma0)) & (wl <= (w + sigma0)) # update the covered regions covered = covered | ind # Take the current nuisance positions as a starting point, and add the regions starting_dict = self.sampler.params.copy() starting_dict["regions"] = regions region_mus = np.array([region_mus[i] for i in range(self.nregions)]) # Setup the priors region_priors = config["region_priors"] region_priors.update({"mus":region_mus}) prior_params = {"regions":region_priors} # do all this crap again cheb_MH_cov = float(config["cheb_jump"])**2 * np.ones((self.npoly,)) cov_MH_cov = np.array([float(config["cov_jump"][key]) for key in self.sampler.cov_tup])**2 region_MH_cov = [float(config["region_jump"][key])**2 for key in C.cov_region_parameters] regions_MH_cov = np.array([region_MH_cov for i in range(self.nregions)]).flatten() nuisance_MH_cov = np.diag(np.concatenate((cheb_MH_cov, cov_MH_cov, regions_MH_cov))) print(starting_dict) print("cov shape {}".format(nuisance_MH_cov.shape)) # Initialize a new sampler, replacing the old one self.sampler = NuisanceSampler(OrderModel=self, starting_param_dict=starting_dict, cov=nuisance_MH_cov, debug=True, outdir=self.noutdir, prior_params=prior_params, order=self.order) self.p0 = self.sampler.p0 # Update the nuisance parameters to the starting values so that we at least have a self.data_mat print("Updating nuisance parameter data products to starting values.") self.update_nuisance(starting_dict) self.lnprob = self.evaluate() # To speed up convergence, try just doing a bunch of nuisance runs before # going into the iteration pattern print("Doing nuisance burn-in for {} samples".format(config["nuisance_burn"])) self.independent_sample(config["nuisance_burn"])