def edge_outlier_clip(data_e, ri, vi, Zi): r_inside = [] v_inside = [] i = 0 while ri[i] <= np.max(data_e['rad']): inner_el = i outer_el = i + 5 inner_r = ri[inner_el] outer_r = ri[outer_el] ''' dens = np.average(Zi[inner_el:outer_el],axis=0) roots = np.sort(np.abs(vi[dens>0.05])) databinned = data_e[np.where((data_e['rad']>=inner_r)&(data_e['rad']<outer_r))] if len(roots) == 0: root = 2 * biweight_midvariance(databinned['vel'].copy(),9.0) elif np.abs(roots[-1]) < 500.0: root = 2 * biweight_midvariance(databinned['vel'].copy(),9.0) elif np.abs(roots[-1]) > 3500.0: root = 3500.0 else: root = np.abs(roots[-1]) r_inside.extend(databinned['rad'][np.where(np.abs(databinned['vel'])<root)]) v_inside.extend(databinned['vel'][np.where(np.abs(databinned['vel'])<root)]) i += 5 data_e = np.vstack((np.array(r_inside),np.array(v_inside))).T return data_e ''' # deriv = (np.average(Zi[inner_el:outer_el],axis=0)[1:]-np.average(Zi[inner_el:outer_el],axis=0)[:-1]) \ # /(vi[1:]-vi[:-1]) roots = np.sort( np.abs(vi[((np.average(Zi[inner_el:outer_el], axis=0)[1:] - np.average(Zi[inner_el:outer_el], axis=0)[:-1]) / (vi[1:] - vi[:-1]))[1:] * ((np.average(Zi[inner_el:outer_el], axis=0)[1:] - np.average(Zi[inner_el:outer_el], axis=0)[:-1]) / (vi[1:] - vi[:-1]))[:-1] < 0])) databinned = data_e[np.where((data_e['rad'] >= inner_r) & (data_e['rad'] < outer_r))] if len(roots) > 1: if roots[1] < 1000.0: if len(roots) > 2: if roots[2] < 1000.0: root = 3 * biweight_midvariance( databinned['vel'].copy(), 9.0) else: root = roots[2] else: root = 3 * biweight_midvariance( databinned['vel'].copy(), 9.0) else: root = roots[1] else: root = 3500.0 r_inside.extend( databinned['rad'][np.where(np.abs(databinned['vel']) < root)]) v_inside.extend( databinned['vel'][np.where(np.abs(databinned['vel']) < root)]) i += 5 data_e = np.vstack((np.array(r_inside), np.array(v_inside))).T return data_e
def getbiweight(z): biweightscale=biweight_midvariance(z) biweightlocation=biweight_location(z) flag=abs(z-biweightlocation)/biweightscale < scale_cut #flag=np.ones(len(z),'bool') repeatflag=1 nloop=0 #print biweightlocation, biweightscale oldbiweightscale=biweightscale while repeatflag: newdata=z[flag] biweightscale=biweight_midvariance(newdata, M=biweightlocation) biweightlocation=biweight_location(newdata, M=biweightlocation) oldflag = flag #flag=abs(z-biweightlocation)/biweightscale < scale_cut nloop += 1 #print nloop, biweightlocation, biweightscale, len(newdata), sum(flag) #if sum(flag == oldflag) == len(flag): # repeatflag=0 # print 'nloop = ', nloop if abs(biweightscale - oldbiweightscale) < 1.: repeatflag=0 #print 'nloop = ', nloop if nloop > 5: repeatflag = 0 oldbiweightscale=biweightscale #print nloop, biweightlocation, biweightscale #flag=abs(z-biweightlocation)/biweightscale < 4. #biweightscale=biweight_midvariance(z[flag],M=biweightlocation) return biweightlocation, biweightscale
def finddups(line): #Filter the data allgood = dataqual[line+'_good'].map(d) #Needs to be bad in any line to be bad baddata = dataqual[line+'_bad'].map(d) somelow = dataqual[line+'_low'].map(d) goodflux = fluxdata[np.logical_not(baddata)] #Get the duplicate OBJIDs of only the good objects dupids = [item for item, count in collections.Counter(goodflux['OBJID']).items() if count > 1] dupobjsarr = [] lineflux1 = [] lineflux2 = [] lineflux3 = [] lineflux4 = [] lineflux5 = [] lineflux6 = [] #Append the data for the good objects for obj in dupids: dupobjsarr.append(fluxdata[fluxdata.OBJID == obj]) #Get the ews for these objects for i in range(0,len(dupobjsarr)): lineflux1.append(dupobjsarr[i].iloc[0][line+'_flux']) lineflux2.append(dupobjsarr[i].iloc[1][line+'_flux']) lineflux3.append(dupobjsarr[i].iloc[0][line+'_flux']) lineflux4.append(dupobjsarr[i].iloc[1][line+'_flux']) lineflux5.append(dupobjsarr[i].iloc[0][line+'_modelabs']) lineflux6.append(dupobjsarr[i].iloc[1][line+'_modelabs']) lineflux3 = np.array(lineflux3) lineflux4 = np.array(lineflux4) lineflux5 = np.array(lineflux5) lineflux6 = np.array(lineflux6) #Make another cut for high S/N flagidx = allgood #Repeat the process just for the high SN objects goodflux = fluxdata[flagidx] linefluxsig1 = [] linefluxsig2 = [] dupids = [item for item, count in collections.Counter(goodflux['OBJID']).items() if count > 1] dupobjsarr = [] for obj in dupids: dupobjsarr.append(fluxdata[fluxdata.OBJID == obj]) for i in range(0,len(dupobjsarr)): linefluxsig1.append(dupobjsarr[i].iloc[0][line+'_flux']) linefluxsig2.append(dupobjsarr[i].iloc[1][line+'_flux']) fluxdiffmed = np.median(np.abs(lineflux4-lineflux3)) fluxdiffbi = np.sqrt(biweight_midvariance(np.abs(lineflux4-lineflux3))) absdiffmed = np.median(np.abs(lineflux6-lineflux5)) absdiffbi = np.sqrt(biweight_midvariance(np.abs(lineflux6-lineflux5))) diffarr = [fluxdiffmed, fluxdiffbi, absdiffmed, absdiffbi] return lineflux1,lineflux2, linefluxsig1, linefluxsig2, diffarr
def find_matches(sources, xc, yc): ''' Matching sources using closest neighbor and clipping ''' dx = (sources['xcentroid'][:, np.newaxis] - xc) dy = (sources['ycentroid'][:, np.newaxis] - yc) dd = np.sqrt(dx**2 + dy**2) ind = np.argmin(dd, axis=1) dxk = dx[np.arange(dx.shape[0]), ind] dyk = dy[np.arange(dy.shape[0]), ind] bvx = biweight_midvariance(dxk) bvy = biweight_midvariance(dyk) mvx = biweight_location(dxk) mvy = biweight_location(dyk) sel = np.where( (np.abs(dxk - mvx) < 3. * bvx) * (np.abs(dyk - mvy) < 3. * bvy))[0] return sel, ind[sel]
def calculate_statistics_no_clipping(self): """ This function ... :return: """ # Compress (remove masked values) flattened = np.ma.array(self.sources_with_noise.data, mask=self.rotation_mask.data).compressed() median = np.median(flattened) biweight_loc = biweight_location(flattened) biweight_midvar = biweight_midvariance(flattened) median_absolute_deviation = mad_std(flattened) #print("median", median) #print("biweigth_loc", biweight_loc) #print("biweight_midvar", biweight_midvar) #print("median_absolute_deviation", median_absolute_deviation) self.statistics.no_clipping = Map() self.statistics.no_clipping.median = median self.statistics.no_clipping.biweight_loc = biweight_loc self.statistics.no_clipping.biweight_midvar = biweight_midvar self.statistics.no_clipping.median_absolute_deviation = median_absolute_deviation
def dispersionYasX(x, y, nbins=10): ''' Dispersion of Y as a function of X Parameters ---------- x : array Independent variable y : array Dependent variable nbins : int Number of bins Return ------ dx : array Binned x sigmay : array Dispersion of y ''' dx = np.zeros(nbins) sigmay = np.zeros(nbins) xstep = (np.amax(x) - np.amin(x)) / nbins xbin = np.amin(x) for i in range(nbins): mask = np.logical_and(x >= xbin, x < xbin + xstep) ybin = y[mask] sigmay[i] = np.sqrt(biweight_midvariance(ybin)) #sigmay[i] = np.std(ybin) dx[i] = xbin + 0.5 * xstep xbin += xstep return dx, sigmay
def calculateSeptZPs(): zp = [] for num in range(len(oct_extracted)): thefile = oct_extracted[num] mags, fluxes = [], [] something = np.array(masteroctlines[num]) with open(thefile, 'r') as f: with open(septextracted[0], 'r') as fs: lines = f.readlines() linessept = fs.readlines() for x in lines: if x.startswith("N"): continue elif int(x.split(' ')[0]) in masteroctlines[num]: mags.append(float(x.split(' ')[len(x.split(' ')) - 1])) b = masterseptlines[num][np.where( something == int(x.split(' ')[0]))[0][0]] if int(linessept[b].split(' ')[0]) in septstars: fluxes.append( float(linessept[b].split(' ')[convergesept])) else: mags.pop() for z in range(len(mags)): zp.append(mags[z] + 2.5 * math.log10(fluxes[z])) plt.figure(69) med = median(zp) bwmv = st.biweight_midvariance(zp) plt.hist(zp, bins=15, color='g'), plt.title("September Zeropoints Histogram") plt.text(min(zp), 7, "Med: %s\n BMWV: %s" % (med, bwmv)) if writeup == True: plt.show() plt.savefig('septzeropoints.png'), plt.clf(), plt.close() pm = u'\u00B1' print "Zeropoint for September: %s %s %s" % (med, pm, bwmv)
def robust_std(im, method='biweight'): # get robust stdev # Method can be biweight or mad, otherwise just normal stdev # im must be numpy array from astropy.stats import biweight_midvariance from scipy.stats import median_absolute_deviation if method == 'biweight': var = biweight_midvariance(im, axis=None, ignore_nan=True) std = np.sqrt(var) elif method == 'mad': std = median_absolute_deviation(im, axis=None, nan_policy='omit') else: std = np.nanstd(im, axis=None) # From Mike's code: """ m = np.nanmedian(im) # median value d = im - m # deviation ad = np.abs(d) # absolute deviation mad = np.nanmedian(ad) # median absolute deviation if mad == 0: std = 0. # no deviation -> zero stdev else: wt = biweight(d/1.483/mad) # weights sum_wt = wt.sum() sum_wt2 = (wt**2).sum() m = (im*wt).sum() / sum_wt # weighted mean d = im-m # deviation from weighted mean var = (d**2 * wt).sum() / (sum_wt-sum_wt2/sum_wt) # weighted var std = n.sqrt(var) # weighted stdev """ return std
def calculateOctZPs(): global octzp, octzperr twommatch, octmatch, Mag, flux, octlineno, chipno = [], [], [], [], [], [] with open("2mass_Oct_matched.txt", 'r') as f: lines = f.readlines() for x in lines: x = x.strip() line = x.split() if not x or x.startswith('#'): continue twommatch.append((int(line[4]) - 1)) octmatch.append((int(line[5]) - 1)) if len(twommatch) != len(set(twommatch)) or len(octmatch) != len( set(octmatch)): print "Error: tweak xyxymatch parameters so that a coordinate in one set of data isn't matched to " \ "multiple coordinates in the other." sys.exit() with open('2mass_RADec.txt', 'r') as f: temptwo_m = np.array(twommatch) with open('Oct_RADec.txt', 'r') as fs: lines = f.readlines() linesoct = fs.readlines() i = 1 for x in lines: z = x.split('#') if x.startswith("#"): continue elif i in twommatch: b = octmatch[np.where(temptwo_m == i)[0][0]] Mag.append(z[1]) p = linesoct[b].split(' ') flux.append(p[4].replace('\n', '')) octlineno.append(p[2]) chipno.append(p[3]) i += 1 zp = [] for num in range(0, len(Mag)): if float(flux[num]) < 1: continue zp.append( float(Mag[num]) + float(2.5 * (math.log10(float(flux[num]))))) # if float(Mag[num]) + float(2.5 * (math.log10(float(flux[num])))) < 26.4: # print flux[num] octzp, octzperr = median(zp), st.biweight_midvariance(zp) print "%s zeropoints calculated in October" % (len(zp)) plt.figure(30), plt.hist( zp, bins=15), plt.title("October Zeropoints Histogram") plt.text(min(zp), len(zp) / 5, "Med: %s \n BWMV: %s" % (octzp, octzperr)) axes = plt.gca() # axes.set_xlim([25, 28]) if writeup == True: plt.show() plt.savefig('oct_zps.png', bbox_inches='tight'), plt.close()
def binMeanStd(x, y, nbins=10): ''' Compute binned median and standard deviation Parameters ---------- x : array Independent variable y : array Dependent variable nbins : int Number of bins Return ------ data : array Binned medean and standard deviations for both x and y ''' #----------------------------------------------------------------------------------------- # Sort x and y according to x indc = np.argsort(x) xsort, ysort = x[indc], y[indc] # Compute mean and standard deviation for each bin nstars = len(xsort) nstars_bin = int(nstars / nbins + 1.) data = np.zeros((nbins, 4)) for i in range(nbins - 1): data[i, 0] = np.median(xsort[i * nstars_bin:(i + 1) * nstars_bin]) data[i, 1] = np.median(ysort[i * nstars_bin:(i + 1) * nstars_bin]) data[i, 2] = np.sqrt( biweight_midvariance(xsort[i * nstars_bin:(i + 1) * nstars_bin])) data[i, 3] = np.sqrt( biweight_midvariance(ysort[i * nstars_bin:(i + 1) * nstars_bin])) data[nbins - 1, 0] = np.median(xsort[(nbins - 1) * nstars_bin:nstars]) data[nbins - 1, 1] = np.median(ysort[(nbins - 1) * nstars_bin:nstars]) data[nbins - 1, 2] = np.sqrt( biweight_midvariance(xsort[(nbins - 1) * nstars_bin:nstars])) data[nbins - 1, 3] = np.sqrt( biweight_midvariance(ysort[(nbins - 1) * nstars_bin:nstars])) return data
def findvdisp(self, r, v, r200, maxv): """ Use astropy.stats biweight sigma clipping variance for the velocity dispersion """ v_cut = v[np.where((r < r200) & (np.abs(v) < maxv))] try: self.gal_vdisp = biweight_midvariance( v_cut[np.where(np.isfinite(v_cut))], 9.0) except: self.gal_vdisp = np.std(v_cut, ddof=1)
def _stats_data(self, stats, mask, scipy, astropy, decimals_mode): data = self.data # The original data size, for computation of valid elements and how # many are masked/invalid. size_initial = data.size # Delete masked values, this will directly convert it to a 1D array # if the mask is not appropriate then ravel it. data = data[~mask] size_masked = data.size # Delete invalid (NaN, Inf) values. This should ensure that the result # is always a 1D array data = data[np.isfinite(data)] size_valid = data.size stats['elements'] = [size_valid] stats['min'] = [np.amin(data)] stats['max'] = [np.amax(data)] stats['mean'] = [np.mean(data)] stats['median'] = [np.median(data)] # Use custom mode defined in this package because scipy.stats.mode is # very, very slow and by default tries to calculate the mode along # axis=0 and not for the whole array. # Take the first element since the second is the number of occurences. stats['mode'] = [mode(data, decimals=decimals_mode)[0]] if astropy: stats['biweight_location'] = [biweight_location(data)] stats['std'] = [np.std(data)] if astropy: stats['mad'] = [mad_std(data)] stats['biweight_midvariance'] = [biweight_midvariance(data)] stats['var'] = [np.var(data)] if scipy: # pragma: no cover if not OPT_DEPS['SCIPY']: log.info('SciPy is not installed.') else: # Passing axis=None should not be important since we already # boolean indexed the array and it's 1D. But it's important # to remember that there default is axis=0 and not axis=None! stats['skew'] = [skew(data, axis=None)] stats['kurtosis'] = [kurtosis(data, axis=None)] stats['masked'] = [size_initial - size_masked] stats['invalid'] = [size_masked - size_valid] return data
def fboot(y): """ Helper function of trendplot. Returns bootstrap error of input 1d-array. Calculates the biweight mean of each bootstrap sample, and then gets the biweight standard deviation of all samples. """ if len(y) > 5: bb = bootstrap(y, bootnum=250, bootfunc=biweight_location) res = biweight_midvariance(bb) else: res = np.float("nan") return res
def calc_scat(self, mass, observable, loglog=True, binobs=True, step=0.05, data_cut=None): ''' Does Mass Mixing Statistics ''' ## First Calculate Scatter of Richness Relationship # Get Data Cut cut = np.where(observable > 0) if obs_cut == None: print ' Find observable value above which vertical scatter is fully encompassed' mp.plot(np.log(observable[cut]), np.log(true_mass[cut]), 'ko', alpha=.75) mp.xlabel('Observable', fontsize=16) mp.ylabel('M200', fontsize=16) mp.show() obs_cut = input('What is that value? (Enter as a float, ex. 1.5) : ') # Correct obs_cut if type(obs_cut) == int or type(obs_cut) == float: obs_cut = np.array([obs_cut]) # Now Calculate Scatter SCAT = [] for i in range(len(obs_cut)): data_cut = np.where(np.log(observable) > obs_cut[i])[0] print 'Number of Data Points:', len(data_cut) step = step obs_max = np.log(observable[data_cut]).max() bins = np.arange(obs_cut[i], obs_max, step) mass_bins = np.array( map( lambda x: (true_mass[data_cut])[np.where( (np.log(observable[data_cut]) > x) & (np.log(observable[data_cut]) < (x + step)))], bins)) mass_avgs = np.array(map(lambda x: np.median(x), mass_bins)) mass_fracs = [] for j in range(len(mass_avgs)): mass_fracs.extend(np.log(mass_bins[j] / mass_avgs[j])) mass_fracs = np.array(mass_fracs) scat = astats.biweight_midvariance(mass_fracs) SCAT.append(scat) return np.array(SCAT)
def chisq_metric(self, chisq, chisq_std_zscore_cut=4.0, return_dict=True): """ chi square metrics chisq : ndarray, shape=(Nants, Ntimes, Nfreqs) ndarray containing chisq for each antenna (single pol) chisq_std_cut : float, default=5.0 sigma tolerance (or z-score tolerance) for std of chisq fluctuations return_dict : bool, default=True return per-antenna metrics as a dictionary, with antenna number as key rather than an ndarray """ # Get chisq statistics chisq_avg = np.median(np.median(chisq[:, :, self.band], axis=0), axis=1).astype(np.float64) chisq_tot_avg = astats.biweight_location( chisq[:, :, self.band]).astype(np.float64) chisq_ant_avg = np.array( list(map(astats.biweight_location, chisq[:, :, self.band]))).astype(np.float64) chisq_ant_std = np.sqrt( np.array( list(map(astats.biweight_midvariance, chisq[:, :, self.band])))) chisq_ant_std_loc = astats.biweight_location(chisq_ant_std).astype( np.float64) chisq_ant_std_scale = np.sqrt( astats.biweight_midvariance(chisq_ant_std)) chisq_ant_std_zscore = (chisq_ant_std - chisq_ant_std_loc) / chisq_ant_std_scale chisq_ant_std_zscore_max = np.max(np.abs(chisq_ant_std_zscore)) chisq_good_sol = chisq_ant_std_zscore_max < chisq_std_zscore_cut # convert to dictionaries if return_dict is True: chisq_ant_std = odict(zip(self.ant_array, chisq_ant_std)) chisq_ant_avg = odict(zip(self.ant_array, chisq_ant_avg)) return (chisq_avg, chisq_tot_avg, chisq_ant_avg, chisq_ant_std, chisq_ant_std_loc, chisq_ant_std_scale, chisq_ant_std_zscore, chisq_ant_std_zscore_max, chisq_good_sol)
def _pixel_stats(pixels, clip_sig=3, n_clip=10, min_pix=50): """ Calculate mode and scale of pixel distribution, clipping outliers. Uses biweight as "robust" estimator of these quantities. :param pixels: Array to calculate statistics for :param clip_sig: Sigma value at which to clip outliers :param n_clip: Number of clipping iterations :param min_pix: Minimum number of retained pixels :return: 2-tuple of distribution mode, scale """ clip_iter = 0 mode, scale = 0, 1 mask = np.ones(pixels.shape, dtype=bool) while True: mode = biweight_location(pixels[mask]) scale = biweight_midvariance(pixels[mask]) mask &= np.abs(pixels - mode) < clip_sig * scale clip_iter += 1 if np.sum(mask) < min_pix or clip_iter >= n_clip: break return mode, scale
def calc_scat(self,mass,observable,loglog=True,binobs=True,step=0.05,data_cut=None): ''' Does Mass Mixing Statistics ''' ## First Calculate Scatter of Richness Relationship # Get Data Cut cut = np.where(observable > 0) if obs_cut == None: print ' Find observable value above which vertical scatter is fully encompassed' mp.plot(np.log(observable[cut]),np.log(true_mass[cut]),'ko',alpha=.75) mp.xlabel('Observable',fontsize=16) mp.ylabel('M200',fontsize=16) mp.show() obs_cut = input('What is that value? (Enter as a float, ex. 1.5) : ') # Correct obs_cut if type(obs_cut) == int or type(obs_cut) == float: obs_cut = np.array([obs_cut]) # Now Calculate Scatter SCAT = [] for i in range(len(obs_cut)): data_cut = np.where(np.log(observable) > obs_cut[i])[0] print 'Number of Data Points:',len(data_cut) step = step obs_max = np.log(observable[data_cut]).max() bins = np.arange(obs_cut[i],obs_max,step) mass_bins = np.array( map(lambda x: (true_mass[data_cut])[np.where((np.log(observable[data_cut])>x)&(np.log(observable[data_cut])<(x+step)))], bins)) mass_avgs = np.array(map(lambda x:np.median(x),mass_bins)) mass_fracs = [] for j in range(len(mass_avgs)): mass_fracs.extend(np.log(mass_bins[j] / mass_avgs[j])) mass_fracs = np.array(mass_fracs) scat = astats.biweight_midvariance(mass_fracs) SCAT.append(scat) return np.array(SCAT)
def richness_est(self,ra,dec,z,pz,gmags,rmags,imags,abs_rmags,haloid,clus_ra,clus_dec,clus_z,clus_rvir=None,gr_slope=None,gr_width=None,gr_inter=None,shiftgap=True,use_specs=False,use_bcg=False,fit_rs=False,spec_list=None,fixed_aperture=False,fixed_vdisp=False,plot_gr=False,plot_sky=False,plot_phase=False,find_pairs=True): ''' Richness estimator, magnitudes should be apparent mags except for abs_rmag z : spectroscopic redshift pz : photometric redshift gr_slope : if fed color-mag (g-r vs. r) slope of fit to red sequence gr_width : if fed color-mag (g-r vs. r) width of fit to red sequence gr_inter : if red color-mag (g-r vs. r) intercept of fit to red sequence spec_list : indexing array specifying gals w/ spectroscopic redshifts, preferably fed as boolean numpy array fixed_aperture : clus_rvir = 1 Mpc fixed_vdisp : clus_vdisp = 1000 km/s (members < clus_vdisp*2) use_specs : includes all spectro members in richness regardless of their color, subject to counting twice use_bcg : use bcg RS_color as cluster RS_color fit_rs : fit a line to the member galaxy RS relationship, as of now this does not work and we take a flat RS relationship plot_gr : plot r vs g-r color diagram with RS fits plot_sky : plot RA and DEC of gals with signal and background annuli plot_phase : plot rdata and vdata phase space find_pairs : run c4 cluster pair identifier on phase spaces - If at least one quarter of the outer annuli doesn't have any galaxies (perhaps because it has run over the observation edge), it will return -99 as a richness ''' # Put Into Class Namespace keys = ['ra','dec','z','pz','gmags','rmags','imags','clus_ra','clus_dec','clus_z','clus_rvir','vel_disp','color_data','color_cut','RS_color','RS_sigma','clus_color_cut','signal','background','richness','mems','phot','spec','spec_mems','deg_per_rvir','SA_outer','SA_inner','outer_red_dense','inner_background','Nphot_inner','Nphot_outer','red_inner','red_outer','all','outer_edge','inner_edge','shift_cut','set','pair','Nspec','gr_slope','gr_inter','gr_width'] self.__dict__.update(ez.create(keys,locals())) # Take rough virial radius measurement, this method is BAD... if clus_rvir == None: clus_rvir = np.exp(-1.86)*len(np.where((rmags < -19.55) & (rdata < 1.0) & (np.abs(vdata) < 3500))[0])**0.51 self.clus_rvir = clus_rvir # Set clus_rvir = 1.0 Mpc if fixed aperture == True if fixed_aperture == True: clus_rvir = 1.0 # Magnitue Cut at 19.1 Apparent R Mag, then at -19 Absolute R Mag and sort by them bright = np.where(rmags < 19.1)[0] data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags]) data = data.T[bright].T ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data bright = np.where(abs_rmags < -19.5)[0] data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags]) data = data.T[bright].T ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data sorts = np.argsort(abs_rmags) data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags]) data = data.T[sorts].T ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data self.__dict__.update(ez.create(keys,locals())) # Separate Into Spectro Z and Photo Z DataSets # Define the Spectro Z catalogue if spec_list == None: spec_cut = np.abs(z) > 1e-4 else: if type(spec_list) == list: spec_list = np.array(spec_list) if spec_list.dtype != 'bool': spec_cut = np.array([False]*len(ra)) spec_cut[spec_list] = True all = {'ra':ra,'dec':dec,'z':z,'pz':pz,'gmags':gmags,'rmags':rmags,'imags':imags,'abs_rmags':abs_rmags} spec = {'ra':ra[spec_cut],'dec':dec[spec_cut],'z':z[spec_cut],'pz':pz[spec_cut],'gmags':gmags[spec_cut],'rmags':rmags[spec_cut],'imags':imags[spec_cut],'abs_rmags':abs_rmags[spec_cut]} phot = {'ra':ra[~spec_cut],'dec':dec[~spec_cut],'z':z[~spec_cut],'pz':pz[~spec_cut],'gmags':gmags[~spec_cut],'rmags':rmags[~spec_cut],'imags':imags[~spec_cut],'abs_rmags':abs_rmags[~spec_cut]} # Project Spectra into radius and velocity ang_d,lum_d = C.zdistance(clus_z,self.H0) angles = C.findangle(spec['ra'],spec['dec'],clus_ra,clus_dec) rdata = angles * ang_d vdata = self.c * (spec['z'] - clus_z) / (1 + clus_z) spec.update({'rdata':rdata,'vdata':vdata}) self.__dict__.update(ez.create(keys,locals())) # Take Hard Phasespace Limits limit = np.where( (rdata < 5) & (np.abs(vdata) < 5000) )[0] clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']]) clus_data = clus_data.T[limit].T rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data spec.update({'rdata':rdata,'vdata':vdata}) self.__dict__.update(ez.create(keys,locals())) # Shiftgapper for Interlopers if shiftgap == True: len_before = np.where((rdata < clus_rvir*1.5)&(np.abs(vdata)<4000))[0].size clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']]) clus_data = C.shiftgapper(clus_data.T).T sorts = np.argsort(clus_data[-1]) clus_data = clus_data.T[sorts].T rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data shift_cut = len_before - np.where((rdata < clus_rvir)&(np.abs(vdata)<4000))[0].size # Measure Velocity Dispersion of all galaxies within 1 * r_vir and np.abs(vdata) < 4000 km/s spec.update({'rdata':rdata,'vdata':vdata}) self.__dict__.update(ez.create(keys,locals())) vel_disp = astats.biweight_midvariance(vdata[np.where((rdata < clus_rvir*1)&(np.abs(vdata) < 4000))]) if fixed_vdisp == True: vel_disp = 1000 # Run Cluster Pair Finder if find_pairs == True: pair, d1_chi, d2_chi, d3_chi, s_chi, double1, double2, double3, single, v_range, bins = pair_find(rdata,vdata) # Calculate Nspec, Nspec = # of galaxies within RVIR try: Nspec = np.where(rdata<clus_rvir)[0].size except: Nspec = 0 self.__dict__.update(ez.create(keys,locals())) # Get Members from Specta, get their red sequence color mems = np.where((spec['rdata'] < clus_rvir)&(np.abs(spec['vdata'])<2*vel_disp))[0] color_data = spec['gmags'] - spec['rmags'] color_cut = np.where((color_data[mems] < 1.2) & (color_data[mems] > 0.65))[0] RS_color = astats.biweight_location(color_data[mems][color_cut]) RS_shift = color_data[mems][color_cut] - RS_color RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)]) if fit_rs == True: clf = linear_model.LinearRegression() set = np.where(np.abs(color_data[mems]-RS_color)<3*RS_sigma)[0] clf.fit(spec['rmags'][mems][set].reshape(set.size,1),color_data[mems][set]) if use_bcg == True: bright = np.argsort(all['abs_rmags'][mems])[0] RS_color = color_data[mems][bright] RS_shift = color_data[mems][color_cut] - RS_color RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)]) # spec_mems is # of members that are within 2 sigma of cluster color clus_color_cut = np.where(np.abs(color_data[mems] - RS_color) < RS_sigma*2)[0] spec_mems = len(clus_color_cut) self.__dict__.update(ez.create(keys,locals())) # If fed gr fit if gr_slope != None: def RS_color(r_mag): return r_mag*gr_slope + gr_inter RS_sigma = gr_width / 2 clus_color_cut = np.where(np.abs(color_data[mems] - RS_color(spec['rmags'])[mems]) < RS_sigma*2)[0] spec_mems = len(clus_color_cut) self.__dict__.update(ez.create(keys,locals())) # Get Rdata from PhotoZ & SpecZ Data Set angles = C.findangle(all['ra'],all['dec'],clus_ra,clus_dec) all['rdata'] = angles * ang_d # Get deg per RVIR proper deg_per_rvir = R.Cosmo.arcsec_per_kpc_proper(clus_z).value * 1e3 * clus_rvir / 3600 # Get Area of Virial Circle out to 1 rvir, and Area of outer annuli, which is annuli from 4rvir < R < 6rvir, or dependent on rvir if clus_rvir < 2.5: outer_edge = 6.0 inner_edge = 4.0 elif clus_rvir >= 2.5 and clus_rvir < 3: outer_edge = 5.0 inner_edge = 3.5 else: outer_edge = 3.0 inner_edge = 2.0 SA_inner = np.pi*deg_per_rvir**2 SA_outer = np.pi * ( (outer_edge*deg_per_rvir)**2 - (inner_edge*deg_per_rvir)**2 ) # Get Number of Cluster Color Galaxies from Photo Data Set in Inner Circle and Outer Annuli RS_color_sig = 2.0 if gr_slope == None: red_inner = np.where(((all['gmags']-all['rmags']) < RS_color + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0] red_outer = np.where(((all['gmags']-all['rmags']) < RS_color + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0] else: red_inner = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0] red_outer = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0] Nphot_inner = len(red_inner) Nphot_outer = len(red_outer) self.__dict__.update(ez.create(keys,locals())) # Get Solid Angle Density of Outer Red Galaxies outer_red_dense = Nphot_outer / SA_outer inner_background = int(np.ceil(outer_red_dense * SA_inner)) # If inner_background is less than Nphot_inner, then Nphot_inner -= inner_background, otherwise Nphot_inner = 0 if inner_background < Nphot_inner: Nphot_inner -= inner_background else: Nphot_inner = 0 # Richness = spec_mems + Nphot_inner or just Nphot_inner if use_specs == True: richness = spec_mems + Nphot_inner else: richness = Nphot_inner self.__dict__.update(ez.create(keys,locals())) # Plot if plot_gr == True: fig,ax = mp.subplots() ax.plot(rmags,gmags-rmags,'ko',alpha=.8) ax.plot(spec['rmags'][mems],color_data[mems],'co') if gr_slope == None: ax.axhline(RS_color,color='r') ax.axhline(RS_color+RS_sigma,color='b') ax.axhline(RS_color-RS_sigma,color='b') else: xdata = np.arange(spec['rmags'][mems].min(),spec['rmags'][mems].max(),.1) ax.plot(xdata,RS_color(xdata),color='r') ax.plot(xdata,RS_color(xdata)+RS_sigma,color='b') ax.plot(xdata,RS_color(xdata)-RS_sigma,color='b') ax.set_xlim(13,19) ax.set_ylim(0,1.3) ax.set_xlabel('Apparent R Mag',fontsize=16) ax.set_ylabel('App G Mag - App R Mag',fontsize=16) ax.set_title('Color-Mag Diagram, Cluster '+str(haloid)) fig.savefig('colormag_'+str(haloid)+'.png',bbox_inches='tight') mp.close(fig) if plot_sky == True: fig,ax = mp.subplots() ax.plot(all['ra'],all['dec'],'ko') ax.plot(all['ra'][red_inner],all['dec'][red_inner],'ro') ax.plot(all['ra'][red_outer],all['dec'][red_outer],'yo') ax.plot(clus_ra,clus_dec,'co',markersize=9) ax.set_xlabel('RA',fontsize=15) ax.set_ylabel('Dec.',fontsize=15) ax.set_title('Richness Annuli for Halo '+str(haloid)) fig.savefig('skyplot_'+str(haloid)+'.png',bbox_inches='tight') mp.close(fig) if plot_phase == True: fig,ax = mp.subplots() ax.plot(spec['rdata'],spec['vdata'],'ko') ax.plot(spec['rdata'][mems],spec['vdata'][mems],'co') bcg = np.where(spec['abs_rmags']==spec['abs_rmags'][mems].min())[0][0] ax.plot(spec['rdata'][bcg],spec['vdata'][bcg],'ro') ax.set_xlim(0,5) ax.set_ylim(-5000,5000) ax.set_xlabel('Radius (Mpc)',fontsize=15) ax.set_ylabel('Velocity (km/s)',fontsize=15) ax.set_title('phasespace haloid '+str(haloid)) fig.savefig('phasespace_'+str(haloid)+'.png',bbox_inches='tight') mp.close(fig) # Check to make sure galaxies exist (somewhat) uniformely in outer annuli (aka. cluster isn't on edge of observation strip) # First, project all galaxies into polar coordinates centered on cluster center x = (all['ra']-clus_ra)/np.cos(clus_dec*np.pi/180) # x coordinate in arcsec (of dec) centered on cluster center y = (all['dec']-clus_dec) all['radius'] = np.sqrt( x**2 + y**2 ) / deg_per_rvir #radius scaled by RVIR all['theta'] = np.arctan( y / x ) # Add corrections to arctan function all['theta'][np.where( (x < 0) & (y > 0) )] += np.pi # Quadrant II all['theta'][np.where( (x < 0) & (y < 0) )] += np.pi # Quadrant III all['theta'][np.where( (x > 0) & (y < 0) )] += 2*np.pi # Quadrant IV # Then break outer annuli into 4 sections and check if at least 1 galaxy exists in each section sizes1 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(0,2*np.pi,4)]) # Do it again but shift theta by np.pi/8 this time sizes2 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(np.pi/8,17*np.pi/8,4)]) if 0 in sizes1 or 0 in sizes2: mp.plot(all['radius'],all['theta'],'ko') mp.xlabel('radius') mp.ylabel('theta') mp.savefig('rth_'+str(haloid)+'.png') mp.close() print 'sizes1=',sizes1 print 'sizes2=',sizes2 return -99 return richness
loc3 = np.sin(67.5 / 180 * np.pi) mr1 = (fluxdata[filters]['ar'] < loc1) mr2 = np.logical_and(fluxdata[filters]['ar'] >= loc1, fluxdata[filters]['ar'] < loc2) mr3 = np.logical_and(fluxdata[filters]['ar'] >= loc2, fluxdata[filters]['ar'] < loc3) mr4 = (fluxdata[filters]['ar'] >= loc3) med1 = np.median(fluxdata[filters][mr1].av) med2 = np.median(fluxdata[filters][mr2].av) med3 = np.median(fluxdata[filters][mr3].av) med4 = np.median(fluxdata[filters][mr4].av) med751 = np.percentile(fluxdata[filters][mr1].av, 75) med752 = np.percentile(fluxdata[filters][mr2].av, 75) med753 = np.percentile(fluxdata[filters][mr3].av, 75) med754 = np.percentile(fluxdata[filters][mr4].av, 75) emed1 = np.sqrt(biweight_midvariance(fluxdata[filters][mr1].av)) emed2 = np.sqrt(biweight_midvariance(fluxdata[filters][mr2].av)) emed3 = np.sqrt(biweight_midvariance(fluxdata[filters][mr3].av)) emed4 = np.sqrt(biweight_midvariance(fluxdata[filters][mr4].av)) ax.errorbar(med1, loc1 / 2, xerr=(emed1 / len(fluxdata[filters][mr1])), color='red', marker='o', ms=ms, lw=lwbw, ls='None') ax.errorbar(med2, (loc1 + loc2) / 2, xerr=(emed2 / len(fluxdata[filters][mr2])), color='red', marker='o',
ticksize = 16 titlefont = 24 legendfont = 16 textfont = 16 fig, axarr5 = plt.subplots(3, 4, figsize=(30, 20)) axarr5 = np.reshape(axarr5, 12) counter = 0 for line in lines: #Compute the scale dataframe and make a plot to display it ax = axarr5[counter] medscale = np.median( np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line + '_scale'])) sigscale = np.sqrt( biweight_midvariance( np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line + '_scale']))) scale_df.at[0, line + '_medscale'] = medscale scale_df.at[0, line + '_sigscale'] = sigscale #Make the plot bins = np.log10(np.arange(0.05, 4, 0.05)) ax.hist(np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line + '_scale']), bins=bins, color='grey', label=None) #Plot the median, 1sig, and 3 sig stddevs ax.plot((medscale, medscale), (-100, 1000), color='red', ls='-', label='Median') ax.plot((medscale - sigscale, medscale - sigscale), (-100, 1000),
def histogram2(band, band_ref, best_weights, training_data, aeg_models, chisquare_models): weights = best_weights[-5, 0:len(band)] Z_bef = np.zeros((len(training_data))) age_bef = np.zeros((len(training_data))) ebv_bef = np.zeros((len(training_data))) Z_aft = np.zeros((len(training_data))) age_aft = np.zeros((len(training_data))) ebv_aft = np.zeros((len(training_data))) plt.subplots(figsize=(15, 7)) for j in range(0, len(training_data)): # chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.ones((len(band)))) chi2, chi1 = cs.full_fit( cs.load_table('chisquare_models_test.dat'), cs.load_table('training_data.dat'), j, ([ 'uJava', 'uJava', 'uJava', 'uJava', 'F378', 'F378', 'F430', 'gSDSS', 'F660', 'F861', 'uJava' ]), ([ 'F660', 'rSDSS', 'iSDSS', 'zSDSS', 'F515', 'gSDSS', 'gSDSS', 'zSDSS', 'gSDSS', 'uJava', 'F515' ]), cs.load_table('best5_100age.dat')[-5, 0:11], # fixar os pesos referentes as cores da idade antes e depois do treino para só ver a # influência dos pesos das cores da metalicidade # np.ones(11), ([ 'iSDSS', 'rSDSS', 'uJava', 'F378', 'F410', 'F430', 'F515', 'F660', 'F660', 'gSDSS', 'iSDSS' ]), ([ 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'iSDSS', 'zSDSS', 'F861', 'iSDSS', 'zSDSS', 'F861' ]), np.ones(11)) # cs.load_table('best5_100Z_test2.dat')[-5, 0:11]) Z_bef[j] = chi2[0] - aeg_models[j, 0] age_bef[j] = (chi2[1] - aeg_models[j, 1]) / 10**9 ebv_bef[j] = chi2[2] - aeg_models[j, 2] # chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.array([weights])) chi2, chi1 = cs.full_fit( cs.load_table('chisquare_models_test.dat'), cs.load_table('training_data.dat'), j, ([ 'uJava', 'uJava', 'uJava', 'uJava', 'F378', 'F378', 'F430', 'gSDSS', 'F660', 'F861', 'uJava' ]), ([ 'F660', 'rSDSS', 'iSDSS', 'zSDSS', 'F515', 'gSDSS', 'gSDSS', 'zSDSS', 'gSDSS', 'uJava', 'F515' ]), cs.load_table('best5_100age.dat')[-5, 0:11], # np.ones(11), ([ 'iSDSS', 'rSDSS', 'uJava', 'F378', 'F410', 'F430', 'F515', 'F660', 'F660', 'gSDSS', 'iSDSS' ]), ([ 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'iSDSS', 'zSDSS', 'F861', 'iSDSS', 'zSDSS', 'F861' ]), # np.ones(11)) cs.load_table('best5_100Z_test4.dat')[-5, 0:11]) Z_aft[j] = chi2[0] - aeg_models[j, 0] age_aft[j] = (chi2[1] - aeg_models[j, 1]) / 10**9 ebv_aft[j] = chi2[2] - aeg_models[j, 2] # N=20 plt.subplot(1, 3, 1) plt.hist(Z_bef, bins=np.arange(-0.055, 0.055, 0.01), color='#aaaa22', rwidth=0.95) bins_Z_aft, Z_aft_data = histOutline(Z_aft, bins=np.arange(-0.055, 0.055, 0.01)) plt.plot(bins_Z_aft, Z_aft_data, '-', color="#1B6AC6", lw=4) # plt.hist(Z_aft, bins=np.arange(-0.055, 0.055, 0.01), color='#aaab22', rwidth=0.95) plt.xlabel(r'$\Delta$Z' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(Z_bef), biweight_midvariance(Z_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(Z_aft), biweight_midvariance(Z_aft))) plt.xlim(-0.05, 0.05) plt.subplot(1, 3, 2) plt.hist(age_bef, bins=np.arange(-1.05, 1.05, 0.1), color='#aaaa22', rwidth=0.95) bins_age_aft, age_aft_data = histOutline(age_aft, bins=np.arange(-1.05, 1.05, 0.1)) plt.plot(bins_age_aft, age_aft_data, '-', color="#1B6AC6", lw=4) # plt.hist(age_aft, bins=np.arange(-1.05, 1.05, 0.1), color='#aaaa22', rwidth=0.95) plt.xlabel(r'$\Delta$age/$10^{9}$' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(age_bef), biweight_midvariance(age_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(age_aft), biweight_midvariance(age_aft))) plt.xlim(-0.5, 0.5) plt.subplot(1, 3, 3) plt.hist(ebv_bef, bins=np.arange(-0.45, 0.45, 0.05), label='Before training', color='#aaaa22', rwidth=0.95) bins_ebv_aft, ebv_aft_data = histOutline(ebv_aft, bins=np.arange(-0.45, 0.45, 0.05)) plt.plot(bins_ebv_aft, ebv_aft_data, '-', color="#1B6AC6", lw=4, label='After training') # plt.hist(ebv_aft, bins=np.arange(-0.4, 0.4, 0.05), label='After training', color='#aaaa22', rwidth=0.95) plt.xlabel('$\Delta$E(B-V)' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(ebv_bef), biweight_midvariance(ebv_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f}, \zeta={1:1.6f}$'.format( np.mean(ebv_aft), biweight_midvariance(ebv_aft))) # plt.xlim(-5,2) plt.legend() # plt.suptitle('Histograms') plt.tight_layout() # plt.savefig('histogram_Z&Age.png') plt.show()
def biweight_midvariance(self): """ The biweight midvariance of the pixel values. """ return biweight_midvariance(self.goodvals)
def makeplot(month): flux_aper, fwhm = [], [] if month == 'oct': for num in range(len(octplot)): thefile = octplot[num] with open(thefile, 'r') as f: lines = f.readlines() for x in lines: line = x.split(' ') flux_aper.append((-2.5 * (math.log10(float(line[24]))))) fwhm.append(line[5]) plt.figure(num) plt.plot(flux_aper, fwhm, 'ro') plt.title( 'c%s Flux vs. FWHM' % (num + 1)), plt.xlabel('-2.5*log_10(flux)'), plt.ylabel('FWHM') axes = plt.gca() axes.set_xlim([-16, -2]), axes.set_ylim([0, 20]) del flux_aper[:], fwhm[:] for line in fileinput.input(thefile, inplace=1): x = line.split(' ') if (not line.startswith('N')) and (minlogflux < (-2.5 * (math.log10(float(x[24])))) < maxlogflux) and \ float(x[5]) < maxfwhm: flux_aper.append((-2.5 * (math.log10(float(x[24]))))) fwhm.append(float(x[5])) sys.stdout.write(line) med = median(fwhm) sig = st.biweight_midvariance(fwhm) del flux_aper[:], fwhm[:] for line in fileinput.input(thefile, inplace=1): x = line.split(' ') if (not line.startswith('N')) and ( (med - sigmamultiple[num] * sig) < float(x[5]) < (med + sigmamultiple[num] * sig)): flux_aper.append((-2.5 * (math.log10(float(x[24]))))) fwhm.append(x[5]) sys.stdout.write(line) plt.plot(flux_aper, fwhm, 'bo') del flux_aper[:], fwhm[:] plt.savefig(('%s_flux vs. fwhm.png' % ('c' + str(num + 1))), bbox_inches='tight') if writeup == True: plt.show() plt.clf() plt.close() elif month == 'sept': thefile = septclean[0] with open(thefile, 'r') as f: lines = f.readlines() for x in lines: line = x.split(' ') flux_aper.append((-2.5 * (math.log10(float(line[26]))))) fwhm.append(line[7]) if float(line[7]) > 80: print line[7] print line[0] plt.figure(1000) plt.plot(flux_aper, fwhm, 'ro') plt.title('Sept Flux vs. FWHM'), plt.xlabel( '-2.5*log_10(flux)'), plt.ylabel('FWHM') axes = plt.gca() # axes.set_xlim([-15, -2]), axes.set_ylim([0, 20]) del flux_aper[:], fwhm[:] for line in fileinput.input(thefile, inplace=1): x = line.split(' ') if (not line.startswith('N')) and ( -15 < (-2.5 * (math.log10(float(x[26])))) < -10): flux_aper.append((-2.5 * (math.log10(float(x[26]))))) fwhm.append(float(x[7])) sys.stdout.write(line) med = median(fwhm) sig = st.biweight_midvariance(fwhm) del flux_aper[:], fwhm[:] for line in fileinput.input(thefile, inplace=1): x = line.split(' ') if ((med - 1.5 * sig) < float(x[7]) < (med + 1.5 * sig)): flux_aper.append((-2.5 * (math.log10(float(x[26]))))) fwhm.append(x[7]) sys.stdout.write(line) plt.plot(flux_aper, fwhm, 'bo') del flux_aper[:], fwhm[:] plt.savefig((sept + 'Sept flux vs. fwhm.png'), bbox_inches='tight') if writeup == True: plt.show() plt.clf() plt.close()
def run_caustic(self, cluster_data): S = CausticSurface(self.cosmo) results = CausticFitResults() results.cosmo = self.cosmo clus_z, r200 = cluster_data.clus_z, cluster_data.r200 v, gal_memberflag, gal_mags = cluster_data.v, cluster_data.gal_memberflag, cluster_data.gal_mags data_table = cluster_data.data_spec if cluster_data.r is None: # calculate angular diameter distance. # Variable self.ang_d ang_d = self.cosmo.angular_diameter_distance(clus_z) # calculate the spherical angles of galaxies from cluster center. # Variable self.angle angle = self.findangle(data_table['RA'], data_table['DEC'], cluster_data.clus_ra, cluster_data.clus_dec) r = angle * ang_d else: r = cluster_data.r # calculate H(z) Hz = self.cosmo.H(clus_z).value # self.cosmo.h*100*np.sqrt(0.25*(1+self.clus_z)**3 + 0.75) # hz = self.Hz/100 #self.Hz / 100.0 #little h(z) # package galaxy data, USE ASTROPY TABLE HERE!!!!! rcol = Table.Column(data=r, name='rad') vcol = Table.Column(data=v, name='vel') data_table.add_column(rcol, 0) data_table.add_column(vcol, 1) if gal_memberflag is not None: memflagcol = Table.Column(data=gal_memberflag, name='MEMFLAG') data_table.add_column(memflagcol) if gal_mags is not None: galmagcol = Table.Column(data=gal_mags, name='MAG') data_table.add_column(galmagcol) # reduce sample within limits if self.cut_sample is True: data_set = self.set_sample(data=data_table, rlimit=self.rlimit, vlimit=self.vlimit) else: data_set = data_table data_set = data_set.to_pandas() # further select sample via shifting gapper if self.gapper is True: data_set = self.shiftgapper(data_set) print('DATA SET SIZE', len(data_set)) ##tries to identify double groups that slip through the gapper process # upper_max = np.max(data_set[:,1][np.where((data_set[:,1]>0.0)&(data_set[:,0]<1.0))]) # lower_max = np.min(data_set[:,1][np.where((data_set[:,1]<0.0)&(data_set[:,0]<1.0))]) # if np.max(np.array([upper_max,-lower_max])) > 1000.0+np.min(np.array([upper_max,-lower_max])): # data_set = data_set[np.where(np.abs(data_set[:,1])<1000.0+np.min(np.array([upper_max,-lower_max])))] # measure Ngal above mag limit # question shouldnt this also be cut by the shiftgapper? try: if cluster_data.abs_flag: abs_mag = data_table['MAG'] else: abs_mag = data_table['MAG'] - self.cosmo.distmod(clus_z).value results.Ngal_1mpc = np.sum((abs_mag < -19.55) & (r < 0.5) & (np.abs(v) < 3500)) except KeyError: abs_mag = np.zeros(len(data_set)) results.Ngal_1mpc = None if r200 is None: vdisp_prelim = biweight_midvariance(data_set['vel'][(data_set['rad'] < 3.0)], 9.0) if np.sum(abs_mag) == 0: r200_mean_prelim = 0.002 * vdisp_prelim + 0.40 r200 = r200_mean_prelim / 1.7 else: r200 = results.Ngal_1mpc ** 0.51 * np.exp(-1.86) if r200 > 3.0: r200 = 3.0 # if 3.0*r200 < 6.0: # rlimit = 3.0*r200 # else: # rlimit = 5.5 else: r200 = r200 if r200 > 3.0: r200 = 3.0 print('Pre_r200=', r200) if self.mirror is True: print('Calculating Density w/Mirrored Data') rads = np.asarray(data_set['rad']) vels = np.asarray(data_set['vel']) self._gaussian_kernel(np.append(rads, rads), np.append(vels, -1 * vels), r200, normalization=Hz, scale=self.kernal_stretch, rmax=self.rgridmax, vmax=self.vgridmax) else: print('Calculating Density') self._gaussian_kernel(data_set['rad'], data_set['vel'], r200, normalization=Hz, scale=self.kernal_stretch, rmax=self.rgridmax, vmax=self.vgridmax) img_tot = self.img / np.max(np.abs(self.img)) # img_grad_tot = self.img_grad/np.max(np.abs(self.img_grad)) # img_inf_tot = self.img_inf/np.max(np.abs(self.img_inf)) if cluster_data.clus_vdisp is None: # pre_vdisp = 9.15*Ngal_1mpc+350.32 # print 'Pre_vdisp=',pre_vdisp # print 'Ngal<1Mpc=',Ngal_1mpc v_cut = data_set['vel'][((data_set['rad'] < r200) & (np.abs(data_set['vel']) < self.vlimit))] try: pre_vdisp2 = biweight_midvariance(v_cut[np.where(np.isfinite(v_cut))], 9.0) except: pre_vdisp2 = np.std(v_cut, ddof=1) print('Vdisp from galaxies=', pre_vdisp2) # if data_set[:,0].size < 15: # v_unc = 0.35 # c_unc_sys = 0.75 # c_unc_int = 0.35 # elif data_set[:,0].size < 25 and data_set[:,0].size >= 15: # v_unc = 0.30 # c_unc_sys = 0.55 # c_unc_int = 0.22 # elif data_set[:,0].size < 50 and data_set[:,0].size >= 25: # v_unc = 0.23 # c_unc_sys = 0.42 # c_unc_int = 0.16 # elif data_set[:,0].size < 100 and data_set[:,0].size >= 50: # v_unc = 0.18 # c_unc_sys = 0.34 # c_unc_int = 0.105 # else: # v_unc = 0.15 # c_unc_sys = 0.29 # c_unc_int = 0.09 # if pre_vdisp2 > 1.75*pre_vdisp: pre_vdisp_comb = 9.15*Ngal_1mpc+450.32 # else: pre_vdisp_comb = pre_vdisp2 # if data_set[:,1][np.where(data_set[:,0]<r200)].size >= 10: # pre_vdisp_comb = biweight_midvariance(data_set[:,1][np.where(data_set[:,0]<r200)],9.0) # else: # pre_vdisp_comb = np.std(data_set[:,1][np.where(data_set[:,0]<r200)],ddof=1) # #pre_vdisp_comb = (pre_vdisp*(pre_vdisp2*v_unc)**2+\ # pre_vdisp2*118.14**2)/(118.14**2+(pre_vdisp2*v_unc)**2) else: pre_vdisp_comb = cluster_data.clus_vdisp print('Combined Vdisp=', pre_vdisp_comb) # beta = 0.5*self.r_range/(self.r_range + r200/4.0) # Identify initial caustic surface and members within the surface print('Calculating initial surface') if self.inflection is False: if gal_memberflag is None: S.findsurface(data_set, self.r_range, self.v_range, img_tot, r200=r200, halo_vdisp=pre_vdisp_comb, beta=None, mirror=self.mirror, edge_perc=self.edge_perc, Hz=Hz, edge_int_remove=self.edge_int_remove, q=self.kernal_stretch, plotphase=False) else: S.findsurface(data_set, self.r_range, self.v_range, img_tot, memberflags=data_set[:, -1], r200=r200, mirror=self.mirror, edge_perc=self.edge_perc, Hz=Hz, q=self.kernal_stretch) else: if gal_memberflag is None: S.findsurface_inf(data_set, self.r_range, self.v_range, img_tot, self.img_inf, r200=r200, halo_vdisp=pre_vdisp_comb, beta=None) else: S.findsurface_inf(data_set, self.r_range, self.v_range, img_tot, self.img_inf, memberflags=data_set[:, -1], r200=r200) results.clus_ra = cluster_data.clus_ra results.clus_dec = cluster_data.clus_dec results.clus_z = cluster_data.clus_z results.caustic_profile = S.Ar_finalD results.caustic_fit = S.vesc_fit results.caustic_edge = np.abs(S.Ar_finalE) results.caustic_fit_edge = S.vesc_fit_e results.gal_vdisp = S.gal_vdisp results.memflag = S.memflag # Estimate the mass based off the caustic profile, beta profile (if given), and concentration (if given) if clus_z is not None: crit = self.cosmo.critical_density(clus_z).to( astunits.solMass / astunits.Mpc ** 3) # 2.7745946e11*(self.cosmo.h)**2.0*(0.25*(1+clus_z)**3.0 + 0.75) Mass = self.calculate_mass(ri=self.r_range, A=results.caustic_profile, crit=crit, r200=r200, fbr=None) Mass2 = self.calculate_mass(ri=self.r_range, A=results.caustic_profile, crit=crit, r200=r200, fbr=self.fbr) MassE = self.calculate_mass(ri=self.r_range, A=results.caustic_edge, crit=crit, r200=r200, fbr=self.fbr) MassF = self.calculate_mass(ri=self.r_range, A=results.caustic_fit, crit=crit, r200=r200, fbr=self.fbr) MassFE = self.calculate_mass(ri=self.r_range, A=results.caustic_fit_edge, crit=crit, r200=r200, fbr=self.fbr) results.crit = crit results.mprof = Mass.massprofile results.mprof_fbeta = Mass2.massprofile results.mprof_edge = MassE.massprofile results.r200_est = Mass.r200_est results.r200_est_fbeta = Mass2.r200_est results.r200_est_edge = MassE.r200_est results.r500_est = Mass.r500_est results.r500_est_fbeta = Mass2.r500_est results.M200_est = Mass.M200_est results.M200_est_fbeta = Mass2.M200_est results.M200_fbeta = Mass2.M200 results.M200_edge = MassE.M200 results.M200_edge_est = MassE.M200_est results.M200_fit = MassF.M200 results.M200_fit_est = MassF.M200_est results.M200_fit_edge = MassFE.M200 results.M200_fit_edge_est = MassFE.M200_est results.M500_est = Mass.M500_est results.M500_est_fbeta = Mass2.M500_est print('r200 estimate: {:.6f}'.format(float(Mass2.r200_est))) print('M200 estimate: {:.6E}'.format(float(Mass2.M200_est))) results.Ngal = np.sum((results.memflag == 1) & (data_set['rad'] <= results.r200_est_fbeta)) # calculate velocity dispersion try: results.vdisp_gal = biweight_midvariance(data_set['vel'][results.memflag == 1], 9.0) except: try: results.vdisp_gal = np.std(data_set['vel'][results.memflag == 1], ddof=1) except: results.vdisp_gal = 0.0 return results
def delay_std(self, return_dict=False): """ Calculate standard deviations of per-antenna delay solutions and aggregate delay solutions. Assign a z-score for each (antenna, time) delay solution w.r.t. aggregate mean and standard deviation. Uses sqrt( astropy.stats.biweight_midvariance ) for a robust measure of the std Input: ------ return_dict : bool, [default=False] if True, return time_std, ant_std and z_scores as a dictionary with "ant" as keys and "times" as keys else, return as ndarrays Output: -------- return ant_avg, ant_std, time_std, agg_std, max_std, z_scores ant_avg : ndarray, shape=(N_ants,) average delay solution across time for each antenna ant_std : ndarray, shape=(N_ants,) standard deviation of delay solutions across time for each antenna time_std : ndarray, shape=(N_times,) standard deviation of delay solutions across ants for each time-stamp agg_std : float aggregate standard deviation of delay solutions across all antennas and all times max_std : float maximum antenna standard deviation of delay solutions z_scores : ndarray, shape=(N_ant, N_times) z_scores (standard scores) for each (antenna, time) delay solution w.r.t. agg_std ant_z_scores : ndarray, shape=(N_ant,) absolute_value(z_scores) for each antenna & time then averaged over time """ # calculate standard deviations ant_avg = self.delay_avgs ant_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations, axis=1)) time_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations, axis=0)) agg_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations)) max_std = np.max(ant_std) # calculate z-scores z_scores = self.delay_fluctuations / agg_std ant_z_scores = np.median(np.abs(z_scores), axis=1) # convert to ordered dict if desired if return_dict is True: ant_avg_d = OrderedDict() time_std_d = OrderedDict() ant_std_d = OrderedDict() z_scores_d = OrderedDict() ant_z_scores_d = OrderedDict() for i, ant in enumerate(self.ants): ant_avg_d[ant] = ant_avg[i] ant_std_d[ant] = ant_std[i] z_scores_d[ant] = z_scores[i] ant_z_scores_d[ant] = ant_z_scores[i] for i, t in enumerate(self.times): time_std_d[t] = time_std[i] ant_avg = ant_avg_d time_std = time_std_d ant_std = ant_std_d z_scores = z_scores_d ant_z_scores = ant_z_scores_d return ant_avg, ant_std, time_std, agg_std, max_std, z_scores, ant_z_scores
def biweight_scale(data, c=9.0, M=None, axis=None, modify_sample_size=False, *, ignore_nan=False): r""" Compute the biweight scale. The biweight scale is a robust statistic for determining the standard deviation of a distribution. It is the square root of the `biweight midvariance <https://en.wikipedia.org/wiki/Robust_measures_of_scale#The_biweight_midvariance>`_. It is given by: .. math:: \zeta_{biscl} = \sqrt{n} \ \frac{\sqrt{\sum_{|u_i| < 1} \ (x_i - M)^2 (1 - u_i^2)^4}} {|(\sum_{|u_i| < 1} \ (1 - u_i^2) (1 - 5u_i^2))|} where :math:`x` is the input data, :math:`M` is the sample median (or the input location) and :math:`u_i` is given by: .. math:: u_{i} = \frac{(x_i - M)}{c * MAD} where :math:`c` is the tuning constant and :math:`MAD` is the `median absolute deviation <https://en.wikipedia.org/wiki/Median_absolute_deviation>`_. The biweight midvariance tuning constant ``c`` is typically 9.0 (the default). For the standard definition of biweight scale, :math:`n` is the total number of points in the array (or along the input ``axis``, if specified). That definition is used if ``modify_sample_size`` is `False`, which is the default. However, if ``modify_sample_size = True``, then :math:`n` is the number of points for which :math:`|u_i| < 1` (i.e. the total number of non-rejected values), i.e. .. math:: n = \sum_{|u_i| < 1} \ 1 which results in a value closer to the true standard deviation for small sample sizes or for a large number of rejected values. Parameters ---------- data : array_like Input array or object that can be converted to an array. ``data`` can be a `~numpy.ma.MaskedArray`. c : float, optional Tuning constant for the biweight estimator (default = 9.0). M : float or array_like, optional The location estimate. If ``M`` is a scalar value, then its value will be used for the entire array (or along each ``axis``, if specified). If ``M`` is an array, then its must be an array containing the location estimate along each ``axis`` of the input array. If `None` (default), then the median of the input array will be used (or along each ``axis``, if specified). axis : `None`, int, or tuple of ints, optional The axis or axes along which the biweight scales are computed. If `None` (default), then the biweight scale of the flattened input array will be computed. modify_sample_size : bool, optional If `False` (default), then the sample size used is the total number of elements in the array (or along the input ``axis``, if specified), which follows the standard definition of biweight scale. If `True`, then the sample size is reduced to correct for any rejected values (i.e. the sample size used includes only the non-rejected values), which results in a value closer to the true standard deviation for small sample sizes or for a large number of rejected values. ignore_nan : bool, optional Whether to ignore NaN values in the input ``data``. Returns ------- biweight_scale : float or `~numpy.ndarray` The biweight scale of the input data. If ``axis`` is `None` then a scalar will be returned, otherwise a `~numpy.ndarray` will be returned. See Also -------- biweight_midvariance, biweight_midcovariance, biweight_location, astropy.stats.mad_std, astropy.stats.median_absolute_deviation References ---------- .. [1] Beers, Flynn, and Gebhardt (1990; AJ 100, 32) (http://adsabs.harvard.edu/abs/1990AJ....100...32B) .. [2] https://www.itl.nist.gov/div898/software/dataplot/refman2/auxillar/biwscale.htm Examples -------- Generate random variates from a Gaussian distribution and return the biweight scale of the distribution: >>> import numpy as np >>> from astropy.stats import biweight_scale >>> rand = np.random.RandomState(12345) >>> biscl = biweight_scale(rand.randn(1000)) >>> print(biscl) # doctest: +FLOAT_CMP 0.986726249291 """ return np.sqrt( biweight_midvariance(data, c=c, M=M, axis=axis, modify_sample_size=modify_sample_size, ignore_nan=ignore_nan))
def make_mask(filename, ext, trail_coords, sublen=75, subwidth=200, order=3, sigma=4, pad=10, plot=False, verbose=False): """Create DQ mask for an image for a given satellite trail. This mask can be added to existing DQ data using :func:`update_dq`. .. note:: Unlike :func:`detsat`, multiprocessing is not available for this function. Parameters ---------- filename : str FITS image filename. ext : int, str, or tuple Extension for science data, as accepted by ``astropy.io.fits``. trail_coords : ndarray One of the trails returned by :func:`detsat`. This must be in the format of ``[[x0, y0], [x1, y1]]``. sublen : int, optional Length of strip to use as the fitting window for the trail. subwidth : int, optional Width of box to fit trail on. order : int, optional The order of the spline interpolation for image rotation. See :func:`skimage.transform.rotate`. sigma : float, optional Sigma of the satellite trail for detection. If points are a given sigma above the background in the subregion then it is marked as a satellite. This may need to be lowered for resolved trails. pad : int, optional Amount of extra padding in pixels to give the satellite mask. plot : bool, optional Plot the result. verbose : bool, optional Print extra information to the terminal, mostly for debugging. Returns ------- mask : ndarray Boolean array marking the satellite trail with `True`. Raises ------ IndexError Invalid subarray indices. ValueError Image has no positive values, trail subarray too small, or trail profile not found. """ if verbose: t_beg = time.time() fname = '{0}[{1}]'.format(filename, ext) image = fits.getdata(filename, ext) dx = image.max() if dx <= 0: raise ValueError('Image has no positive values') # rescale the image image = image / dx # make sure everything is at least 0 image[image < 0] = 0 (x0, y0), (x1, y1) = trail_coords # p0, p1 # Find out how much to rotate the image rad = np.arctan2(y1 - y0, x1 - x0) newrad = (np.pi * 2) - rad deg = np.degrees(rad) if verbose: print('Rotation: {0}'.format(deg)) rotate = transform.rotate(image, deg, resize=True, order=order) if plot and plt is not None: plt.ion() mean = np.median(image) stddev = image.std() lower = mean - stddev upper = mean + stddev fig1, ax1 = plt.subplots() ax1.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax1.set_title(fname) fig2, ax2 = plt.subplots() ax2.imshow(rotate, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax2.set_title('{0} rotated by {1} deg'.format(fname, deg)) plt.draw() # Will do all of this in the loop, but want to make sure there is a # good point first and that there is indeed a profile to fit. # get starting point sx, sy = _rotate_point((x0, y0), newrad, image.shape, rotate.shape) # start with one subarray around p0 dx = int(subwidth / 2) ix0, ix1, iy0, iy1 = _get_valid_indices( rotate.shape, sx - dx, sx + dx, sy - sublen, sy + sublen) subr = rotate[iy0:iy1, ix0:ix1] if len(subr) <= sublen: raise ValueError('Trail subarray size is {0} but expected {1} or ' 'larger'.format(len(subr), sublen)) # Flatten the array so we are looking along rows # Take median of each row, should filter out most outliers # This list will get appended in the loop medarr = np.median(subr, axis=1) flat = [medarr] # get the outliers #mean = biweight_location(medarr) mean = sigma_clipped_stats(medarr)[0] stddev = biweight_midvariance(medarr) # only flag things that are sigma from the mean z = np.where(medarr > (mean + (sigma * stddev)))[0] if plot and plt is not None: fig1, ax1 = plt.subplots() ax1.plot(medarr, 'b.') ax1.plot(z, medarr[z], 'r.') ax1.set_xlabel('Index') ax1.set_ylabel('Value') ax1.set_title('Median array in flat[0]') plt.draw() # Make sure there is something in the first pass before trying to move on if len(z) < 1: raise ValueError( 'First look at finding a profile failed. ' 'Nothing found at {0} from background! ' 'Adjust parameters and try again.'.format(sigma)) # get the bounds of the flagged points lower = z.min() upper = z.max() diff = upper - lower # add in a pading value to make sure all of the wings are accounted for lower = lower - pad upper = upper + pad # for plotting see how the profile was made (append to plot above) if plot and plt is not None: padind = np.arange(lower, upper) ax1.plot(padind, medarr[padind], 'yx') plt.draw() # start to create a mask mask = np.zeros(rotate.shape) lowerx, upperx, lowery, uppery = _get_valid_indices( mask.shape, np.floor(sx - subwidth), np.ceil(sx + subwidth), np.floor(sy - sublen + lower), np.ceil(sy - sublen + upper)) mask[lowery:uppery, lowerx:upperx] = 1 done = False first = True nextx = upperx # np.ceil(sx + subwidth) centery = np.ceil(lowery + diff) # np.ceil(sy - sublen + lower + diff) counter = 0 while not done: # move to the right of the centerpoint first. do the same # as above but keep moving right until the edge is hit. ix0, ix1, iy0, iy1 = _get_valid_indices( rotate.shape, nextx - dx, nextx + dx, centery - sublen, centery + sublen) subr = rotate[iy0:iy1, ix0:ix1] # determines the edge, if the subr is not good, then the edge was # hit. if 0 in subr.shape: if verbose: print('Hit edge, subr shape={0}, first={1}'.format( subr.shape, first)) if first: first = False centery = sy nextx = sx else: done = True continue medarr = np.median(subr, axis=1) flat.append(medarr) #mean = biweight_location(medarr) mean = sigma_clipped_stats(medarr, sigma=sigma)[0] stddev = biweight_midvariance(medarr) # Might give RuntimeWarning z = np.where(medarr > (mean + (sigma * stddev)))[0] if len(z) < 1: if first: if verbose: print('No good profile found for counter={0}. Start ' 'moving left from starting point.'.format(counter)) centery = sy nextx = sx first = False else: if verbose: print('z={0} is less than 1, subr shape={1}, ' 'we are done'.format(z, subr.shape)) done = True continue # get the bounds of the flagged points lower = z.min() upper = z.max() diff = upper - lower # add in a pading value to make sure all of the wings # are accounted for lower = np.floor(lower - pad) upper = np.ceil(upper + pad) lowerx, upperx, lowery, uppery = _get_valid_indices( mask.shape, np.floor(nextx - subwidth), np.ceil(nextx + subwidth), np.floor(centery - sublen + lower), np.ceil(centery - sublen + upper)) mask[lowery:uppery, lowerx:upperx] = 1 lower_p = (lowerx, lowery) upper_p = (upperx, uppery) lower_t = _rotate_point( lower_p, newrad, image.shape, rotate.shape, reverse=True) upper_t = _rotate_point( upper_p, newrad, image.shape, rotate.shape, reverse=True) lowy = np.floor(lower_t[1]) highy = np.ceil(upper_t[1]) lowx = np.floor(lower_t[0]) highx = np.ceil(upper_t[0]) # Reset the next subr to be at the center of the profile if first: nextx = nextx + dx centery = lowery + diff # centery - sublen + lower + diff if (nextx + subwidth) > rotate.shape[1]: if verbose: print('Hit rotate edge at counter={0}'.format(counter)) first = False elif (highy > image.shape[0]) or (highx > image.shape[1]): if verbose: print('Hit image edge at counter={0}'.format(counter)) first = False if not first: centery = sy nextx = sx # Not first, this is the pass the other way. else: nextx = nextx - dx centery = lowery + diff # centery - sublen + lower + diff if (nextx - subwidth) < 0: if verbose: print('Hit rotate edge at counter={0}'.format(counter)) done = True elif (highy > image.shape[0]) or (highx > image.shape[1]): if verbose: print('Hit image edge at counter={0}'.format(counter)) done = True counter += 1 # make sure it does not try to go infinetly if counter > 500: if verbose: print('Too many loops, exiting') done = True # End while rot = transform.rotate(mask, -deg, resize=True, order=1) ix0 = (rot.shape[1] - image.shape[1]) / 2 iy0 = (rot.shape[0] - image.shape[0]) / 2 lowerx, upperx, lowery, uppery = _get_valid_indices( rot.shape, ix0, image.shape[1] + ix0, iy0, image.shape[0] + iy0) mask = rot[lowery:uppery, lowerx:upperx] if mask.shape != image.shape: warnings.warn('Output mask shape is {0} but input image shape is ' '{1}'.format(mask.shape, image.shape), AstropyUserWarning) # Change to boolean mask mask = mask.astype(np.bool) if plot and plt is not None: # debugging array test = image.copy() test[mask] = 0 mean = np.median(test) stddev = test.std() lower = mean - stddev upper = mean + stddev fig1, ax1 = plt.subplots() ax1.imshow(test, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax1.set_title('Masked image') fig2, ax2 = plt.subplots() ax2.imshow(mask, cmap=plt.cm.gray) ax2.set_title('DQ mask') plt.draw() if verbose: t_end = time.time() print('Run time: {0} s'.format(t_end - t_beg)) return mask
def calc_background_rms(self, data): return biweight_midvariance(self.sigma_clip(data), c=self.c, M=self.M)
def histogram(band, band_ref, best_weights, training_data, aeg_models, chisquare_models): weights = best_weights[-5, 0:len(band)] print(weights) Z_bef = np.zeros((len(training_data))) age_bef = np.zeros((len(training_data))) ebv_bef = np.zeros((len(training_data))) Z_aft = np.zeros((len(training_data))) age_aft = np.zeros((len(training_data))) ebv_aft = np.zeros((len(training_data))) plt.subplots(figsize=(15, 7)) for j in range(0, len(training_data)): chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.ones((len(band)))) Z_bef[j] = chi[0] - aeg_models[j, 0] age_bef[j] = (chi[1] - aeg_models[j, 1]) / 10**9 ebv_bef[j] = chi[2] - aeg_models[j, 2] for j in range(0, len(training_data)): chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.array([weights])) Z_aft[j] = chi[0] - aeg_models[j, 0] age_aft[j] = (chi[1] - aeg_models[j, 1]) / 10**9 ebv_aft[j] = chi[2] - aeg_models[j, 2] plt.subplot(1, 3, 1) plt.hist(Z_bef, bins=np.arange(-0.055, 0.055, 0.01), color='#aaaa22', rwidth=0.95) bins_Z_aft, Z_aft_data = histOutline(Z_aft, bins=np.arange(-0.055, 0.055, 0.01)) plt.plot(bins_Z_aft, Z_aft_data, '-', color="#1B6AC6", lw=4) plt.xlabel(r'$\Delta$Z' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(Z_bef), biweight_midvariance(Z_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(Z_aft), biweight_midvariance(Z_aft))) plt.xlim(-0.05, 0.05) plt.subplot(1, 3, 2) plt.hist(age_bef, bins=np.arange(-1.05, 1.05, 0.1), color='#aaaa22', rwidth=0.95) bins_age_aft, age_aft_data = histOutline(age_aft, bins=np.arange(-1.05, 1.05, 0.1)) plt.plot(bins_age_aft, age_aft_data, '-', color="#1B6AC6", lw=4) plt.xlabel(r'$\Delta$age/$10^{9}$' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(age_bef), biweight_midvariance(age_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(age_aft), biweight_midvariance(age_aft))) plt.xlim(-0.5, 0.5) plt.subplot(1, 3, 3) plt.hist(ebv_bef, bins=np.arange(-0.45, 0.45, 0.05), label='Before training', color='#aaaa22', rwidth=0.95) bins_ebv_aft, ebv_aft_data = histOutline(ebv_aft, bins=np.arange(-0.45, 0.45, 0.05)) plt.plot(bins_ebv_aft, ebv_aft_data, '-', color="#1B6AC6", lw=4, label='After training') plt.xlabel('$\Delta$E(B-V)' + '\n' + 'Before training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format( np.mean(ebv_bef), biweight_midvariance(ebv_bef)) + '\n' + 'After training: ' + r'$\mu={0:1.6f}, \zeta={1:1.6f}$'.format( np.mean(ebv_aft), biweight_midvariance(ebv_aft))) # plt.xlim(-5,2) plt.legend() # plt.suptitle('Histograms') plt.tight_layout() # plt.savefig('histogram_Z&Age.png') plt.show()
def __init__(self, calfits_files, use_gp=True): """ Input: ------ calfits_files : str or list filename for a *.first.calfits file or a list of .first.calfits files (time-ordered) of the same polarization use_gp : bool, default=True use gaussian process model to subtract underlying smooth delay solution behavior over time from fluctuations Result: ------- self.UVC : pyuvdata.UVCal() instance self.delays : ndarray, shape=(N_ant, N_times) firstcal delay solutions in nanoseconds self.delay_avgs : ndarray, shape=(N_ant,) median delay solutions across time [nanosec] self.delay_fluctuations : ndarray, shape=(N_ant, N_times) firstcal delay solution fluctuations from time average [nanosec] self.frac_JD : ndarray, shape=(N_times,) ndarray containing time-stamps of each integration in units of the fraction of current JD i.e. 2457966.53433 -> 0.53433 self.ants : ndarray, shape=(N_ants,) ndarray containing antenna numbers self.pol : str Polarization, 'y' or 'x' currently supported """ # Instantiate UVCal and read calfits self.UVC = UVCal() self.UVC.read_calfits(calfits_files) if len(self.UVC.jones_array) > 1: raise ValueError('Sorry, only single pol firstcal solutions are ' 'currently supported.') pol_dict = {-5: 'x', -6: 'y'} try: self.pol = pol_dict[self.UVC.jones_array[0]] except KeyError: raise ValueError('Sorry, only calibration polarizations "x" and ' '"y" are currently supported.') # get file prefix if type(calfits_files) is list: calfits_file = calfits_files[0] else: calfits_file = calfits_files self.fc_basename = os.path.basename(calfits_file) self.fc_filename = calfits_file self.fc_filestem = '.'.join(self.fc_filename.split('.')[:-1]) # get other relevant arrays self.times = self.UVC.time_array self.start_JD = np.floor(self.times).min() self.frac_JD = self.times - self.start_JD self.minutes = 24 * 60 * (self.frac_JD - self.frac_JD.min()) self.Nants = self.UVC.Nants_data self.ants = self.UVC.ant_array self.version_str = hera_qm_version_str self.history = '' # Get the firstcal delays and/or gains and/or rotated antennas if self.UVC.cal_type == 'gain': # get delays freqs = self.UVC.freq_array.squeeze() fc_gains = np.moveaxis(self.UVC.gain_array, 2, 3)[:, 0, :, :, 0] fc_phi = np.unwrap(np.angle(fc_gains)) d_nu = np.median(np.diff(freqs)) d_phi = np.median(fc_phi[:, :, 1:] - fc_phi[:, :, :-1], axis=2) gain_slope = (d_phi / d_nu) self.delays = gain_slope / (-2*np.pi) self.gains = fc_gains # get delay offsets at nu = 0 Hz, and then get rotated antennas self.offsets = fc_phi[:, :, 0] - gain_slope * freqs[0] self.rot_ants = np.unique(list(map(lambda x: self.ants[x], (np.isclose(np.pi, np.abs(self.offsets) % (2 * np.pi), atol=1.0)).T))).tolist() elif self.UVC.cal_type == 'delay': self.delays = self.UVC.delay_array.squeeze() self.gains = None self.offsets = None self.rot_ants = [] # Calculate avg delay solution and subtract to get delay_fluctuations self.delays = self.delays * 1e9 self.delay_avgs = np.median(self.delays, axis=1) self.delay_fluctuations = (self.delays.T - self.delay_avgs).T # use gaussian process model to subtract underlying mean function if use_gp is True and self.sklearn_import is True: # initialize GP kernel. # RBF is a squared exponential kernel with a minimum length_scale_bound of 0.01 JD, meaning # the GP solution won't have time fluctuations quicker than ~0.01 JD, which will preserve # short time fluctuations. WhiteKernel is a Gaussian white noise component with a fiducial # noise level of 0.01 nanoseconds. Both of these are hyperparameters that are fit for via # a gradient descent algorithm in the GP.fit() routine, so length_scale=0.2 and # noise_level=0.01 are just initial conditions and are not the final hyperparameter solution kernel = gp.kernels.RBF(length_scale=0.2, length_scale_bounds=(0.01, 1.0)) + gp.kernels.WhiteKernel(noise_level=0.01) x = self.frac_JD.reshape(-1, 1) self.delay_smooths = [] # iterate over each antenna for i in range(self.Nants): # get ydata y = copy.copy(self.delay_fluctuations[i]) # scale by std ystd = np.sqrt(astats.biweight_midvariance(y)) y /= ystd # fit GP and remove from delay fluctuations GP = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=0) GP.fit(x, y) ymodel = GP.predict(x).ravel() * ystd self.delay_fluctuations[i] -= ymodel self.delay_smooths.append(ymodel) self.delay_smooths = np.array(self.delay_smooths)
def make_mask(filename, ext, trail_coords, sublen=75, subwidth=200, order=3, sigma=4, pad=10, plot=False, verbose=False): """Create DQ mask for an image for a given satellite trail. This mask can be added to existing DQ data using :func:`update_dq`. .. note:: Unlike :func:`detsat`, multiprocessing is not available for this function. Parameters ---------- filename : str FITS image filename. ext : int, str, or tuple Extension for science data, as accepted by ``astropy.io.fits``. trail_coords : ndarray One of the trails returned by :func:`detsat`. This must be in the format of ``[[x0, y0], [x1, y1]]``. sublen : int, optional Length of strip to use as the fitting window for the trail. subwidth : int, optional Width of box to fit trail on. order : int, optional The order of the spline interpolation for image rotation. See :func:`skimage.transform.rotate`. sigma : float, optional Sigma of the satellite trail for detection. If points are a given sigma above the background in the subregion then it is marked as a satellite. This may need to be lowered for resolved trails. pad : int, optional Amount of extra padding in pixels to give the satellite mask. plot : bool, optional Plot the result. verbose : bool, optional Print extra information to the terminal, mostly for debugging. Returns ------- mask : ndarray Boolean array marking the satellite trail with `True`. Raises ------ ImportError Missing scipy or skimage>=0.11 packages. IndexError Invalid subarray indices. ValueError Image has no positive values, trail subarray too small, or trail profile not found. """ if not HAS_OPDEP: raise ImportError('Missing scipy or skimage>=0.11 packages') if verbose: t_beg = time.time() fname = '{0}[{1}]'.format(filename, ext) image = fits.getdata(filename, ext) dx = image.max() if dx <= 0: raise ValueError('Image has no positive values') # rescale the image image = image / dx # make sure everything is at least 0 image[image < 0] = 0 (x0, y0), (x1, y1) = trail_coords # p0, p1 # Find out how much to rotate the image rad = np.arctan2(y1 - y0, x1 - x0) newrad = (np.pi * 2) - rad deg = np.degrees(rad) if verbose: print('Rotation: {0}'.format(deg)) rotate = transform.rotate(image, deg, resize=True, order=order) if plot and plt is not None: plt.ion() mean = np.median(image) stddev = image.std() lower = mean - stddev upper = mean + stddev fig1, ax1 = plt.subplots() ax1.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax1.set_title(fname) fig2, ax2 = plt.subplots() ax2.imshow(rotate, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax2.set_title('{0} rotated by {1} deg'.format(fname, deg)) plt.draw() # Will do all of this in the loop, but want to make sure there is a # good point first and that there is indeed a profile to fit. # get starting point sx, sy = _rotate_point((x0, y0), newrad, image.shape, rotate.shape) # start with one subarray around p0 dx = int(subwidth / 2) ix0, ix1, iy0, iy1 = _get_valid_indices(rotate.shape, sx - dx, sx + dx, sy - sublen, sy + sublen) subr = rotate[iy0:iy1, ix0:ix1] if len(subr) <= sublen: raise ValueError('Trail subarray size is {0} but expected {1} or ' 'larger'.format(len(subr), sublen)) # Flatten the array so we are looking along rows # Take median of each row, should filter out most outliers # This list will get appended in the loop medarr = np.median(subr, axis=1) flat = [medarr] # get the outliers #mean = biweight_location(medarr) mean = sigma_clipped_stats(medarr)[0] stddev = biweight_midvariance(medarr) # only flag things that are sigma from the mean z = np.where(medarr > (mean + (sigma * stddev)))[0] if plot and plt is not None: fig1, ax1 = plt.subplots() ax1.plot(medarr, 'b.') ax1.plot(z, medarr[z], 'r.') ax1.set_xlabel('Index') ax1.set_ylabel('Value') ax1.set_title('Median array in flat[0]') plt.draw() # Make sure there is something in the first pass before trying to move on if len(z) < 1: raise ValueError('First look at finding a profile failed. ' 'Nothing found at {0} from background! ' 'Adjust parameters and try again.'.format(sigma)) # get the bounds of the flagged points lower = z.min() upper = z.max() diff = upper - lower # add in a pading value to make sure all of the wings are accounted for lower = lower - pad upper = upper + pad # for plotting see how the profile was made (append to plot above) if plot and plt is not None: padind = np.arange(lower, upper) ax1.plot(padind, medarr[padind], 'yx') plt.draw() # start to create a mask mask = np.zeros(rotate.shape) lowerx, upperx, lowery, uppery = _get_valid_indices( mask.shape, np.floor(sx - subwidth), np.ceil(sx + subwidth), np.floor(sy - sublen + lower), np.ceil(sy - sublen + upper)) mask[lowery:uppery, lowerx:upperx] = 1 done = False first = True nextx = upperx # np.ceil(sx + subwidth) centery = np.ceil(lowery + diff) # np.ceil(sy - sublen + lower + diff) counter = 0 while not done: # move to the right of the centerpoint first. do the same # as above but keep moving right until the edge is hit. ix0, ix1, iy0, iy1 = _get_valid_indices(rotate.shape, nextx - dx, nextx + dx, centery - sublen, centery + sublen) subr = rotate[iy0:iy1, ix0:ix1] # determines the edge, if the subr is not good, then the edge was # hit. if 0 in subr.shape: if verbose: print('Hit edge, subr shape={0}, first={1}'.format( subr.shape, first)) if first: first = False centery = sy nextx = sx else: done = True continue medarr = np.median(subr, axis=1) flat.append(medarr) #mean = biweight_location(medarr) mean = sigma_clipped_stats(medarr, sigma=sigma)[0] stddev = biweight_midvariance(medarr) # Might give RuntimeWarning z = np.where(medarr > (mean + (sigma * stddev)))[0] if len(z) < 1: if first: if verbose: print('No good profile found for counter={0}. Start ' 'moving left from starting point.'.format(counter)) centery = sy nextx = sx first = False else: if verbose: print('z={0} is less than 1, subr shape={1}, ' 'we are done'.format(z, subr.shape)) done = True continue # get the bounds of the flagged points lower = z.min() upper = z.max() diff = upper - lower # add in a pading value to make sure all of the wings # are accounted for lower = np.floor(lower - pad) upper = np.ceil(upper + pad) lowerx, upperx, lowery, uppery = _get_valid_indices( mask.shape, np.floor(nextx - subwidth), np.ceil(nextx + subwidth), np.floor(centery - sublen + lower), np.ceil(centery - sublen + upper)) mask[lowery:uppery, lowerx:upperx] = 1 lower_p = (lowerx, lowery) upper_p = (upperx, uppery) lower_t = _rotate_point(lower_p, newrad, image.shape, rotate.shape, reverse=True) upper_t = _rotate_point(upper_p, newrad, image.shape, rotate.shape, reverse=True) lowy = np.floor(lower_t[1]) highy = np.ceil(upper_t[1]) lowx = np.floor(lower_t[0]) highx = np.ceil(upper_t[0]) # Reset the next subr to be at the center of the profile if first: nextx = nextx + dx centery = lowery + diff # centery - sublen + lower + diff if (nextx + subwidth) > rotate.shape[1]: if verbose: print('Hit rotate edge at counter={0}'.format(counter)) first = False elif (highy > image.shape[0]) or (highx > image.shape[1]): if verbose: print('Hit image edge at counter={0}'.format(counter)) first = False if not first: centery = sy nextx = sx # Not first, this is the pass the other way. else: nextx = nextx - dx centery = lowery + diff # centery - sublen + lower + diff if (nextx - subwidth) < 0: if verbose: print('Hit rotate edge at counter={0}'.format(counter)) done = True elif (highy > image.shape[0]) or (highx > image.shape[1]): if verbose: print('Hit image edge at counter={0}'.format(counter)) done = True counter += 1 # make sure it does not try to go infinetly if counter > 500: if verbose: print('Too many loops, exiting') done = True # End while rot = transform.rotate(mask, -deg, resize=True, order=1) ix0 = (rot.shape[1] - image.shape[1]) / 2 iy0 = (rot.shape[0] - image.shape[0]) / 2 lowerx, upperx, lowery, uppery = _get_valid_indices( rot.shape, ix0, image.shape[1] + ix0, iy0, image.shape[0] + iy0) mask = rot[lowery:uppery, lowerx:upperx] if mask.shape != image.shape: warnings.warn( 'Output mask shape is {0} but input image shape is ' '{1}'.format(mask.shape, image.shape), AstropyUserWarning) # Change to boolean mask mask = mask.astype(np.bool) if plot and plt is not None: # debugging array test = image.copy() test[mask] = 0 mean = np.median(test) stddev = test.std() lower = mean - stddev upper = mean + stddev fig1, ax1 = plt.subplots() ax1.imshow(test, vmin=lower, vmax=upper, cmap=plt.cm.gray) ax1.set_title('Masked image') fig2, ax2 = plt.subplots() ax2.imshow(mask, cmap=plt.cm.gray) ax2.set_title('DQ mask') plt.draw() if verbose: t_end = time.time() print('Run time: {0} s'.format(t_end - t_beg)) return mask
def fbistd(y): """Helper function of trendplot. Returns biweight std of input 1d-array (see astropy.stats)""" return biweight_midvariance(y)
mr1 = (fluxdata[allgood]['ar'] < 0.25) mr2 = np.logical_and(fluxdata[allgood]['ar'] >= 0.25, fluxdata[allgood]['ar'] < 0.5) mr3 = np.logical_and(fluxdata[allgood]['ar'] >= 0.5, fluxdata[allgood]['ar'] < 0.75) mr4 = (fluxdata[allgood]['ar'] >= 0.75) med1 = np.median(fluxdata[allgood][mr1].av) med2 = np.median(fluxdata[allgood][mr2].av) med3 = np.median(fluxdata[allgood][mr3].av) med4 = np.median(fluxdata[allgood][mr4].av) med751 = np.percentile(fluxdata[allgood][mr1].av, 75) med752 = np.percentile(fluxdata[allgood][mr2].av, 75) med753 = np.percentile(fluxdata[allgood][mr3].av, 75) med754 = np.percentile(fluxdata[allgood][mr4].av, 75) emed1 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr1].av)) emed2 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr2].av)) emed3 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr3].av)) emed4 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr4].av)) ms = 12 lwbw = 2 for ax in axarr: ax2 = axarr2[c] ax3 = axarr3[c] ax4 = axarr4[c] ax5 = axarr5[c] if c == 0: col = 'good' filt = allgood color = 'blue'
def trendplot( xin, yin, crange=None, nbin=None, xycut=None, prange=None, ystat=None, estat=None, psize=None, pcolor=None, cmarker=None, ccolor=None, csize=None, cls=None, clw=None, sigfact=None, sigcolor=None, sigls=None, noscatter=False, noplot=False, ): """ XY scatter plot with tendency curves and optional errorbars + sigma curves. Parameters ---------- **xin,yin** : 1d-arrays. Input data. Required. **crange** : list of form [xlim1,xlim2]. x-axis range that the tendency curve will span. Defaults to data [min,max]. **nbin** : int. Nr of bins of tendency curve. **xycut** : list of form [xlim1,xlim2,ylim1,ylim2]. Rectangular window to cut input data by. Defaults to data [min,max]. **prange** : list of form [xlim1,xlim2,ylim1,ylim2]. Rectangular window for plotting. Defaults to data [min,max]. **ystat** : y-axis statistic. Default: "median". * "median" : median of data in bin. * "mean" : mean of data in bin. * "bimean" : biweight mean of data in bin (from astropy.stats). **estat** : y-axis error bar statistic. * "std" : standard deviation. * "bistd" : biweight standard deviation(from astropy.stats). * "poisson" : poisson error. * "boot" : boostrap sampling error, i.e. the error of each bin is bistd(bimean(bootsamples)). Defaults to 250 samples. **psize** : point size of scatter plot. Default: 1 **pcolor** : point color of scatter plot. Default: "0.5" (gray) **cmarker** : marker of tendency curve. Default: "o" **ccolor** : color of tendency curve. Default: "red" **csize** : size of tendency curve markers. Default: 4 **cls** : linestyle of tendency curve. Default: "-" **clw** : line width of tendency curve. Default: 1 **sigfact** : float. Plot 1,2,3 or x sigma lines above and below tendency curve. Uses bistd(). **sigcolor** : color of sigma lines. Default: "k" **sigls** : linestyle of sigma lines. Default: "--" **noscatter** : bool. Do not plot scatter points. Default: False **noplot** : bool. Do not plot anything and return curves instead. Default: False Returns ------- (only when noplot=True) * **(px,py)** : coordinates of curve, if estat is not set * **(px,py,ey)**: coordinates of curve + errors, if estat is set Examples -------- :: import trend as t x = np.arange(0,100,0.1) y = 1.2*x + np.random.randn(len(x))*15 y[300:600] = 55. + np.random.randn(300)*6 # Simple plot t.trendplot(x,y) # More bins, 1-sigma error lines t.trendplot(x,y, nbin=20, sigfact=1, estat='std') # Custom curve range, custom ystat, boostrap error bars t.trendplot(x,y,crange=[30,80],ystat='mean',estat='boot',ccolor='blue') # Don't plot but get the biweight mean tendency curve with boostrap errors px,py,ey = t.trendplot(x,y,ystat='bimean',estat='boot',noplot=True) """ # Default plotting styles if psize is None: psize = 1 # scatter point size if pcolor is None: pcolor = "0.5" # scatter point color if cmarker is None: cmarker = "o" # t. curve point style if csize is None: csize = 4 # t. curve point size if ccolor is None: ccolor = "red" # t. curve color if cls is None: cls = "-" # t. curve line style if clw is None: clw = 1 # t. curve line width if sigcolor is None: sigcolor = "k" # sigma line color if sigls is None: sigls = "--" # sigma line style # Default number of bins if nbin is None: nbin = 10 # Default y-axis statistic if ystat is None: ystat = "median" # Default xrange that the curve will spawn if crange is None: crange = [np.min(xin), np.max(xin)] # Cut input values. Add some tolerance if xycut is None: tol = 0.01 xycut = [0.0, 0.0, 0.0, 0.0] xycut[0] = np.min(xin) - tol xycut[1] = np.max(xin) + tol xycut[2] = np.min(yin) - tol xycut[3] = np.max(yin) + tol idxin = np.where((xin > xycut[0]) & (xin < xycut[1]) & (yin > xycut[2]) & (yin < xycut[3])) xin, yin = xin[idxin], yin[idxin] # Default plot range for entire plot. Do this after cutting input? if prange is None: prange = [np.min(xin), np.max(xin), np.min(yin), np.max(yin)] # Bin x-coordinates xcounts, xedges = np.histogram(xin, bins=nbin, range=crange) # Find curve x-axis (mid-point of bins) hbsiz = abs(xedges[0] - xedges[1]) * 0.5 px = xedges[:-1] + hbsiz # drop last # Find curve y-axis (mean, median, etc.) py = np.zeros(nbin) ey = np.zeros(nbin) sigmav = np.zeros(nbin) ystatoptions = {"median": fmed, "mean": fmean, "bimean": fbimean} estatoptions = {"std": fstd, "poisson": fpoisson, "boot": fboot, "bistd": fbistd} for l, r, i in zip(xedges[:-1], xedges[1:], range(nbin)): ydat = yin[(xin > l) & (xin < r)] py[i] = ystatoptions[ystat](ydat) if estat is not None: ey[i] = estatoptions[estat](ydat) if sigfact is not None: sigmav[i] = biweight_midvariance(ydat) # print(i,'--',l,'--',r,'--',elm,'--',py[i]) # Do plot or get results ================================================= if noplot == False: if noscatter == False: plt.scatter(xin, yin, marker=".", s=psize, color=pcolor) # scatter points print cls plt.plot(px, py, color=ccolor, linestyle=cls, linewidth=clw) # curve line plt.plot( px, py, marker=cmarker, markersize=csize, color=ccolor, markeredgecolor=ccolor, linestyle="none" ) # curve points if estat is not None: plt.errorbar(px, py, yerr=ey, fmt="none", ecolor=ccolor) # curve errorbars if sigfact is not None: plt.plot(px, py + sigfact * sigmav, color=sigcolor, linestyle=sigls) # sigma lines plt.plot(px, py - sigfact * sigmav, color=sigcolor, linestyle=sigls) # Reset plot limits to desired range plt.xlim(prange[0], prange[1]) plt.ylim(prange[2], prange[3]) else: if estat is not None: res = (px, py, ey) else: res = (px, py) return res
def _gaussian_kernel(self, rvalues, vvalues, r200, normalization=100.0, scale=10.0, rres=200, vres=220, rmax=6.0, vmax=5000.0): """ Uses a 2D gaussian kernel to estimate the density of the phase space. As of now, the maximum radius extends to 6Mpc and the maximum velocity allowed is 5000km/s The "q" parameter is termed "scale" here which we have set to 10 as default, but can go as high as 50. "normalization" is simply H0 "r/vres" can be any value, but are recommended to be above 150 "adj" is a custom value and changes the size of uniform filters when used (not normally needed) Parameters ---------- rvalues : r-coordinates of points in phase space vvalues : v-coordinates of points in phase space r200 : Required estimate of r200 to calculate a rough dispersion normalization = 100 : This is equivalent to H0. Default is H0=100 scale = 10 : "q" parameter in Diaferio 99. Literature says this can be between 10-50 rres = 200 : r-grid resolution vres = 220 : v-grid resolution rmax = 6.0 : Maximum r-grid value. If data points exceed this amount either increase this value or cut sample to be within this value. vmax = 5000 : Maximum/minimum v-grid value. If data points exceed this amount either increase this value or cut sample to be within this value. Returns ------- self.r_range : array of r-grid values self.v_range : array of v-grid values self.img : smoothed density image self.img_grad : first derivative of img self.img_inf : second derivative of img """ if np.max(rvalues) >= rmax: raise Exception( 'Bounding Error: Please either increase your rmax value or trim your sample to be r < ' + str(rmax)) if np.max(np.abs(vvalues)) >= vmax: raise Exception( 'Bounding Error: Please either increase your vmax value or trim your sample to be v < ' + str(vmax)) vvalues = vvalues / (normalization * scale) self.r_range = np.arange(0, rmax, 0.05) r_range_bin = np.arange(0, rmax + 0.05, 0.05) rres = self.r_range.size self.v_range = np.arange(-vmax / (normalization * scale), vmax / (normalization * scale), 0.05) * normalization * scale v_range_bin = np.arange(-1 * vmax / (normalization * scale), (vmax / (normalization * scale)) + 0.05, 0.05) # vres = self.v_range.size r_scale = (rvalues / rmax) * rres v_scale = ((vvalues * (normalization * scale) + vmax) / (vmax * 2.0)) * self.v_range.size # self.ksize_r = (4.0/(3.0*rvalues.size))**(1/5.0)*np.std(r_scale[rvalues<r200]) ksize_r = (4.0 / (3.0 * rvalues.size)) ** (1 / 5.0) * \ np.sqrt(( biweight_midvariance((r_scale[rvalues < r200]).copy(), 9.0) ** 2 + biweight_midvariance((v_scale[rvalues < r200]).copy(), 9.0) ** 2) / 2.0 ) ksize_r *= 1.0 ksize_v = ksize_r # (4.0/(3.0*rvalues.size))**(1/5.0)*np.std(v_scale[rvalues<r200]) imgr, redge, vedge = np.histogram2d(rvalues, vvalues, bins=(r_range_bin, v_range_bin)) self.img = ndi.gaussian_filter(imgr, (ksize_r, ksize_v), mode='reflect') self.img_grad = ndi.gaussian_gradient_magnitude(imgr, (ksize_r, ksize_v)) self.img_inf = ndi.gaussian_gradient_magnitude(ndi.gaussian_gradient_magnitude(imgr, (ksize_r, ksize_v)), (ksize_r, ksize_v))
clus_dec = DEC[i] clus_z = Z[i] rvir = RVIR[i] ang_d,lum_d = C.zdistance(clus_z,H0) angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec) rdata = angles * ang_d vdata = c * (gal_z - clus_z) / (1 + clus_z) # Calculate HVD rlim = rvir * 1.25 # Mpc rad_cut = np.where(rdata < rlim)[0] if len(rad_cut) == 0: continue elif len(rad_cut) > 2: hvd = astats.biweight_midvariance(vdata[np.where((rdata < rlim)&(np.abs(vdata)<3000))]) else: hvd = np.std(vdata[np.where(rdata < rlim)]) Nspec.append( np.where((rdata < rvir) & (np.abs(vdata) < hvd * 1.5))[0].size ) HVD.append(hvd) good2.append(i) if i % 200 == 0: print i Nspec,HVD,good2 = np.array(Nspec),np.array(HVD),np.array(good2) N200 = [] for i in good2: N200.append(new[i][2,1,4,9] - newb[i][2,1,4,1]) N200 = np.array(N200) new_halos = halos[good2]
ones[~np.isfinite(ffskysub)] = np.nan wave_here = (def_wave > 4550) & (def_wave <= 4650) for star in stars: detectid = star["detectid"] ra, dec = star["ra_1"], star["dec_1"] rsqs = distsq(ra, dec, shot_tab["ra"], shot_tab["dec"]) mask_here = rsqs < 10**2 N = np.nansum(ones[mask_here][:, wave_here], axis=1) flux_here = ffskysub[mask_here][:, wave_here] weights_here = weights[mask_here][:, wave_here] these_fibers_b = biweight_location(flux_here, ignore_nan=True, axis=1) these_errs_b = np.sqrt( biweight_midvariance(flux_here, ignore_nan=True, axis=1)) / np.sqrt(N - 1) weight_sum = np.nansum(weights_here, axis=1) mean_flux_weighted = np.nansum(flux_here * weights_here, axis=1) / weight_sum diff_squared = (flux_here.T - mean_flux_weighted)**2 diff_squared = diff_squared.T mean_error_weighted = np.sqrt( np.nansum(diff_squared * weights_here, axis=1) / ((N - 1) * weight_sum)) rs = np.sqrt(rsqs[mask_here]) p0 = [np.nanmax(mean_flux_weighted), 1.3] mask_finite = np.isfinite(mean_flux_weighted) & np.isfinite( mean_error_weighted)
def _cal_contrast(image, mask, pixel_scale, zeropoint, sigma=1.0, scale_arcsec=60, minfrac=0.8, minback=6, verbose=True, logger=None): """ Calculate the surface brightness detection limit on a given angular scale. The error of SB limit is calculated according to https://stats.stackexchange.com/questions/631/standard-deviation-of-standard-deviation. Parameters: image (numpy 2-D array): input image. mask (numpy 2-D array): if you want to mask out a pixel, set its value to 1; otherwise set to zero. pixel_scale (float): pixel scale of the input image, in the unit of ``arcsec/pixel``. zeropoint (float): photometric zeropoint of the input image. sigma (float): indicates the detection threshold on which the SB limit will be calculated. scale_arcsec (float): on which scale we calculate SB limit, in the unit of ``arcsec``. If ``scale_arcsec=60``, this function prints out SB limit on the scale of 60 arcsec * 60 arcsec square. minfrac (float): Must be less than 1.0. We discard super-pixels in which less than ``minfrac`` fraction of pixels are available. Hence super-pixels with too many pixels masked out are discarded. minback (int): Given a super-pixel, we discard it (set to zero) if there are less than ``minback`` non-zero super-pixels surrounding it. verbose (bool): whether print out results. logger (``logging.logger`` object): logger for this function. Default is ``None``. """ import copy from astropy.stats import biweight_midvariance from astropy.stats import biweight_location ## read image ny, nx = image.shape scale_pix = scale_arcsec / pixel_scale # scale in pixel scale_x = np.array([scale_pix, int(scale_pix), int(scale_pix), int(scale_pix) + 1]) scale_y = np.array([scale_pix, int(scale_pix), int(scale_pix) + 1, int(scale_pix) + 1]) area = scale_x * scale_y area -= area[0] area = abs(area)[1:] # d1, d2, d3 bin_x = int(scale_x[np.argmin(area) + 1]) bin_y = int(scale_y[np.argmin(area) + 1]) area_ratio = bin_x * bin_y / scale_pix**2 if verbose: if logger is not None: logger.info("Determine surface brightness detection limit") logger.info(' - Binning factors: dx = {0}, dy = {1}'.format(bin_x, bin_y)) logger.info(' - Used bin area / True bin area = {:.5f}'.format(area_ratio)) else: print('# Determine surface brightness detection limit') print(' - Binning factors: dx = {0}, dy = {1}'.format(bin_x, bin_y)) print(' - Used bin area / True bin area = {:.5f}'.format(area_ratio)) nbins_x = np.int(nx / bin_x) nbins_y = np.int(ny / bin_y) im_var = np.zeros((nbins_y, nbins_x)) im_loc = np.zeros((nbins_y, nbins_x)) im_frac = np.zeros((nbins_y, nbins_x)) im_fluct = np.zeros((nbins_y, nbins_x)) for i in range(nbins_x - 1): for j in range(nbins_y - 1): x1, x2, y1, y2 = i * bin_x, (i + 1) * bin_x, j * bin_y, (j + 1) * bin_y im_sec = image[y1:y2, x1:x2] im_mask_sec = mask[y1:y2, x1:x2] im_sec_in = im_sec[(im_mask_sec == 0)] if im_sec_in.size > 0: im_var[j, i] = biweight_midvariance(im_sec_in) im_loc[j, i] = biweight_location(im_sec_in) im_frac[j, i] = 1 - np.float(im_sec_in.size) / np.float(im_sec.size) ''' temp = copy.deepcopy(image) temp[mask==1] = np.nan # var im_var = block_reduce(temp, (bin_y, bin_x), func=np.nanvar, cval=np.nan) # loc im_loc = block_reduce(temp, (bin_y, bin_x), func=np.nanmedian, cval=np.nan) # frac im_frac = block_reduce(mask, (bin_y, bin_x), func=np.sum, cval=np.nan) / (bin_x * bin_y) ''' # calculate fluctuation for i in range(1, nbins_x - 1): for j in range(1, nbins_y - 1): backvec = im_loc[j-1:j+2, i-1:i+2] backvec = np.delete(backvec.flatten(), 4) maskvec = im_frac[j-1:j+2, i-1:i+2] maskvec = np.delete(maskvec.flatten(), 4) # fraction of being masked out backvec_in = backvec[(maskvec < 1 - minfrac)] if len(backvec_in) > minback: im_fluct[j,i] = im_loc[j,i] - biweight_location(backvec_in) im_fluct_in = im_fluct[im_fluct != 0] sig_adu = np.sqrt(biweight_midvariance(im_fluct_in)) * 0.80 # 8/9 is area correction sig_adu *= sigma dsig_adu = sig_adu / np.sqrt(2 * (im_fluct_in.size - 1)) dsig_adu *= sigma # For the standard deviation of standard deviation, see this: https://stats.stackexchange.com/questions/631/standard-deviation-of-standard-deviation # convert to magnitudes sb_lim = zeropoint - 2.5 * np.log10(sig_adu / pixel_scale**2) dsb_lim = 2.5 * np.log10(1 + 1/np.sqrt(im_fluct_in.size)) #2.5 / np.log(10) / sig_adu * dsig_adu if verbose: if logger is not None: logger.info(' - {0:.1f}-sigma variation in counts = {1:.4f} +- {2:.4f}'.format(sigma, sig_adu, dsig_adu)) logger.info(' - {0:.1f}-sigma surface brightness limit on {1} arcsec scale is {2:.4f} +- {3:.4f}'.format(sigma, scale_arcsec, sb_lim, dsb_lim)) else: print(' - {0:.1f}-sigma variation in counts = {1:.4f} +- {2:.4f}'.format(sigma, sig_adu, dsig_adu)) print(' - {0:.1f}-sigma surface brightness limit on {1} arcsec scale is {2:.4f} +- {3:.4f}'.format(sigma, scale_arcsec, sb_lim, dsb_lim)) return (sb_lim, dsb_lim), (sig_adu, dsig_adu), [im_fluct, im_loc, im_var, im_frac]