def edge_outlier_clip(data_e, ri, vi, Zi):
r_inside = []
v_inside = []
i = 0
while ri[i] <= np.max(data_e['rad']):
inner_el = i
outer_el = i + 5
inner_r = ri[inner_el]
outer_r = ri[outer_el]
'''
dens = np.average(Zi[inner_el:outer_el],axis=0)
roots = np.sort(np.abs(vi[dens>0.05]))
databinned = data_e[np.where((data_e['rad']>=inner_r)&(data_e['rad']<outer_r))]
if len(roots) == 0:
root = 2 * biweight_midvariance(databinned['vel'].copy(),9.0)
elif np.abs(roots[-1]) < 500.0:
root = 2 * biweight_midvariance(databinned['vel'].copy(),9.0)
elif np.abs(roots[-1]) > 3500.0:
root = 3500.0
else:
root = np.abs(roots[-1])
r_inside.extend(databinned['rad'][np.where(np.abs(databinned['vel'])<root)])
v_inside.extend(databinned['vel'][np.where(np.abs(databinned['vel'])<root)])
i += 5
data_e = np.vstack((np.array(r_inside),np.array(v_inside))).T
return data_e
'''
# deriv = (np.average(Zi[inner_el:outer_el],axis=0)[1:]-np.average(Zi[inner_el:outer_el],axis=0)[:-1]) \
# /(vi[1:]-vi[:-1])
roots = np.sort(
np.abs(vi[((np.average(Zi[inner_el:outer_el], axis=0)[1:] -
np.average(Zi[inner_el:outer_el], axis=0)[:-1]) /
(vi[1:] - vi[:-1]))[1:] *
((np.average(Zi[inner_el:outer_el], axis=0)[1:] -
np.average(Zi[inner_el:outer_el], axis=0)[:-1]) /
(vi[1:] - vi[:-1]))[:-1] < 0]))
databinned = data_e[np.where((data_e['rad'] >= inner_r)
& (data_e['rad'] < outer_r))]
if len(roots) > 1:
if roots[1] < 1000.0:
if len(roots) > 2:
if roots[2] < 1000.0:
root = 3 * biweight_midvariance(
databinned['vel'].copy(), 9.0)
else:
root = roots[2]
else:
root = 3 * biweight_midvariance(
databinned['vel'].copy(), 9.0)
else:
root = roots[1]
else:
root = 3500.0
r_inside.extend(
databinned['rad'][np.where(np.abs(databinned['vel']) < root)])
v_inside.extend(
databinned['vel'][np.where(np.abs(databinned['vel']) < root)])
i += 5
data_e = np.vstack((np.array(r_inside), np.array(v_inside))).T
return data_e
def getbiweight(z):
biweightscale=biweight_midvariance(z)
biweightlocation=biweight_location(z)
flag=abs(z-biweightlocation)/biweightscale < scale_cut
#flag=np.ones(len(z),'bool')
repeatflag=1
nloop=0
#print biweightlocation, biweightscale
oldbiweightscale=biweightscale
while repeatflag:
newdata=z[flag]
biweightscale=biweight_midvariance(newdata, M=biweightlocation)
biweightlocation=biweight_location(newdata, M=biweightlocation)
oldflag = flag
#flag=abs(z-biweightlocation)/biweightscale < scale_cut
nloop += 1
#print nloop, biweightlocation, biweightscale, len(newdata), sum(flag)
#if sum(flag == oldflag) == len(flag):
# repeatflag=0
# print 'nloop = ', nloop
if abs(biweightscale - oldbiweightscale) < 1.:
repeatflag=0
#print 'nloop = ', nloop
if nloop > 5:
repeatflag = 0
oldbiweightscale=biweightscale
#print nloop, biweightlocation, biweightscale
#flag=abs(z-biweightlocation)/biweightscale < 4.
#biweightscale=biweight_midvariance(z[flag],M=biweightlocation)
return biweightlocation, biweightscale
def finddups(line):
#Filter the data
allgood = dataqual[line+'_good'].map(d)
#Needs to be bad in any line to be bad
baddata = dataqual[line+'_bad'].map(d)
somelow = dataqual[line+'_low'].map(d)
goodflux = fluxdata[np.logical_not(baddata)]
#Get the duplicate OBJIDs of only the good objects
dupids = [item for item, count in collections.Counter(goodflux['OBJID']).items() if count > 1]
dupobjsarr = []
lineflux1 = []
lineflux2 = []
lineflux3 = []
lineflux4 = []
lineflux5 = []
lineflux6 = []
#Append the data for the good objects
for obj in dupids:
dupobjsarr.append(fluxdata[fluxdata.OBJID == obj])
#Get the ews for these objects
for i in range(0,len(dupobjsarr)):
lineflux1.append(dupobjsarr[i].iloc[0][line+'_flux'])
lineflux2.append(dupobjsarr[i].iloc[1][line+'_flux'])
lineflux3.append(dupobjsarr[i].iloc[0][line+'_flux'])
lineflux4.append(dupobjsarr[i].iloc[1][line+'_flux'])
lineflux5.append(dupobjsarr[i].iloc[0][line+'_modelabs'])
lineflux6.append(dupobjsarr[i].iloc[1][line+'_modelabs'])
lineflux3 = np.array(lineflux3)
lineflux4 = np.array(lineflux4)
lineflux5 = np.array(lineflux5)
lineflux6 = np.array(lineflux6)
#Make another cut for high S/N
flagidx = allgood
#Repeat the process just for the high SN objects
goodflux = fluxdata[flagidx]
linefluxsig1 = []
linefluxsig2 = []
dupids = [item for item, count in collections.Counter(goodflux['OBJID']).items() if count > 1]
dupobjsarr = []
for obj in dupids:
dupobjsarr.append(fluxdata[fluxdata.OBJID == obj])
for i in range(0,len(dupobjsarr)):
linefluxsig1.append(dupobjsarr[i].iloc[0][line+'_flux'])
linefluxsig2.append(dupobjsarr[i].iloc[1][line+'_flux'])
fluxdiffmed = np.median(np.abs(lineflux4-lineflux3))
fluxdiffbi = np.sqrt(biweight_midvariance(np.abs(lineflux4-lineflux3)))
absdiffmed = np.median(np.abs(lineflux6-lineflux5))
absdiffbi = np.sqrt(biweight_midvariance(np.abs(lineflux6-lineflux5)))
diffarr = [fluxdiffmed, fluxdiffbi, absdiffmed, absdiffbi]
return lineflux1,lineflux2, linefluxsig1, linefluxsig2, diffarr
def find_matches(sources, xc, yc):
''' Matching sources using closest neighbor and clipping '''
dx = (sources['xcentroid'][:, np.newaxis] - xc)
dy = (sources['ycentroid'][:, np.newaxis] - yc)
dd = np.sqrt(dx**2 + dy**2)
ind = np.argmin(dd, axis=1)
dxk = dx[np.arange(dx.shape[0]), ind]
dyk = dy[np.arange(dy.shape[0]), ind]
bvx = biweight_midvariance(dxk)
bvy = biweight_midvariance(dyk)
mvx = biweight_location(dxk)
mvy = biweight_location(dyk)
sel = np.where(
(np.abs(dxk - mvx) < 3. * bvx) * (np.abs(dyk - mvy) < 3. * bvy))[0]
return sel, ind[sel]
def calculate_statistics_no_clipping(self):
"""
This function ...
:return:
"""
# Compress (remove masked values)
flattened = np.ma.array(self.sources_with_noise.data, mask=self.rotation_mask.data).compressed()
median = np.median(flattened)
biweight_loc = biweight_location(flattened)
biweight_midvar = biweight_midvariance(flattened)
median_absolute_deviation = mad_std(flattened)
#print("median", median)
#print("biweigth_loc", biweight_loc)
#print("biweight_midvar", biweight_midvar)
#print("median_absolute_deviation", median_absolute_deviation)
self.statistics.no_clipping = Map()
self.statistics.no_clipping.median = median
self.statistics.no_clipping.biweight_loc = biweight_loc
self.statistics.no_clipping.biweight_midvar = biweight_midvar
self.statistics.no_clipping.median_absolute_deviation = median_absolute_deviation
def dispersionYasX(x, y, nbins=10):
'''
Dispersion of Y as a function of X
Parameters
----------
x : array
Independent variable
y : array
Dependent variable
nbins : int
Number of bins
Return
------
dx : array
Binned x
sigmay : array
Dispersion of y
'''
dx = np.zeros(nbins)
sigmay = np.zeros(nbins)
xstep = (np.amax(x) - np.amin(x)) / nbins
xbin = np.amin(x)
for i in range(nbins):
mask = np.logical_and(x >= xbin, x < xbin + xstep)
ybin = y[mask]
sigmay[i] = np.sqrt(biweight_midvariance(ybin))
#sigmay[i] = np.std(ybin)
dx[i] = xbin + 0.5 * xstep
xbin += xstep
return dx, sigmay
def calculate_statistics_no_clipping(self):
"""
This function ...
:return:
"""
# Compress (remove masked values)
flattened = np.ma.array(self.sources_with_noise.data,
mask=self.rotation_mask.data).compressed()
median = np.median(flattened)
biweight_loc = biweight_location(flattened)
biweight_midvar = biweight_midvariance(flattened)
median_absolute_deviation = mad_std(flattened)
#print("median", median)
#print("biweigth_loc", biweight_loc)
#print("biweight_midvar", biweight_midvar)
#print("median_absolute_deviation", median_absolute_deviation)
self.statistics.no_clipping = Map()
self.statistics.no_clipping.median = median
self.statistics.no_clipping.biweight_loc = biweight_loc
self.statistics.no_clipping.biweight_midvar = biweight_midvar
self.statistics.no_clipping.median_absolute_deviation = median_absolute_deviation
def calculateSeptZPs():
zp = []
for num in range(len(oct_extracted)):
thefile = oct_extracted[num]
mags, fluxes = [], []
something = np.array(masteroctlines[num])
with open(thefile, 'r') as f:
with open(septextracted[0], 'r') as fs:
lines = f.readlines()
linessept = fs.readlines()
for x in lines:
if x.startswith("N"):
continue
elif int(x.split(' ')[0]) in masteroctlines[num]:
mags.append(float(x.split(' ')[len(x.split(' ')) - 1]))
b = masterseptlines[num][np.where(
something == int(x.split(' ')[0]))[0][0]]
if int(linessept[b].split(' ')[0]) in septstars:
fluxes.append(
float(linessept[b].split(' ')[convergesept]))
else:
mags.pop()
for z in range(len(mags)):
zp.append(mags[z] + 2.5 * math.log10(fluxes[z]))
plt.figure(69)
med = median(zp)
bwmv = st.biweight_midvariance(zp)
plt.hist(zp, bins=15,
color='g'), plt.title("September Zeropoints Histogram")
plt.text(min(zp), 7, "Med: %s\n BMWV: %s" % (med, bwmv))
if writeup == True:
plt.show()
plt.savefig('septzeropoints.png'), plt.clf(), plt.close()
pm = u'\u00B1'
print "Zeropoint for September: %s %s %s" % (med, pm, bwmv)
def robust_std(im, method='biweight'):
# get robust stdev
# Method can be biweight or mad, otherwise just normal stdev
# im must be numpy array
from astropy.stats import biweight_midvariance
from scipy.stats import median_absolute_deviation
if method == 'biweight':
var = biweight_midvariance(im, axis=None, ignore_nan=True)
std = np.sqrt(var)
elif method == 'mad':
std = median_absolute_deviation(im, axis=None, nan_policy='omit')
else:
std = np.nanstd(im, axis=None)
# From Mike's code:
"""
m = np.nanmedian(im) # median value
d = im - m # deviation
ad = np.abs(d) # absolute deviation
mad = np.nanmedian(ad) # median absolute deviation
if mad == 0: std = 0. # no deviation -> zero stdev
else:
wt = biweight(d/1.483/mad) # weights
sum_wt = wt.sum()
sum_wt2 = (wt**2).sum()
m = (im*wt).sum() / sum_wt # weighted mean
d = im-m # deviation from weighted mean
var = (d**2 * wt).sum() / (sum_wt-sum_wt2/sum_wt) # weighted var
std = n.sqrt(var) # weighted stdev
"""
return std
def calculateOctZPs():
global octzp, octzperr
twommatch, octmatch, Mag, flux, octlineno, chipno = [], [], [], [], [], []
with open("2mass_Oct_matched.txt", 'r') as f:
lines = f.readlines()
for x in lines:
x = x.strip()
line = x.split()
if not x or x.startswith('#'):
continue
twommatch.append((int(line[4]) - 1))
octmatch.append((int(line[5]) - 1))
if len(twommatch) != len(set(twommatch)) or len(octmatch) != len(
set(octmatch)):
print "Error: tweak xyxymatch parameters so that a coordinate in one set of data isn't matched to " \
"multiple coordinates in the other."
sys.exit()
with open('2mass_RADec.txt', 'r') as f:
temptwo_m = np.array(twommatch)
with open('Oct_RADec.txt', 'r') as fs:
lines = f.readlines()
linesoct = fs.readlines()
i = 1
for x in lines:
z = x.split('#')
if x.startswith("#"):
continue
elif i in twommatch:
b = octmatch[np.where(temptwo_m == i)[0][0]]
Mag.append(z[1])
p = linesoct[b].split(' ')
flux.append(p[4].replace('\n', ''))
octlineno.append(p[2])
chipno.append(p[3])
i += 1
zp = []
for num in range(0, len(Mag)):
if float(flux[num]) < 1:
continue
zp.append(
float(Mag[num]) + float(2.5 * (math.log10(float(flux[num])))))
# if float(Mag[num]) + float(2.5 * (math.log10(float(flux[num])))) < 26.4:
# print flux[num]
octzp, octzperr = median(zp), st.biweight_midvariance(zp)
print "%s zeropoints calculated in October" % (len(zp))
plt.figure(30), plt.hist(
zp, bins=15), plt.title("October Zeropoints Histogram")
plt.text(min(zp), len(zp) / 5, "Med: %s \n BWMV: %s" % (octzp, octzperr))
axes = plt.gca()
# axes.set_xlim([25, 28])
if writeup == True:
plt.show()
plt.savefig('oct_zps.png', bbox_inches='tight'), plt.close()
def binMeanStd(x, y, nbins=10):
'''
Compute binned median and standard deviation
Parameters
----------
x : array
Independent variable
y : array
Dependent variable
nbins : int
Number of bins
Return
------
data : array
Binned medean and standard deviations for both x and y
'''
#-----------------------------------------------------------------------------------------
# Sort x and y according to x
indc = np.argsort(x)
xsort, ysort = x[indc], y[indc]
# Compute mean and standard deviation for each bin
nstars = len(xsort)
nstars_bin = int(nstars / nbins + 1.)
data = np.zeros((nbins, 4))
for i in range(nbins - 1):
data[i, 0] = np.median(xsort[i * nstars_bin:(i + 1) * nstars_bin])
data[i, 1] = np.median(ysort[i * nstars_bin:(i + 1) * nstars_bin])
data[i, 2] = np.sqrt(
biweight_midvariance(xsort[i * nstars_bin:(i + 1) * nstars_bin]))
data[i, 3] = np.sqrt(
biweight_midvariance(ysort[i * nstars_bin:(i + 1) * nstars_bin]))
data[nbins - 1, 0] = np.median(xsort[(nbins - 1) * nstars_bin:nstars])
data[nbins - 1, 1] = np.median(ysort[(nbins - 1) * nstars_bin:nstars])
data[nbins - 1, 2] = np.sqrt(
biweight_midvariance(xsort[(nbins - 1) * nstars_bin:nstars]))
data[nbins - 1, 3] = np.sqrt(
biweight_midvariance(ysort[(nbins - 1) * nstars_bin:nstars]))
return data
def findvdisp(self, r, v, r200, maxv):
"""
Use astropy.stats biweight sigma clipping variance for the velocity dispersion
"""
v_cut = v[np.where((r < r200) & (np.abs(v) < maxv))]
try:
self.gal_vdisp = biweight_midvariance(
v_cut[np.where(np.isfinite(v_cut))], 9.0)
except:
self.gal_vdisp = np.std(v_cut, ddof=1)
def _stats_data(self, stats, mask, scipy, astropy, decimals_mode):
data = self.data
# The original data size, for computation of valid elements and how
# many are masked/invalid.
size_initial = data.size
# Delete masked values, this will directly convert it to a 1D array
# if the mask is not appropriate then ravel it.
data = data[~mask]
size_masked = data.size
# Delete invalid (NaN, Inf) values. This should ensure that the result
# is always a 1D array
data = data[np.isfinite(data)]
size_valid = data.size
stats['elements'] = [size_valid]
stats['min'] = [np.amin(data)]
stats['max'] = [np.amax(data)]
stats['mean'] = [np.mean(data)]
stats['median'] = [np.median(data)]
# Use custom mode defined in this package because scipy.stats.mode is
# very, very slow and by default tries to calculate the mode along
# axis=0 and not for the whole array.
# Take the first element since the second is the number of occurences.
stats['mode'] = [mode(data, decimals=decimals_mode)[0]]
if astropy:
stats['biweight_location'] = [biweight_location(data)]
stats['std'] = [np.std(data)]
if astropy:
stats['mad'] = [mad_std(data)]
stats['biweight_midvariance'] = [biweight_midvariance(data)]
stats['var'] = [np.var(data)]
if scipy: # pragma: no cover
if not OPT_DEPS['SCIPY']:
log.info('SciPy is not installed.')
else:
# Passing axis=None should not be important since we already
# boolean indexed the array and it's 1D. But it's important
# to remember that there default is axis=0 and not axis=None!
stats['skew'] = [skew(data, axis=None)]
stats['kurtosis'] = [kurtosis(data, axis=None)]
stats['masked'] = [size_initial - size_masked]
stats['invalid'] = [size_masked - size_valid]
return data
def fboot(y):
"""
Helper function of trendplot. Returns bootstrap error of input 1d-array.
Calculates the biweight mean of each bootstrap sample, and then gets the
biweight standard deviation of all samples.
"""
if len(y) > 5:
bb = bootstrap(y, bootnum=250, bootfunc=biweight_location)
res = biweight_midvariance(bb)
else:
res = np.float("nan")
return res
def calc_scat(self,
mass,
observable,
loglog=True,
binobs=True,
step=0.05,
data_cut=None):
''' Does Mass Mixing Statistics '''
## First Calculate Scatter of Richness Relationship
# Get Data Cut
cut = np.where(observable > 0)
if obs_cut == None:
print ' Find observable value above which vertical scatter is fully encompassed'
mp.plot(np.log(observable[cut]),
np.log(true_mass[cut]),
'ko',
alpha=.75)
mp.xlabel('Observable', fontsize=16)
mp.ylabel('M200', fontsize=16)
mp.show()
obs_cut = input('What is that value? (Enter as a float, ex. 1.5) : ')
# Correct obs_cut
if type(obs_cut) == int or type(obs_cut) == float:
obs_cut = np.array([obs_cut])
# Now Calculate Scatter
SCAT = []
for i in range(len(obs_cut)):
data_cut = np.where(np.log(observable) > obs_cut[i])[0]
print 'Number of Data Points:', len(data_cut)
step = step
obs_max = np.log(observable[data_cut]).max()
bins = np.arange(obs_cut[i], obs_max, step)
mass_bins = np.array(
map(
lambda x: (true_mass[data_cut])[np.where(
(np.log(observable[data_cut]) > x) &
(np.log(observable[data_cut]) < (x + step)))], bins))
mass_avgs = np.array(map(lambda x: np.median(x), mass_bins))
mass_fracs = []
for j in range(len(mass_avgs)):
mass_fracs.extend(np.log(mass_bins[j] / mass_avgs[j]))
mass_fracs = np.array(mass_fracs)
scat = astats.biweight_midvariance(mass_fracs)
SCAT.append(scat)
return np.array(SCAT)
def chisq_metric(self, chisq, chisq_std_zscore_cut=4.0, return_dict=True):
"""
chi square metrics
chisq : ndarray, shape=(Nants, Ntimes, Nfreqs)
ndarray containing chisq for each antenna (single pol)
chisq_std_cut : float, default=5.0
sigma tolerance (or z-score tolerance) for std of chisq fluctuations
return_dict : bool, default=True
return per-antenna metrics as a dictionary, with antenna number as key
rather than an ndarray
"""
# Get chisq statistics
chisq_avg = np.median(np.median(chisq[:, :, self.band], axis=0),
axis=1).astype(np.float64)
chisq_tot_avg = astats.biweight_location(
chisq[:, :, self.band]).astype(np.float64)
chisq_ant_avg = np.array(
list(map(astats.biweight_location,
chisq[:, :, self.band]))).astype(np.float64)
chisq_ant_std = np.sqrt(
np.array(
list(map(astats.biweight_midvariance, chisq[:, :,
self.band]))))
chisq_ant_std_loc = astats.biweight_location(chisq_ant_std).astype(
np.float64)
chisq_ant_std_scale = np.sqrt(
astats.biweight_midvariance(chisq_ant_std))
chisq_ant_std_zscore = (chisq_ant_std -
chisq_ant_std_loc) / chisq_ant_std_scale
chisq_ant_std_zscore_max = np.max(np.abs(chisq_ant_std_zscore))
chisq_good_sol = chisq_ant_std_zscore_max < chisq_std_zscore_cut
# convert to dictionaries
if return_dict is True:
chisq_ant_std = odict(zip(self.ant_array, chisq_ant_std))
chisq_ant_avg = odict(zip(self.ant_array, chisq_ant_avg))
return (chisq_avg, chisq_tot_avg, chisq_ant_avg, chisq_ant_std,
chisq_ant_std_loc, chisq_ant_std_scale, chisq_ant_std_zscore,
chisq_ant_std_zscore_max, chisq_good_sol)
def _pixel_stats(pixels, clip_sig=3, n_clip=10, min_pix=50):
"""
Calculate mode and scale of pixel distribution, clipping outliers. Uses
biweight as "robust" estimator of these quantities.
:param pixels: Array to calculate statistics for
:param clip_sig: Sigma value at which to clip outliers
:param n_clip: Number of clipping iterations
:param min_pix: Minimum number of retained pixels
:return: 2-tuple of distribution mode, scale
"""
clip_iter = 0
mode, scale = 0, 1
mask = np.ones(pixels.shape, dtype=bool)
while True:
mode = biweight_location(pixels[mask])
scale = biweight_midvariance(pixels[mask])
mask &= np.abs(pixels - mode) < clip_sig * scale
clip_iter += 1
if np.sum(mask) < min_pix or clip_iter >= n_clip:
break
return mode, scale
def calc_scat(self,mass,observable,loglog=True,binobs=True,step=0.05,data_cut=None):
''' Does Mass Mixing Statistics '''
## First Calculate Scatter of Richness Relationship
# Get Data Cut
cut = np.where(observable > 0)
if obs_cut == None:
print ' Find observable value above which vertical scatter is fully encompassed'
mp.plot(np.log(observable[cut]),np.log(true_mass[cut]),'ko',alpha=.75)
mp.xlabel('Observable',fontsize=16)
mp.ylabel('M200',fontsize=16)
mp.show()
obs_cut = input('What is that value? (Enter as a float, ex. 1.5) : ')
# Correct obs_cut
if type(obs_cut) == int or type(obs_cut) == float: obs_cut = np.array([obs_cut])
# Now Calculate Scatter
SCAT = []
for i in range(len(obs_cut)):
data_cut = np.where(np.log(observable) > obs_cut[i])[0]
print 'Number of Data Points:',len(data_cut)
step = step
obs_max = np.log(observable[data_cut]).max()
bins = np.arange(obs_cut[i],obs_max,step)
mass_bins = np.array( map(lambda x: (true_mass[data_cut])[np.where((np.log(observable[data_cut])>x)&(np.log(observable[data_cut])<(x+step)))], bins))
mass_avgs = np.array(map(lambda x:np.median(x),mass_bins))
mass_fracs = []
for j in range(len(mass_avgs)):
mass_fracs.extend(np.log(mass_bins[j] / mass_avgs[j]))
mass_fracs = np.array(mass_fracs)
scat = astats.biweight_midvariance(mass_fracs)
SCAT.append(scat)
return np.array(SCAT)
def richness_est(self,ra,dec,z,pz,gmags,rmags,imags,abs_rmags,haloid,clus_ra,clus_dec,clus_z,clus_rvir=None,gr_slope=None,gr_width=None,gr_inter=None,shiftgap=True,use_specs=False,use_bcg=False,fit_rs=False,spec_list=None,fixed_aperture=False,fixed_vdisp=False,plot_gr=False,plot_sky=False,plot_phase=False,find_pairs=True):
''' Richness estimator, magnitudes should be apparent mags except for abs_rmag
z : spectroscopic redshift
pz : photometric redshift
gr_slope : if fed color-mag (g-r vs. r) slope of fit to red sequence
gr_width : if fed color-mag (g-r vs. r) width of fit to red sequence
gr_inter : if red color-mag (g-r vs. r) intercept of fit to red sequence
spec_list : indexing array specifying gals w/ spectroscopic redshifts, preferably fed as boolean numpy array
fixed_aperture : clus_rvir = 1 Mpc
fixed_vdisp : clus_vdisp = 1000 km/s (members < clus_vdisp*2)
use_specs : includes all spectro members in richness regardless of their color, subject to counting twice
use_bcg : use bcg RS_color as cluster RS_color
fit_rs : fit a line to the member galaxy RS relationship, as of now this does not work and we take a flat RS relationship
plot_gr : plot r vs g-r color diagram with RS fits
plot_sky : plot RA and DEC of gals with signal and background annuli
plot_phase : plot rdata and vdata phase space
find_pairs : run c4 cluster pair identifier on phase spaces
- If at least one quarter of the outer annuli doesn't have any galaxies (perhaps because it has run over the observation edge),
it will return -99 as a richness
'''
# Put Into Class Namespace
keys = ['ra','dec','z','pz','gmags','rmags','imags','clus_ra','clus_dec','clus_z','clus_rvir','vel_disp','color_data','color_cut','RS_color','RS_sigma','clus_color_cut','signal','background','richness','mems','phot','spec','spec_mems','deg_per_rvir','SA_outer','SA_inner','outer_red_dense','inner_background','Nphot_inner','Nphot_outer','red_inner','red_outer','all','outer_edge','inner_edge','shift_cut','set','pair','Nspec','gr_slope','gr_inter','gr_width']
self.__dict__.update(ez.create(keys,locals()))
# Take rough virial radius measurement, this method is BAD...
if clus_rvir == None:
clus_rvir = np.exp(-1.86)*len(np.where((rmags < -19.55) & (rdata < 1.0) & (np.abs(vdata) < 3500))[0])**0.51
self.clus_rvir = clus_rvir
# Set clus_rvir = 1.0 Mpc if fixed aperture == True
if fixed_aperture == True:
clus_rvir = 1.0
# Magnitue Cut at 19.1 Apparent R Mag, then at -19 Absolute R Mag and sort by them
bright = np.where(rmags < 19.1)[0]
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[bright].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
bright = np.where(abs_rmags < -19.5)[0]
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[bright].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
sorts = np.argsort(abs_rmags)
data = np.vstack([ra,dec,z,pz,gmags,rmags,imags,abs_rmags])
data = data.T[sorts].T
ra,dec,z,pz,gmags,rmags,imags,abs_rmags = data
self.__dict__.update(ez.create(keys,locals()))
# Separate Into Spectro Z and Photo Z DataSets
# Define the Spectro Z catalogue
if spec_list == None:
spec_cut = np.abs(z) > 1e-4
else:
if type(spec_list) == list: spec_list = np.array(spec_list)
if spec_list.dtype != 'bool':
spec_cut = np.array([False]*len(ra))
spec_cut[spec_list] = True
all = {'ra':ra,'dec':dec,'z':z,'pz':pz,'gmags':gmags,'rmags':rmags,'imags':imags,'abs_rmags':abs_rmags}
spec = {'ra':ra[spec_cut],'dec':dec[spec_cut],'z':z[spec_cut],'pz':pz[spec_cut],'gmags':gmags[spec_cut],'rmags':rmags[spec_cut],'imags':imags[spec_cut],'abs_rmags':abs_rmags[spec_cut]}
phot = {'ra':ra[~spec_cut],'dec':dec[~spec_cut],'z':z[~spec_cut],'pz':pz[~spec_cut],'gmags':gmags[~spec_cut],'rmags':rmags[~spec_cut],'imags':imags[~spec_cut],'abs_rmags':abs_rmags[~spec_cut]}
# Project Spectra into radius and velocity
ang_d,lum_d = C.zdistance(clus_z,self.H0)
angles = C.findangle(spec['ra'],spec['dec'],clus_ra,clus_dec)
rdata = angles * ang_d
vdata = self.c * (spec['z'] - clus_z) / (1 + clus_z)
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
# Take Hard Phasespace Limits
limit = np.where( (rdata < 5) & (np.abs(vdata) < 5000) )[0]
clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']])
clus_data = clus_data.T[limit].T
rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
# Shiftgapper for Interlopers
if shiftgap == True:
len_before = np.where((rdata < clus_rvir*1.5)&(np.abs(vdata)<4000))[0].size
clus_data = np.vstack([rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags']])
clus_data = C.shiftgapper(clus_data.T).T
sorts = np.argsort(clus_data[-1])
clus_data = clus_data.T[sorts].T
rdata,vdata,spec['ra'],spec['dec'],spec['z'],spec['pz'],spec['gmags'],spec['rmags'],spec['imags'],spec['abs_rmags'] = clus_data
shift_cut = len_before - np.where((rdata < clus_rvir)&(np.abs(vdata)<4000))[0].size
# Measure Velocity Dispersion of all galaxies within 1 * r_vir and np.abs(vdata) < 4000 km/s
spec.update({'rdata':rdata,'vdata':vdata})
self.__dict__.update(ez.create(keys,locals()))
vel_disp = astats.biweight_midvariance(vdata[np.where((rdata < clus_rvir*1)&(np.abs(vdata) < 4000))])
if fixed_vdisp == True: vel_disp = 1000
# Run Cluster Pair Finder
if find_pairs == True:
pair, d1_chi, d2_chi, d3_chi, s_chi, double1, double2, double3, single, v_range, bins = pair_find(rdata,vdata)
# Calculate Nspec, Nspec = # of galaxies within RVIR
try:
Nspec = np.where(rdata<clus_rvir)[0].size
except:
Nspec = 0
self.__dict__.update(ez.create(keys,locals()))
# Get Members from Specta, get their red sequence color
mems = np.where((spec['rdata'] < clus_rvir)&(np.abs(spec['vdata'])<2*vel_disp))[0]
color_data = spec['gmags'] - spec['rmags']
color_cut = np.where((color_data[mems] < 1.2) & (color_data[mems] > 0.65))[0]
RS_color = astats.biweight_location(color_data[mems][color_cut])
RS_shift = color_data[mems][color_cut] - RS_color
RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)])
if fit_rs == True:
clf = linear_model.LinearRegression()
set = np.where(np.abs(color_data[mems]-RS_color)<3*RS_sigma)[0]
clf.fit(spec['rmags'][mems][set].reshape(set.size,1),color_data[mems][set])
if use_bcg == True:
bright = np.argsort(all['abs_rmags'][mems])[0]
RS_color = color_data[mems][bright]
RS_shift = color_data[mems][color_cut] - RS_color
RS_sigma = astats.biweight_midvariance(RS_shift[np.where(np.abs(RS_shift)<.15)])
# spec_mems is # of members that are within 2 sigma of cluster color
clus_color_cut = np.where(np.abs(color_data[mems] - RS_color) < RS_sigma*2)[0]
spec_mems = len(clus_color_cut)
self.__dict__.update(ez.create(keys,locals()))
# If fed gr fit
if gr_slope != None:
def RS_color(r_mag): return r_mag*gr_slope + gr_inter
RS_sigma = gr_width / 2
clus_color_cut = np.where(np.abs(color_data[mems] - RS_color(spec['rmags'])[mems]) < RS_sigma*2)[0]
spec_mems = len(clus_color_cut)
self.__dict__.update(ez.create(keys,locals()))
# Get Rdata from PhotoZ & SpecZ Data Set
angles = C.findangle(all['ra'],all['dec'],clus_ra,clus_dec)
all['rdata'] = angles * ang_d
# Get deg per RVIR proper
deg_per_rvir = R.Cosmo.arcsec_per_kpc_proper(clus_z).value * 1e3 * clus_rvir / 3600
# Get Area of Virial Circle out to 1 rvir, and Area of outer annuli, which is annuli from 4rvir < R < 6rvir, or dependent on rvir
if clus_rvir < 2.5:
outer_edge = 6.0
inner_edge = 4.0
elif clus_rvir >= 2.5 and clus_rvir < 3:
outer_edge = 5.0
inner_edge = 3.5
else:
outer_edge = 3.0
inner_edge = 2.0
SA_inner = np.pi*deg_per_rvir**2
SA_outer = np.pi * ( (outer_edge*deg_per_rvir)**2 - (inner_edge*deg_per_rvir)**2 )
# Get Number of Cluster Color Galaxies from Photo Data Set in Inner Circle and Outer Annuli
RS_color_sig = 2.0
if gr_slope == None:
red_inner = np.where(((all['gmags']-all['rmags']) < RS_color + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0]
red_outer = np.where(((all['gmags']-all['rmags']) < RS_color + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0]
else:
red_inner = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + RS_color_sig*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - RS_color_sig*RS_sigma)&(all['rdata']<clus_rvir))[0]
red_outer = np.where(((all['gmags']-all['rmags']) < RS_color(all['rmags']) + 1.5*RS_sigma)&((all['gmags']-all['rmags']) > RS_color(all['rmags']) - 1.5*RS_sigma)&(all['rdata']<outer_edge*clus_rvir)&(all['rdata']>inner_edge*clus_rvir))[0]
Nphot_inner = len(red_inner)
Nphot_outer = len(red_outer)
self.__dict__.update(ez.create(keys,locals()))
# Get Solid Angle Density of Outer Red Galaxies
outer_red_dense = Nphot_outer / SA_outer
inner_background = int(np.ceil(outer_red_dense * SA_inner))
# If inner_background is less than Nphot_inner, then Nphot_inner -= inner_background, otherwise Nphot_inner = 0
if inner_background < Nphot_inner:
Nphot_inner -= inner_background
else:
Nphot_inner = 0
# Richness = spec_mems + Nphot_inner or just Nphot_inner
if use_specs == True:
richness = spec_mems + Nphot_inner
else:
richness = Nphot_inner
self.__dict__.update(ez.create(keys,locals()))
# Plot
if plot_gr == True:
fig,ax = mp.subplots()
ax.plot(rmags,gmags-rmags,'ko',alpha=.8)
ax.plot(spec['rmags'][mems],color_data[mems],'co')
if gr_slope == None:
ax.axhline(RS_color,color='r')
ax.axhline(RS_color+RS_sigma,color='b')
ax.axhline(RS_color-RS_sigma,color='b')
else:
xdata = np.arange(spec['rmags'][mems].min(),spec['rmags'][mems].max(),.1)
ax.plot(xdata,RS_color(xdata),color='r')
ax.plot(xdata,RS_color(xdata)+RS_sigma,color='b')
ax.plot(xdata,RS_color(xdata)-RS_sigma,color='b')
ax.set_xlim(13,19)
ax.set_ylim(0,1.3)
ax.set_xlabel('Apparent R Mag',fontsize=16)
ax.set_ylabel('App G Mag - App R Mag',fontsize=16)
ax.set_title('Color-Mag Diagram, Cluster '+str(haloid))
fig.savefig('colormag_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
if plot_sky == True:
fig,ax = mp.subplots()
ax.plot(all['ra'],all['dec'],'ko')
ax.plot(all['ra'][red_inner],all['dec'][red_inner],'ro')
ax.plot(all['ra'][red_outer],all['dec'][red_outer],'yo')
ax.plot(clus_ra,clus_dec,'co',markersize=9)
ax.set_xlabel('RA',fontsize=15)
ax.set_ylabel('Dec.',fontsize=15)
ax.set_title('Richness Annuli for Halo '+str(haloid))
fig.savefig('skyplot_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
if plot_phase == True:
fig,ax = mp.subplots()
ax.plot(spec['rdata'],spec['vdata'],'ko')
ax.plot(spec['rdata'][mems],spec['vdata'][mems],'co')
bcg = np.where(spec['abs_rmags']==spec['abs_rmags'][mems].min())[0][0]
ax.plot(spec['rdata'][bcg],spec['vdata'][bcg],'ro')
ax.set_xlim(0,5)
ax.set_ylim(-5000,5000)
ax.set_xlabel('Radius (Mpc)',fontsize=15)
ax.set_ylabel('Velocity (km/s)',fontsize=15)
ax.set_title('phasespace haloid '+str(haloid))
fig.savefig('phasespace_'+str(haloid)+'.png',bbox_inches='tight')
mp.close(fig)
# Check to make sure galaxies exist (somewhat) uniformely in outer annuli (aka. cluster isn't on edge of observation strip)
# First, project all galaxies into polar coordinates centered on cluster center
x = (all['ra']-clus_ra)/np.cos(clus_dec*np.pi/180) # x coordinate in arcsec (of dec) centered on cluster center
y = (all['dec']-clus_dec)
all['radius'] = np.sqrt( x**2 + y**2 ) / deg_per_rvir #radius scaled by RVIR
all['theta'] = np.arctan( y / x )
# Add corrections to arctan function
all['theta'][np.where( (x < 0) & (y > 0) )] += np.pi # Quadrant II
all['theta'][np.where( (x < 0) & (y < 0) )] += np.pi # Quadrant III
all['theta'][np.where( (x > 0) & (y < 0) )] += 2*np.pi # Quadrant IV
# Then break outer annuli into 4 sections and check if at least 1 galaxy exists in each section
sizes1 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(0,2*np.pi,4)])
# Do it again but shift theta by np.pi/8 this time
sizes2 = np.array([np.where((np.abs(all['theta']-i)<=np.pi/2)&(all['radius']>inner_edge)&(all['radius']<15))[0].size for i in np.linspace(np.pi/8,17*np.pi/8,4)])
if 0 in sizes1 or 0 in sizes2:
mp.plot(all['radius'],all['theta'],'ko')
mp.xlabel('radius')
mp.ylabel('theta')
mp.savefig('rth_'+str(haloid)+'.png')
mp.close()
print 'sizes1=',sizes1
print 'sizes2=',sizes2
return -99
return richness
loc3 = np.sin(67.5 / 180 * np.pi)
mr1 = (fluxdata[filters]['ar'] < loc1)
mr2 = np.logical_and(fluxdata[filters]['ar'] >= loc1,
fluxdata[filters]['ar'] < loc2)
mr3 = np.logical_and(fluxdata[filters]['ar'] >= loc2,
fluxdata[filters]['ar'] < loc3)
mr4 = (fluxdata[filters]['ar'] >= loc3)
med1 = np.median(fluxdata[filters][mr1].av)
med2 = np.median(fluxdata[filters][mr2].av)
med3 = np.median(fluxdata[filters][mr3].av)
med4 = np.median(fluxdata[filters][mr4].av)
med751 = np.percentile(fluxdata[filters][mr1].av, 75)
med752 = np.percentile(fluxdata[filters][mr2].av, 75)
med753 = np.percentile(fluxdata[filters][mr3].av, 75)
med754 = np.percentile(fluxdata[filters][mr4].av, 75)
emed1 = np.sqrt(biweight_midvariance(fluxdata[filters][mr1].av))
emed2 = np.sqrt(biweight_midvariance(fluxdata[filters][mr2].av))
emed3 = np.sqrt(biweight_midvariance(fluxdata[filters][mr3].av))
emed4 = np.sqrt(biweight_midvariance(fluxdata[filters][mr4].av))
ax.errorbar(med1,
loc1 / 2,
xerr=(emed1 / len(fluxdata[filters][mr1])),
color='red',
marker='o',
ms=ms,
lw=lwbw,
ls='None')
ax.errorbar(med2, (loc1 + loc2) / 2,
xerr=(emed2 / len(fluxdata[filters][mr2])),
color='red',
marker='o',
ticksize = 16
titlefont = 24
legendfont = 16
textfont = 16
fig, axarr5 = plt.subplots(3, 4, figsize=(30, 20))
axarr5 = np.reshape(axarr5, 12)
counter = 0
for line in lines:
#Compute the scale dataframe and make a plot to display it
ax = axarr5[counter]
medscale = np.median(
np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line + '_scale']))
sigscale = np.sqrt(
biweight_midvariance(
np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line +
'_scale'])))
scale_df.at[0, line + '_medscale'] = medscale
scale_df.at[0, line + '_sigscale'] = sigscale
#Make the plot
bins = np.log10(np.arange(0.05, 4, 0.05))
ax.hist(np.log10(fluxdata[fluxdata[line + '_scale'] > 0][line + '_scale']),
bins=bins,
color='grey',
label=None)
#Plot the median, 1sig, and 3 sig stddevs
ax.plot((medscale, medscale), (-100, 1000),
color='red',
ls='-',
label='Median')
ax.plot((medscale - sigscale, medscale - sigscale), (-100, 1000),
def histogram2(band, band_ref, best_weights, training_data, aeg_models,
chisquare_models):
weights = best_weights[-5, 0:len(band)]
Z_bef = np.zeros((len(training_data)))
age_bef = np.zeros((len(training_data)))
ebv_bef = np.zeros((len(training_data)))
Z_aft = np.zeros((len(training_data)))
age_aft = np.zeros((len(training_data)))
ebv_aft = np.zeros((len(training_data)))
plt.subplots(figsize=(15, 7))
for j in range(0, len(training_data)):
# chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.ones((len(band))))
chi2, chi1 = cs.full_fit(
cs.load_table('chisquare_models_test.dat'),
cs.load_table('training_data.dat'),
j,
([
'uJava', 'uJava', 'uJava', 'uJava', 'F378', 'F378', 'F430',
'gSDSS', 'F660', 'F861', 'uJava'
]),
([
'F660', 'rSDSS', 'iSDSS', 'zSDSS', 'F515', 'gSDSS', 'gSDSS',
'zSDSS', 'gSDSS', 'uJava', 'F515'
]),
cs.load_table('best5_100age.dat')[-5, 0:11],
# fixar os pesos referentes as cores da idade antes e depois do treino para só ver a
# influência dos pesos das cores da metalicidade
# np.ones(11),
([
'iSDSS', 'rSDSS', 'uJava', 'F378', 'F410', 'F430', 'F515',
'F660', 'F660', 'gSDSS', 'iSDSS'
]),
([
'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'iSDSS', 'zSDSS',
'F861', 'iSDSS', 'zSDSS', 'F861'
]),
np.ones(11))
# cs.load_table('best5_100Z_test2.dat')[-5, 0:11])
Z_bef[j] = chi2[0] - aeg_models[j, 0]
age_bef[j] = (chi2[1] - aeg_models[j, 1]) / 10**9
ebv_bef[j] = chi2[2] - aeg_models[j, 2]
# chi = cs.fit(chisquare_models, training_data, j, band, band_ref, np.array([weights]))
chi2, chi1 = cs.full_fit(
cs.load_table('chisquare_models_test.dat'),
cs.load_table('training_data.dat'),
j,
([
'uJava', 'uJava', 'uJava', 'uJava', 'F378', 'F378', 'F430',
'gSDSS', 'F660', 'F861', 'uJava'
]),
([
'F660', 'rSDSS', 'iSDSS', 'zSDSS', 'F515', 'gSDSS', 'gSDSS',
'zSDSS', 'gSDSS', 'uJava', 'F515'
]),
cs.load_table('best5_100age.dat')[-5, 0:11],
# np.ones(11),
([
'iSDSS', 'rSDSS', 'uJava', 'F378', 'F410', 'F430', 'F515',
'F660', 'F660', 'gSDSS', 'iSDSS'
]),
([
'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'zSDSS', 'iSDSS', 'zSDSS',
'F861', 'iSDSS', 'zSDSS', 'F861'
]),
# np.ones(11))
cs.load_table('best5_100Z_test4.dat')[-5, 0:11])
Z_aft[j] = chi2[0] - aeg_models[j, 0]
age_aft[j] = (chi2[1] - aeg_models[j, 1]) / 10**9
ebv_aft[j] = chi2[2] - aeg_models[j, 2]
# N=20
plt.subplot(1, 3, 1)
plt.hist(Z_bef,
bins=np.arange(-0.055, 0.055, 0.01),
color='#aaaa22',
rwidth=0.95)
bins_Z_aft, Z_aft_data = histOutline(Z_aft,
bins=np.arange(-0.055, 0.055, 0.01))
plt.plot(bins_Z_aft, Z_aft_data, '-', color="#1B6AC6", lw=4)
# plt.hist(Z_aft, bins=np.arange(-0.055, 0.055, 0.01), color='#aaab22', rwidth=0.95)
plt.xlabel(r'$\Delta$Z' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(Z_bef), biweight_midvariance(Z_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(Z_aft), biweight_midvariance(Z_aft)))
plt.xlim(-0.05, 0.05)
plt.subplot(1, 3, 2)
plt.hist(age_bef,
bins=np.arange(-1.05, 1.05, 0.1),
color='#aaaa22',
rwidth=0.95)
bins_age_aft, age_aft_data = histOutline(age_aft,
bins=np.arange(-1.05, 1.05, 0.1))
plt.plot(bins_age_aft, age_aft_data, '-', color="#1B6AC6", lw=4)
# plt.hist(age_aft, bins=np.arange(-1.05, 1.05, 0.1), color='#aaaa22', rwidth=0.95)
plt.xlabel(r'$\Delta$age/$10^{9}$' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(age_bef), biweight_midvariance(age_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(age_aft), biweight_midvariance(age_aft)))
plt.xlim(-0.5, 0.5)
plt.subplot(1, 3, 3)
plt.hist(ebv_bef,
bins=np.arange(-0.45, 0.45, 0.05),
label='Before training',
color='#aaaa22',
rwidth=0.95)
bins_ebv_aft, ebv_aft_data = histOutline(ebv_aft,
bins=np.arange(-0.45, 0.45, 0.05))
plt.plot(bins_ebv_aft,
ebv_aft_data,
'-',
color="#1B6AC6",
lw=4,
label='After training')
# plt.hist(ebv_aft, bins=np.arange(-0.4, 0.4, 0.05), label='After training', color='#aaaa22', rwidth=0.95)
plt.xlabel('$\Delta$E(B-V)' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(ebv_bef), biweight_midvariance(ebv_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f}, \zeta={1:1.6f}$'.format(
np.mean(ebv_aft), biweight_midvariance(ebv_aft)))
# plt.xlim(-5,2)
plt.legend()
# plt.suptitle('Histograms')
plt.tight_layout()
# plt.savefig('histogram_Z&Age.png')
plt.show()
def biweight_midvariance(self):
"""
The biweight midvariance of the pixel values.
"""
return biweight_midvariance(self.goodvals)
def makeplot(month):
flux_aper, fwhm = [], []
if month == 'oct':
for num in range(len(octplot)):
thefile = octplot[num]
with open(thefile, 'r') as f:
lines = f.readlines()
for x in lines:
line = x.split(' ')
flux_aper.append((-2.5 * (math.log10(float(line[24])))))
fwhm.append(line[5])
plt.figure(num)
plt.plot(flux_aper, fwhm, 'ro')
plt.title(
'c%s Flux vs. FWHM' %
(num + 1)), plt.xlabel('-2.5*log_10(flux)'), plt.ylabel('FWHM')
axes = plt.gca()
axes.set_xlim([-16, -2]), axes.set_ylim([0, 20])
del flux_aper[:], fwhm[:]
for line in fileinput.input(thefile, inplace=1):
x = line.split(' ')
if (not line.startswith('N')) and (minlogflux < (-2.5 * (math.log10(float(x[24])))) < maxlogflux) and \
float(x[5]) < maxfwhm:
flux_aper.append((-2.5 * (math.log10(float(x[24])))))
fwhm.append(float(x[5]))
sys.stdout.write(line)
med = median(fwhm)
sig = st.biweight_midvariance(fwhm)
del flux_aper[:], fwhm[:]
for line in fileinput.input(thefile, inplace=1):
x = line.split(' ')
if (not line.startswith('N')) and (
(med - sigmamultiple[num] * sig) < float(x[5]) <
(med + sigmamultiple[num] * sig)):
flux_aper.append((-2.5 * (math.log10(float(x[24])))))
fwhm.append(x[5])
sys.stdout.write(line)
plt.plot(flux_aper, fwhm, 'bo')
del flux_aper[:], fwhm[:]
plt.savefig(('%s_flux vs. fwhm.png' % ('c' + str(num + 1))),
bbox_inches='tight')
if writeup == True:
plt.show()
plt.clf()
plt.close()
elif month == 'sept':
thefile = septclean[0]
with open(thefile, 'r') as f:
lines = f.readlines()
for x in lines:
line = x.split(' ')
flux_aper.append((-2.5 * (math.log10(float(line[26])))))
fwhm.append(line[7])
if float(line[7]) > 80:
print line[7]
print line[0]
plt.figure(1000)
plt.plot(flux_aper, fwhm, 'ro')
plt.title('Sept Flux vs. FWHM'), plt.xlabel(
'-2.5*log_10(flux)'), plt.ylabel('FWHM')
axes = plt.gca()
# axes.set_xlim([-15, -2]), axes.set_ylim([0, 20])
del flux_aper[:], fwhm[:]
for line in fileinput.input(thefile, inplace=1):
x = line.split(' ')
if (not line.startswith('N')) and (
-15 < (-2.5 * (math.log10(float(x[26])))) < -10):
flux_aper.append((-2.5 * (math.log10(float(x[26])))))
fwhm.append(float(x[7]))
sys.stdout.write(line)
med = median(fwhm)
sig = st.biweight_midvariance(fwhm)
del flux_aper[:], fwhm[:]
for line in fileinput.input(thefile, inplace=1):
x = line.split(' ')
if ((med - 1.5 * sig) < float(x[7]) < (med + 1.5 * sig)):
flux_aper.append((-2.5 * (math.log10(float(x[26])))))
fwhm.append(x[7])
sys.stdout.write(line)
plt.plot(flux_aper, fwhm, 'bo')
del flux_aper[:], fwhm[:]
plt.savefig((sept + 'Sept flux vs. fwhm.png'), bbox_inches='tight')
if writeup == True:
plt.show()
plt.clf()
plt.close()
def run_caustic(self, cluster_data):
S = CausticSurface(self.cosmo)
results = CausticFitResults()
results.cosmo = self.cosmo
clus_z, r200 = cluster_data.clus_z, cluster_data.r200
v, gal_memberflag, gal_mags = cluster_data.v, cluster_data.gal_memberflag, cluster_data.gal_mags
data_table = cluster_data.data_spec
if cluster_data.r is None:
# calculate angular diameter distance.
# Variable self.ang_d
ang_d = self.cosmo.angular_diameter_distance(clus_z)
# calculate the spherical angles of galaxies from cluster center.
# Variable self.angle
angle = self.findangle(data_table['RA'], data_table['DEC'],
cluster_data.clus_ra, cluster_data.clus_dec)
r = angle * ang_d
else:
r = cluster_data.r
# calculate H(z)
Hz = self.cosmo.H(clus_z).value # self.cosmo.h*100*np.sqrt(0.25*(1+self.clus_z)**3 + 0.75)
# hz = self.Hz/100 #self.Hz / 100.0 #little h(z)
# package galaxy data, USE ASTROPY TABLE HERE!!!!!
rcol = Table.Column(data=r, name='rad')
vcol = Table.Column(data=v, name='vel')
data_table.add_column(rcol, 0)
data_table.add_column(vcol, 1)
if gal_memberflag is not None:
memflagcol = Table.Column(data=gal_memberflag, name='MEMFLAG')
data_table.add_column(memflagcol)
if gal_mags is not None:
galmagcol = Table.Column(data=gal_mags, name='MAG')
data_table.add_column(galmagcol)
# reduce sample within limits
if self.cut_sample is True:
data_set = self.set_sample(data=data_table, rlimit=self.rlimit, vlimit=self.vlimit)
else:
data_set = data_table
data_set = data_set.to_pandas()
# further select sample via shifting gapper
if self.gapper is True:
data_set = self.shiftgapper(data_set)
print('DATA SET SIZE', len(data_set))
##tries to identify double groups that slip through the gapper process
# upper_max = np.max(data_set[:,1][np.where((data_set[:,1]>0.0)&(data_set[:,0]<1.0))])
# lower_max = np.min(data_set[:,1][np.where((data_set[:,1]<0.0)&(data_set[:,0]<1.0))])
# if np.max(np.array([upper_max,-lower_max])) > 1000.0+np.min(np.array([upper_max,-lower_max])):
# data_set = data_set[np.where(np.abs(data_set[:,1])<1000.0+np.min(np.array([upper_max,-lower_max])))]
# measure Ngal above mag limit
# question shouldnt this also be cut by the shiftgapper?
try:
if cluster_data.abs_flag:
abs_mag = data_table['MAG']
else:
abs_mag = data_table['MAG'] - self.cosmo.distmod(clus_z).value
results.Ngal_1mpc = np.sum((abs_mag < -19.55) & (r < 0.5) & (np.abs(v) < 3500))
except KeyError:
abs_mag = np.zeros(len(data_set))
results.Ngal_1mpc = None
if r200 is None:
vdisp_prelim = biweight_midvariance(data_set['vel'][(data_set['rad'] < 3.0)], 9.0)
if np.sum(abs_mag) == 0:
r200_mean_prelim = 0.002 * vdisp_prelim + 0.40
r200 = r200_mean_prelim / 1.7
else:
r200 = results.Ngal_1mpc ** 0.51 * np.exp(-1.86)
if r200 > 3.0:
r200 = 3.0
# if 3.0*r200 < 6.0:
# rlimit = 3.0*r200
# else:
# rlimit = 5.5
else:
r200 = r200
if r200 > 3.0:
r200 = 3.0
print('Pre_r200=', r200)
if self.mirror is True:
print('Calculating Density w/Mirrored Data')
rads = np.asarray(data_set['rad'])
vels = np.asarray(data_set['vel'])
self._gaussian_kernel(np.append(rads, rads),
np.append(vels, -1 * vels), r200,
normalization=Hz, scale=self.kernal_stretch,
rmax=self.rgridmax, vmax=self.vgridmax)
else:
print('Calculating Density')
self._gaussian_kernel(data_set['rad'], data_set['vel'], r200,
normalization=Hz, scale=self.kernal_stretch,
rmax=self.rgridmax, vmax=self.vgridmax)
img_tot = self.img / np.max(np.abs(self.img))
# img_grad_tot = self.img_grad/np.max(np.abs(self.img_grad))
# img_inf_tot = self.img_inf/np.max(np.abs(self.img_inf))
if cluster_data.clus_vdisp is None:
# pre_vdisp = 9.15*Ngal_1mpc+350.32
# print 'Pre_vdisp=',pre_vdisp
# print 'Ngal<1Mpc=',Ngal_1mpc
v_cut = data_set['vel'][((data_set['rad'] < r200) & (np.abs(data_set['vel']) < self.vlimit))]
try:
pre_vdisp2 = biweight_midvariance(v_cut[np.where(np.isfinite(v_cut))], 9.0)
except:
pre_vdisp2 = np.std(v_cut, ddof=1)
print('Vdisp from galaxies=', pre_vdisp2)
# if data_set[:,0].size < 15:
# v_unc = 0.35
# c_unc_sys = 0.75
# c_unc_int = 0.35
# elif data_set[:,0].size < 25 and data_set[:,0].size >= 15:
# v_unc = 0.30
# c_unc_sys = 0.55
# c_unc_int = 0.22
# elif data_set[:,0].size < 50 and data_set[:,0].size >= 25:
# v_unc = 0.23
# c_unc_sys = 0.42
# c_unc_int = 0.16
# elif data_set[:,0].size < 100 and data_set[:,0].size >= 50:
# v_unc = 0.18
# c_unc_sys = 0.34
# c_unc_int = 0.105
# else:
# v_unc = 0.15
# c_unc_sys = 0.29
# c_unc_int = 0.09
# if pre_vdisp2 > 1.75*pre_vdisp: pre_vdisp_comb = 9.15*Ngal_1mpc+450.32
# else:
pre_vdisp_comb = pre_vdisp2
# if data_set[:,1][np.where(data_set[:,0]<r200)].size >= 10:
# pre_vdisp_comb = biweight_midvariance(data_set[:,1][np.where(data_set[:,0]<r200)],9.0)
# else:
# pre_vdisp_comb = np.std(data_set[:,1][np.where(data_set[:,0]<r200)],ddof=1)
# #pre_vdisp_comb = (pre_vdisp*(pre_vdisp2*v_unc)**2+\
# pre_vdisp2*118.14**2)/(118.14**2+(pre_vdisp2*v_unc)**2)
else:
pre_vdisp_comb = cluster_data.clus_vdisp
print('Combined Vdisp=', pre_vdisp_comb)
# beta = 0.5*self.r_range/(self.r_range + r200/4.0)
# Identify initial caustic surface and members within the surface
print('Calculating initial surface')
if self.inflection is False:
if gal_memberflag is None:
S.findsurface(data_set, self.r_range, self.v_range, img_tot,
r200=r200, halo_vdisp=pre_vdisp_comb, beta=None,
mirror=self.mirror, edge_perc=self.edge_perc, Hz=Hz,
edge_int_remove=self.edge_int_remove, q=self.kernal_stretch, plotphase=False)
else:
S.findsurface(data_set, self.r_range, self.v_range, img_tot,
memberflags=data_set[:, -1], r200=r200,
mirror=self.mirror, edge_perc=self.edge_perc, Hz=Hz, q=self.kernal_stretch)
else:
if gal_memberflag is None:
S.findsurface_inf(data_set, self.r_range, self.v_range, img_tot,
self.img_inf, r200=r200, halo_vdisp=pre_vdisp_comb,
beta=None)
else:
S.findsurface_inf(data_set, self.r_range, self.v_range, img_tot,
self.img_inf, memberflags=data_set[:, -1], r200=r200)
results.clus_ra = cluster_data.clus_ra
results.clus_dec = cluster_data.clus_dec
results.clus_z = cluster_data.clus_z
results.caustic_profile = S.Ar_finalD
results.caustic_fit = S.vesc_fit
results.caustic_edge = np.abs(S.Ar_finalE)
results.caustic_fit_edge = S.vesc_fit_e
results.gal_vdisp = S.gal_vdisp
results.memflag = S.memflag
# Estimate the mass based off the caustic profile, beta profile (if given), and concentration (if given)
if clus_z is not None:
crit = self.cosmo.critical_density(clus_z).to(
astunits.solMass / astunits.Mpc ** 3) # 2.7745946e11*(self.cosmo.h)**2.0*(0.25*(1+clus_z)**3.0 + 0.75)
Mass = self.calculate_mass(ri=self.r_range, A=results.caustic_profile,
crit=crit, r200=r200, fbr=None)
Mass2 = self.calculate_mass(ri=self.r_range, A=results.caustic_profile,
crit=crit, r200=r200, fbr=self.fbr)
MassE = self.calculate_mass(ri=self.r_range, A=results.caustic_edge,
crit=crit, r200=r200, fbr=self.fbr)
MassF = self.calculate_mass(ri=self.r_range, A=results.caustic_fit,
crit=crit, r200=r200, fbr=self.fbr)
MassFE = self.calculate_mass(ri=self.r_range, A=results.caustic_fit_edge,
crit=crit, r200=r200, fbr=self.fbr)
results.crit = crit
results.mprof = Mass.massprofile
results.mprof_fbeta = Mass2.massprofile
results.mprof_edge = MassE.massprofile
results.r200_est = Mass.r200_est
results.r200_est_fbeta = Mass2.r200_est
results.r200_est_edge = MassE.r200_est
results.r500_est = Mass.r500_est
results.r500_est_fbeta = Mass2.r500_est
results.M200_est = Mass.M200_est
results.M200_est_fbeta = Mass2.M200_est
results.M200_fbeta = Mass2.M200
results.M200_edge = MassE.M200
results.M200_edge_est = MassE.M200_est
results.M200_fit = MassF.M200
results.M200_fit_est = MassF.M200_est
results.M200_fit_edge = MassFE.M200
results.M200_fit_edge_est = MassFE.M200_est
results.M500_est = Mass.M500_est
results.M500_est_fbeta = Mass2.M500_est
print('r200 estimate: {:.6f}'.format(float(Mass2.r200_est)))
print('M200 estimate: {:.6E}'.format(float(Mass2.M200_est)))
results.Ngal = np.sum((results.memflag == 1) & (data_set['rad'] <= results.r200_est_fbeta))
# calculate velocity dispersion
try:
results.vdisp_gal = biweight_midvariance(data_set['vel'][results.memflag == 1], 9.0)
except:
try:
results.vdisp_gal = np.std(data_set['vel'][results.memflag == 1], ddof=1)
except:
results.vdisp_gal = 0.0
return results
def delay_std(self, return_dict=False):
"""
Calculate standard deviations of per-antenna delay solutions
and aggregate delay solutions. Assign a z-score for
each (antenna, time) delay solution w.r.t. aggregate
mean and standard deviation.
Uses sqrt( astropy.stats.biweight_midvariance ) for a robust measure
of the std
Input:
------
return_dict : bool, [default=False]
if True, return time_std, ant_std and z_scores
as a dictionary with "ant" as keys
and "times" as keys
else, return as ndarrays
Output:
--------
return ant_avg, ant_std, time_std, agg_std, max_std, z_scores
ant_avg : ndarray, shape=(N_ants,)
average delay solution across time for each antenna
ant_std : ndarray, shape=(N_ants,)
standard deviation of delay solutions across time
for each antenna
time_std : ndarray, shape=(N_times,)
standard deviation of delay solutions across ants
for each time-stamp
agg_std : float
aggregate standard deviation of delay solutions
across all antennas and all times
max_std : float
maximum antenna standard deviation of delay solutions
z_scores : ndarray, shape=(N_ant, N_times)
z_scores (standard scores) for each (antenna, time)
delay solution w.r.t. agg_std
ant_z_scores : ndarray, shape=(N_ant,)
absolute_value(z_scores) for each antenna & time
then averaged over time
"""
# calculate standard deviations
ant_avg = self.delay_avgs
ant_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations, axis=1))
time_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations, axis=0))
agg_std = np.sqrt(astats.biweight_midvariance(self.delay_fluctuations))
max_std = np.max(ant_std)
# calculate z-scores
z_scores = self.delay_fluctuations / agg_std
ant_z_scores = np.median(np.abs(z_scores), axis=1)
# convert to ordered dict if desired
if return_dict is True:
ant_avg_d = OrderedDict()
time_std_d = OrderedDict()
ant_std_d = OrderedDict()
z_scores_d = OrderedDict()
ant_z_scores_d = OrderedDict()
for i, ant in enumerate(self.ants):
ant_avg_d[ant] = ant_avg[i]
ant_std_d[ant] = ant_std[i]
z_scores_d[ant] = z_scores[i]
ant_z_scores_d[ant] = ant_z_scores[i]
for i, t in enumerate(self.times):
time_std_d[t] = time_std[i]
ant_avg = ant_avg_d
time_std = time_std_d
ant_std = ant_std_d
z_scores = z_scores_d
ant_z_scores = ant_z_scores_d
return ant_avg, ant_std, time_std, agg_std, max_std, z_scores, ant_z_scores
def biweight_scale(data, c=9.0, M=None, axis=None, modify_sample_size=False,
*, ignore_nan=False):
r"""
Compute the biweight scale.
The biweight scale is a robust statistic for determining the
standard deviation of a distribution. It is the square root of the
`biweight midvariance
<https://en.wikipedia.org/wiki/Robust_measures_of_scale#The_biweight_midvariance>`_.
It is given by:
.. math::
\zeta_{biscl} = \sqrt{n} \ \frac{\sqrt{\sum_{|u_i| < 1} \
(x_i - M)^2 (1 - u_i^2)^4}} {|(\sum_{|u_i| < 1} \
(1 - u_i^2) (1 - 5u_i^2))|}
where :math:`x` is the input data, :math:`M` is the sample median
(or the input location) and :math:`u_i` is given by:
.. math::
u_{i} = \frac{(x_i - M)}{c * MAD}
where :math:`c` is the tuning constant and :math:`MAD` is the
`median absolute deviation
<https://en.wikipedia.org/wiki/Median_absolute_deviation>`_. The
biweight midvariance tuning constant ``c`` is typically 9.0 (the
default).
For the standard definition of biweight scale, :math:`n` is the
total number of points in the array (or along the input ``axis``, if
specified). That definition is used if ``modify_sample_size`` is
`False`, which is the default.
However, if ``modify_sample_size = True``, then :math:`n` is the
number of points for which :math:`|u_i| < 1` (i.e. the total number
of non-rejected values), i.e.
.. math::
n = \sum_{|u_i| < 1} \ 1
which results in a value closer to the true standard deviation for
small sample sizes or for a large number of rejected values.
Parameters
----------
data : array_like
Input array or object that can be converted to an array.
``data`` can be a `~numpy.ma.MaskedArray`.
c : float, optional
Tuning constant for the biweight estimator (default = 9.0).
M : float or array_like, optional
The location estimate. If ``M`` is a scalar value, then its
value will be used for the entire array (or along each ``axis``,
if specified). If ``M`` is an array, then its must be an array
containing the location estimate along each ``axis`` of the
input array. If `None` (default), then the median of the input
array will be used (or along each ``axis``, if specified).
axis : `None`, int, or tuple of ints, optional
The axis or axes along which the biweight scales are computed.
If `None` (default), then the biweight scale of the flattened
input array will be computed.
modify_sample_size : bool, optional
If `False` (default), then the sample size used is the total
number of elements in the array (or along the input ``axis``, if
specified), which follows the standard definition of biweight
scale. If `True`, then the sample size is reduced to correct
for any rejected values (i.e. the sample size used includes only
the non-rejected values), which results in a value closer to the
true standard deviation for small sample sizes or for a large
number of rejected values.
ignore_nan : bool, optional
Whether to ignore NaN values in the input ``data``.
Returns
-------
biweight_scale : float or `~numpy.ndarray`
The biweight scale of the input data. If ``axis`` is `None`
then a scalar will be returned, otherwise a `~numpy.ndarray`
will be returned.
See Also
--------
biweight_midvariance, biweight_midcovariance, biweight_location, astropy.stats.mad_std, astropy.stats.median_absolute_deviation
References
----------
.. [1] Beers, Flynn, and Gebhardt (1990; AJ 100, 32) (http://adsabs.harvard.edu/abs/1990AJ....100...32B)
.. [2] https://www.itl.nist.gov/div898/software/dataplot/refman2/auxillar/biwscale.htm
Examples
--------
Generate random variates from a Gaussian distribution and return the
biweight scale of the distribution:
>>> import numpy as np
>>> from astropy.stats import biweight_scale
>>> rand = np.random.RandomState(12345)
>>> biscl = biweight_scale(rand.randn(1000))
>>> print(biscl) # doctest: +FLOAT_CMP
0.986726249291
"""
return np.sqrt(
biweight_midvariance(data, c=c, M=M, axis=axis,
modify_sample_size=modify_sample_size,
ignore_nan=ignore_nan))
def make_mask(filename, ext, trail_coords, sublen=75, subwidth=200, order=3,
sigma=4, pad=10, plot=False, verbose=False):
"""Create DQ mask for an image for a given satellite trail.
This mask can be added to existing DQ data using :func:`update_dq`.
.. note::
Unlike :func:`detsat`, multiprocessing is not available for
this function.
Parameters
----------
filename : str
FITS image filename.
ext : int, str, or tuple
Extension for science data, as accepted by ``astropy.io.fits``.
trail_coords : ndarray
One of the trails returned by :func:`detsat`.
This must be in the format of ``[[x0, y0], [x1, y1]]``.
sublen : int, optional
Length of strip to use as the fitting window for the trail.
subwidth : int, optional
Width of box to fit trail on.
order : int, optional
The order of the spline interpolation for image rotation.
See :func:`skimage.transform.rotate`.
sigma : float, optional
Sigma of the satellite trail for detection. If points are
a given sigma above the background in the subregion then it is
marked as a satellite. This may need to be lowered for resolved
trails.
pad : int, optional
Amount of extra padding in pixels to give the satellite mask.
plot : bool, optional
Plot the result.
verbose : bool, optional
Print extra information to the terminal, mostly for debugging.
Returns
-------
mask : ndarray
Boolean array marking the satellite trail with `True`.
Raises
------
IndexError
Invalid subarray indices.
ValueError
Image has no positive values, trail subarray too small, or
trail profile not found.
"""
if verbose:
t_beg = time.time()
fname = '{0}[{1}]'.format(filename, ext)
image = fits.getdata(filename, ext)
dx = image.max()
if dx <= 0:
raise ValueError('Image has no positive values')
# rescale the image
image = image / dx
# make sure everything is at least 0
image[image < 0] = 0
(x0, y0), (x1, y1) = trail_coords # p0, p1
# Find out how much to rotate the image
rad = np.arctan2(y1 - y0, x1 - x0)
newrad = (np.pi * 2) - rad
deg = np.degrees(rad)
if verbose:
print('Rotation: {0}'.format(deg))
rotate = transform.rotate(image, deg, resize=True, order=order)
if plot and plt is not None:
plt.ion()
mean = np.median(image)
stddev = image.std()
lower = mean - stddev
upper = mean + stddev
fig1, ax1 = plt.subplots()
ax1.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax1.set_title(fname)
fig2, ax2 = plt.subplots()
ax2.imshow(rotate, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax2.set_title('{0} rotated by {1} deg'.format(fname, deg))
plt.draw()
# Will do all of this in the loop, but want to make sure there is a
# good point first and that there is indeed a profile to fit.
# get starting point
sx, sy = _rotate_point((x0, y0), newrad, image.shape, rotate.shape)
# start with one subarray around p0
dx = int(subwidth / 2)
ix0, ix1, iy0, iy1 = _get_valid_indices(
rotate.shape, sx - dx, sx + dx, sy - sublen, sy + sublen)
subr = rotate[iy0:iy1, ix0:ix1]
if len(subr) <= sublen:
raise ValueError('Trail subarray size is {0} but expected {1} or '
'larger'.format(len(subr), sublen))
# Flatten the array so we are looking along rows
# Take median of each row, should filter out most outliers
# This list will get appended in the loop
medarr = np.median(subr, axis=1)
flat = [medarr]
# get the outliers
#mean = biweight_location(medarr)
mean = sigma_clipped_stats(medarr)[0]
stddev = biweight_midvariance(medarr)
# only flag things that are sigma from the mean
z = np.where(medarr > (mean + (sigma * stddev)))[0]
if plot and plt is not None:
fig1, ax1 = plt.subplots()
ax1.plot(medarr, 'b.')
ax1.plot(z, medarr[z], 'r.')
ax1.set_xlabel('Index')
ax1.set_ylabel('Value')
ax1.set_title('Median array in flat[0]')
plt.draw()
# Make sure there is something in the first pass before trying to move on
if len(z) < 1:
raise ValueError(
'First look at finding a profile failed. '
'Nothing found at {0} from background! '
'Adjust parameters and try again.'.format(sigma))
# get the bounds of the flagged points
lower = z.min()
upper = z.max()
diff = upper - lower
# add in a pading value to make sure all of the wings are accounted for
lower = lower - pad
upper = upper + pad
# for plotting see how the profile was made (append to plot above)
if plot and plt is not None:
padind = np.arange(lower, upper)
ax1.plot(padind, medarr[padind], 'yx')
plt.draw()
# start to create a mask
mask = np.zeros(rotate.shape)
lowerx, upperx, lowery, uppery = _get_valid_indices(
mask.shape, np.floor(sx - subwidth), np.ceil(sx + subwidth),
np.floor(sy - sublen + lower), np.ceil(sy - sublen + upper))
mask[lowery:uppery, lowerx:upperx] = 1
done = False
first = True
nextx = upperx # np.ceil(sx + subwidth)
centery = np.ceil(lowery + diff) # np.ceil(sy - sublen + lower + diff)
counter = 0
while not done:
# move to the right of the centerpoint first. do the same
# as above but keep moving right until the edge is hit.
ix0, ix1, iy0, iy1 = _get_valid_indices(
rotate.shape, nextx - dx, nextx + dx,
centery - sublen, centery + sublen)
subr = rotate[iy0:iy1, ix0:ix1]
# determines the edge, if the subr is not good, then the edge was
# hit.
if 0 in subr.shape:
if verbose:
print('Hit edge, subr shape={0}, first={1}'.format(
subr.shape, first))
if first:
first = False
centery = sy
nextx = sx
else:
done = True
continue
medarr = np.median(subr, axis=1)
flat.append(medarr)
#mean = biweight_location(medarr)
mean = sigma_clipped_stats(medarr, sigma=sigma)[0]
stddev = biweight_midvariance(medarr) # Might give RuntimeWarning
z = np.where(medarr > (mean + (sigma * stddev)))[0]
if len(z) < 1:
if first:
if verbose:
print('No good profile found for counter={0}. Start '
'moving left from starting point.'.format(counter))
centery = sy
nextx = sx
first = False
else:
if verbose:
print('z={0} is less than 1, subr shape={1}, '
'we are done'.format(z, subr.shape))
done = True
continue
# get the bounds of the flagged points
lower = z.min()
upper = z.max()
diff = upper - lower
# add in a pading value to make sure all of the wings
# are accounted for
lower = np.floor(lower - pad)
upper = np.ceil(upper + pad)
lowerx, upperx, lowery, uppery = _get_valid_indices(
mask.shape,
np.floor(nextx - subwidth),
np.ceil(nextx + subwidth),
np.floor(centery - sublen + lower),
np.ceil(centery - sublen + upper))
mask[lowery:uppery, lowerx:upperx] = 1
lower_p = (lowerx, lowery)
upper_p = (upperx, uppery)
lower_t = _rotate_point(
lower_p, newrad, image.shape, rotate.shape, reverse=True)
upper_t = _rotate_point(
upper_p, newrad, image.shape, rotate.shape, reverse=True)
lowy = np.floor(lower_t[1])
highy = np.ceil(upper_t[1])
lowx = np.floor(lower_t[0])
highx = np.ceil(upper_t[0])
# Reset the next subr to be at the center of the profile
if first:
nextx = nextx + dx
centery = lowery + diff # centery - sublen + lower + diff
if (nextx + subwidth) > rotate.shape[1]:
if verbose:
print('Hit rotate edge at counter={0}'.format(counter))
first = False
elif (highy > image.shape[0]) or (highx > image.shape[1]):
if verbose:
print('Hit image edge at counter={0}'.format(counter))
first = False
if not first:
centery = sy
nextx = sx
# Not first, this is the pass the other way.
else:
nextx = nextx - dx
centery = lowery + diff # centery - sublen + lower + diff
if (nextx - subwidth) < 0:
if verbose:
print('Hit rotate edge at counter={0}'.format(counter))
done = True
elif (highy > image.shape[0]) or (highx > image.shape[1]):
if verbose:
print('Hit image edge at counter={0}'.format(counter))
done = True
counter += 1
# make sure it does not try to go infinetly
if counter > 500:
if verbose:
print('Too many loops, exiting')
done = True
# End while
rot = transform.rotate(mask, -deg, resize=True, order=1)
ix0 = (rot.shape[1] - image.shape[1]) / 2
iy0 = (rot.shape[0] - image.shape[0]) / 2
lowerx, upperx, lowery, uppery = _get_valid_indices(
rot.shape, ix0, image.shape[1] + ix0, iy0, image.shape[0] + iy0)
mask = rot[lowery:uppery, lowerx:upperx]
if mask.shape != image.shape:
warnings.warn('Output mask shape is {0} but input image shape is '
'{1}'.format(mask.shape, image.shape), AstropyUserWarning)
# Change to boolean mask
mask = mask.astype(np.bool)
if plot and plt is not None:
# debugging array
test = image.copy()
test[mask] = 0
mean = np.median(test)
stddev = test.std()
lower = mean - stddev
upper = mean + stddev
fig1, ax1 = plt.subplots()
ax1.imshow(test, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax1.set_title('Masked image')
fig2, ax2 = plt.subplots()
ax2.imshow(mask, cmap=plt.cm.gray)
ax2.set_title('DQ mask')
plt.draw()
if verbose:
t_end = time.time()
print('Run time: {0} s'.format(t_end - t_beg))
return mask
def biweight_midvariance(self):
"""
The biweight midvariance of the pixel values.
"""
return biweight_midvariance(self.goodvals)
def calc_background_rms(self, data):
return biweight_midvariance(self.sigma_clip(data), c=self.c, M=self.M)
def histogram(band, band_ref, best_weights, training_data, aeg_models,
chisquare_models):
weights = best_weights[-5, 0:len(band)]
print(weights)
Z_bef = np.zeros((len(training_data)))
age_bef = np.zeros((len(training_data)))
ebv_bef = np.zeros((len(training_data)))
Z_aft = np.zeros((len(training_data)))
age_aft = np.zeros((len(training_data)))
ebv_aft = np.zeros((len(training_data)))
plt.subplots(figsize=(15, 7))
for j in range(0, len(training_data)):
chi = cs.fit(chisquare_models, training_data, j, band, band_ref,
np.ones((len(band))))
Z_bef[j] = chi[0] - aeg_models[j, 0]
age_bef[j] = (chi[1] - aeg_models[j, 1]) / 10**9
ebv_bef[j] = chi[2] - aeg_models[j, 2]
for j in range(0, len(training_data)):
chi = cs.fit(chisquare_models, training_data, j, band, band_ref,
np.array([weights]))
Z_aft[j] = chi[0] - aeg_models[j, 0]
age_aft[j] = (chi[1] - aeg_models[j, 1]) / 10**9
ebv_aft[j] = chi[2] - aeg_models[j, 2]
plt.subplot(1, 3, 1)
plt.hist(Z_bef,
bins=np.arange(-0.055, 0.055, 0.01),
color='#aaaa22',
rwidth=0.95)
bins_Z_aft, Z_aft_data = histOutline(Z_aft,
bins=np.arange(-0.055, 0.055, 0.01))
plt.plot(bins_Z_aft, Z_aft_data, '-', color="#1B6AC6", lw=4)
plt.xlabel(r'$\Delta$Z' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(Z_bef), biweight_midvariance(Z_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(Z_aft), biweight_midvariance(Z_aft)))
plt.xlim(-0.05, 0.05)
plt.subplot(1, 3, 2)
plt.hist(age_bef,
bins=np.arange(-1.05, 1.05, 0.1),
color='#aaaa22',
rwidth=0.95)
bins_age_aft, age_aft_data = histOutline(age_aft,
bins=np.arange(-1.05, 1.05, 0.1))
plt.plot(bins_age_aft, age_aft_data, '-', color="#1B6AC6", lw=4)
plt.xlabel(r'$\Delta$age/$10^{9}$' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(age_bef), biweight_midvariance(age_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(age_aft), biweight_midvariance(age_aft)))
plt.xlim(-0.5, 0.5)
plt.subplot(1, 3, 3)
plt.hist(ebv_bef,
bins=np.arange(-0.45, 0.45, 0.05),
label='Before training',
color='#aaaa22',
rwidth=0.95)
bins_ebv_aft, ebv_aft_data = histOutline(ebv_aft,
bins=np.arange(-0.45, 0.45, 0.05))
plt.plot(bins_ebv_aft,
ebv_aft_data,
'-',
color="#1B6AC6",
lw=4,
label='After training')
plt.xlabel('$\Delta$E(B-V)' + '\n' + 'Before training: ' +
r'$\mu={0:1.6f},\ \zeta={1:1.6f}$'.format(
np.mean(ebv_bef), biweight_midvariance(ebv_bef)) + '\n' +
'After training: ' + r'$\mu={0:1.6f}, \zeta={1:1.6f}$'.format(
np.mean(ebv_aft), biweight_midvariance(ebv_aft)))
# plt.xlim(-5,2)
plt.legend()
# plt.suptitle('Histograms')
plt.tight_layout()
# plt.savefig('histogram_Z&Age.png')
plt.show()
def __init__(self, calfits_files, use_gp=True):
"""
Input:
------
calfits_files : str or list
filename for a *.first.calfits file
or a list of .first.calfits files (time-ordered)
of the same polarization
use_gp : bool, default=True
use gaussian process model to
subtract underlying smooth delay solution
behavior over time from fluctuations
Result:
-------
self.UVC : pyuvdata.UVCal() instance
self.delays : ndarray, shape=(N_ant, N_times)
firstcal delay solutions in nanoseconds
self.delay_avgs : ndarray, shape=(N_ant,)
median delay solutions across time [nanosec]
self.delay_fluctuations : ndarray, shape=(N_ant, N_times)
firstcal delay solution fluctuations from time average [nanosec]
self.frac_JD : ndarray, shape=(N_times,)
ndarray containing time-stamps of each integration
in units of the fraction of current JD
i.e. 2457966.53433 -> 0.53433
self.ants : ndarray, shape=(N_ants,)
ndarray containing antenna numbers
self.pol : str
Polarization, 'y' or 'x' currently supported
"""
# Instantiate UVCal and read calfits
self.UVC = UVCal()
self.UVC.read_calfits(calfits_files)
if len(self.UVC.jones_array) > 1:
raise ValueError('Sorry, only single pol firstcal solutions are '
'currently supported.')
pol_dict = {-5: 'x', -6: 'y'}
try:
self.pol = pol_dict[self.UVC.jones_array[0]]
except KeyError:
raise ValueError('Sorry, only calibration polarizations "x" and '
'"y" are currently supported.')
# get file prefix
if type(calfits_files) is list:
calfits_file = calfits_files[0]
else:
calfits_file = calfits_files
self.fc_basename = os.path.basename(calfits_file)
self.fc_filename = calfits_file
self.fc_filestem = '.'.join(self.fc_filename.split('.')[:-1])
# get other relevant arrays
self.times = self.UVC.time_array
self.start_JD = np.floor(self.times).min()
self.frac_JD = self.times - self.start_JD
self.minutes = 24 * 60 * (self.frac_JD - self.frac_JD.min())
self.Nants = self.UVC.Nants_data
self.ants = self.UVC.ant_array
self.version_str = hera_qm_version_str
self.history = ''
# Get the firstcal delays and/or gains and/or rotated antennas
if self.UVC.cal_type == 'gain':
# get delays
freqs = self.UVC.freq_array.squeeze()
fc_gains = np.moveaxis(self.UVC.gain_array, 2, 3)[:, 0, :, :, 0]
fc_phi = np.unwrap(np.angle(fc_gains))
d_nu = np.median(np.diff(freqs))
d_phi = np.median(fc_phi[:, :, 1:] - fc_phi[:, :, :-1], axis=2)
gain_slope = (d_phi / d_nu)
self.delays = gain_slope / (-2*np.pi)
self.gains = fc_gains
# get delay offsets at nu = 0 Hz, and then get rotated antennas
self.offsets = fc_phi[:, :, 0] - gain_slope * freqs[0]
self.rot_ants = np.unique(list(map(lambda x: self.ants[x], (np.isclose(np.pi, np.abs(self.offsets) % (2 * np.pi), atol=1.0)).T))).tolist()
elif self.UVC.cal_type == 'delay':
self.delays = self.UVC.delay_array.squeeze()
self.gains = None
self.offsets = None
self.rot_ants = []
# Calculate avg delay solution and subtract to get delay_fluctuations
self.delays = self.delays * 1e9
self.delay_avgs = np.median(self.delays, axis=1)
self.delay_fluctuations = (self.delays.T - self.delay_avgs).T
# use gaussian process model to subtract underlying mean function
if use_gp is True and self.sklearn_import is True:
# initialize GP kernel.
# RBF is a squared exponential kernel with a minimum length_scale_bound of 0.01 JD, meaning
# the GP solution won't have time fluctuations quicker than ~0.01 JD, which will preserve
# short time fluctuations. WhiteKernel is a Gaussian white noise component with a fiducial
# noise level of 0.01 nanoseconds. Both of these are hyperparameters that are fit for via
# a gradient descent algorithm in the GP.fit() routine, so length_scale=0.2 and
# noise_level=0.01 are just initial conditions and are not the final hyperparameter solution
kernel = gp.kernels.RBF(length_scale=0.2, length_scale_bounds=(0.01, 1.0)) + gp.kernels.WhiteKernel(noise_level=0.01)
x = self.frac_JD.reshape(-1, 1)
self.delay_smooths = []
# iterate over each antenna
for i in range(self.Nants):
# get ydata
y = copy.copy(self.delay_fluctuations[i])
# scale by std
ystd = np.sqrt(astats.biweight_midvariance(y))
y /= ystd
# fit GP and remove from delay fluctuations
GP = gp.GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=0)
GP.fit(x, y)
ymodel = GP.predict(x).ravel() * ystd
self.delay_fluctuations[i] -= ymodel
self.delay_smooths.append(ymodel)
self.delay_smooths = np.array(self.delay_smooths)
def make_mask(filename,
ext,
trail_coords,
sublen=75,
subwidth=200,
order=3,
sigma=4,
pad=10,
plot=False,
verbose=False):
"""Create DQ mask for an image for a given satellite trail.
This mask can be added to existing DQ data using :func:`update_dq`.
.. note::
Unlike :func:`detsat`, multiprocessing is not available for
this function.
Parameters
----------
filename : str
FITS image filename.
ext : int, str, or tuple
Extension for science data, as accepted by ``astropy.io.fits``.
trail_coords : ndarray
One of the trails returned by :func:`detsat`.
This must be in the format of ``[[x0, y0], [x1, y1]]``.
sublen : int, optional
Length of strip to use as the fitting window for the trail.
subwidth : int, optional
Width of box to fit trail on.
order : int, optional
The order of the spline interpolation for image rotation.
See :func:`skimage.transform.rotate`.
sigma : float, optional
Sigma of the satellite trail for detection. If points are
a given sigma above the background in the subregion then it is
marked as a satellite. This may need to be lowered for resolved
trails.
pad : int, optional
Amount of extra padding in pixels to give the satellite mask.
plot : bool, optional
Plot the result.
verbose : bool, optional
Print extra information to the terminal, mostly for debugging.
Returns
-------
mask : ndarray
Boolean array marking the satellite trail with `True`.
Raises
------
ImportError
Missing scipy or skimage>=0.11 packages.
IndexError
Invalid subarray indices.
ValueError
Image has no positive values, trail subarray too small, or
trail profile not found.
"""
if not HAS_OPDEP:
raise ImportError('Missing scipy or skimage>=0.11 packages')
if verbose:
t_beg = time.time()
fname = '{0}[{1}]'.format(filename, ext)
image = fits.getdata(filename, ext)
dx = image.max()
if dx <= 0:
raise ValueError('Image has no positive values')
# rescale the image
image = image / dx
# make sure everything is at least 0
image[image < 0] = 0
(x0, y0), (x1, y1) = trail_coords # p0, p1
# Find out how much to rotate the image
rad = np.arctan2(y1 - y0, x1 - x0)
newrad = (np.pi * 2) - rad
deg = np.degrees(rad)
if verbose:
print('Rotation: {0}'.format(deg))
rotate = transform.rotate(image, deg, resize=True, order=order)
if plot and plt is not None:
plt.ion()
mean = np.median(image)
stddev = image.std()
lower = mean - stddev
upper = mean + stddev
fig1, ax1 = plt.subplots()
ax1.imshow(image, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax1.set_title(fname)
fig2, ax2 = plt.subplots()
ax2.imshow(rotate, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax2.set_title('{0} rotated by {1} deg'.format(fname, deg))
plt.draw()
# Will do all of this in the loop, but want to make sure there is a
# good point first and that there is indeed a profile to fit.
# get starting point
sx, sy = _rotate_point((x0, y0), newrad, image.shape, rotate.shape)
# start with one subarray around p0
dx = int(subwidth / 2)
ix0, ix1, iy0, iy1 = _get_valid_indices(rotate.shape, sx - dx, sx + dx,
sy - sublen, sy + sublen)
subr = rotate[iy0:iy1, ix0:ix1]
if len(subr) <= sublen:
raise ValueError('Trail subarray size is {0} but expected {1} or '
'larger'.format(len(subr), sublen))
# Flatten the array so we are looking along rows
# Take median of each row, should filter out most outliers
# This list will get appended in the loop
medarr = np.median(subr, axis=1)
flat = [medarr]
# get the outliers
#mean = biweight_location(medarr)
mean = sigma_clipped_stats(medarr)[0]
stddev = biweight_midvariance(medarr)
# only flag things that are sigma from the mean
z = np.where(medarr > (mean + (sigma * stddev)))[0]
if plot and plt is not None:
fig1, ax1 = plt.subplots()
ax1.plot(medarr, 'b.')
ax1.plot(z, medarr[z], 'r.')
ax1.set_xlabel('Index')
ax1.set_ylabel('Value')
ax1.set_title('Median array in flat[0]')
plt.draw()
# Make sure there is something in the first pass before trying to move on
if len(z) < 1:
raise ValueError('First look at finding a profile failed. '
'Nothing found at {0} from background! '
'Adjust parameters and try again.'.format(sigma))
# get the bounds of the flagged points
lower = z.min()
upper = z.max()
diff = upper - lower
# add in a pading value to make sure all of the wings are accounted for
lower = lower - pad
upper = upper + pad
# for plotting see how the profile was made (append to plot above)
if plot and plt is not None:
padind = np.arange(lower, upper)
ax1.plot(padind, medarr[padind], 'yx')
plt.draw()
# start to create a mask
mask = np.zeros(rotate.shape)
lowerx, upperx, lowery, uppery = _get_valid_indices(
mask.shape, np.floor(sx - subwidth), np.ceil(sx + subwidth),
np.floor(sy - sublen + lower), np.ceil(sy - sublen + upper))
mask[lowery:uppery, lowerx:upperx] = 1
done = False
first = True
nextx = upperx # np.ceil(sx + subwidth)
centery = np.ceil(lowery + diff) # np.ceil(sy - sublen + lower + diff)
counter = 0
while not done:
# move to the right of the centerpoint first. do the same
# as above but keep moving right until the edge is hit.
ix0, ix1, iy0, iy1 = _get_valid_indices(rotate.shape, nextx - dx,
nextx + dx, centery - sublen,
centery + sublen)
subr = rotate[iy0:iy1, ix0:ix1]
# determines the edge, if the subr is not good, then the edge was
# hit.
if 0 in subr.shape:
if verbose:
print('Hit edge, subr shape={0}, first={1}'.format(
subr.shape, first))
if first:
first = False
centery = sy
nextx = sx
else:
done = True
continue
medarr = np.median(subr, axis=1)
flat.append(medarr)
#mean = biweight_location(medarr)
mean = sigma_clipped_stats(medarr, sigma=sigma)[0]
stddev = biweight_midvariance(medarr) # Might give RuntimeWarning
z = np.where(medarr > (mean + (sigma * stddev)))[0]
if len(z) < 1:
if first:
if verbose:
print('No good profile found for counter={0}. Start '
'moving left from starting point.'.format(counter))
centery = sy
nextx = sx
first = False
else:
if verbose:
print('z={0} is less than 1, subr shape={1}, '
'we are done'.format(z, subr.shape))
done = True
continue
# get the bounds of the flagged points
lower = z.min()
upper = z.max()
diff = upper - lower
# add in a pading value to make sure all of the wings
# are accounted for
lower = np.floor(lower - pad)
upper = np.ceil(upper + pad)
lowerx, upperx, lowery, uppery = _get_valid_indices(
mask.shape, np.floor(nextx - subwidth), np.ceil(nextx + subwidth),
np.floor(centery - sublen + lower),
np.ceil(centery - sublen + upper))
mask[lowery:uppery, lowerx:upperx] = 1
lower_p = (lowerx, lowery)
upper_p = (upperx, uppery)
lower_t = _rotate_point(lower_p,
newrad,
image.shape,
rotate.shape,
reverse=True)
upper_t = _rotate_point(upper_p,
newrad,
image.shape,
rotate.shape,
reverse=True)
lowy = np.floor(lower_t[1])
highy = np.ceil(upper_t[1])
lowx = np.floor(lower_t[0])
highx = np.ceil(upper_t[0])
# Reset the next subr to be at the center of the profile
if first:
nextx = nextx + dx
centery = lowery + diff # centery - sublen + lower + diff
if (nextx + subwidth) > rotate.shape[1]:
if verbose:
print('Hit rotate edge at counter={0}'.format(counter))
first = False
elif (highy > image.shape[0]) or (highx > image.shape[1]):
if verbose:
print('Hit image edge at counter={0}'.format(counter))
first = False
if not first:
centery = sy
nextx = sx
# Not first, this is the pass the other way.
else:
nextx = nextx - dx
centery = lowery + diff # centery - sublen + lower + diff
if (nextx - subwidth) < 0:
if verbose:
print('Hit rotate edge at counter={0}'.format(counter))
done = True
elif (highy > image.shape[0]) or (highx > image.shape[1]):
if verbose:
print('Hit image edge at counter={0}'.format(counter))
done = True
counter += 1
# make sure it does not try to go infinetly
if counter > 500:
if verbose:
print('Too many loops, exiting')
done = True
# End while
rot = transform.rotate(mask, -deg, resize=True, order=1)
ix0 = (rot.shape[1] - image.shape[1]) / 2
iy0 = (rot.shape[0] - image.shape[0]) / 2
lowerx, upperx, lowery, uppery = _get_valid_indices(
rot.shape, ix0, image.shape[1] + ix0, iy0, image.shape[0] + iy0)
mask = rot[lowery:uppery, lowerx:upperx]
if mask.shape != image.shape:
warnings.warn(
'Output mask shape is {0} but input image shape is '
'{1}'.format(mask.shape, image.shape), AstropyUserWarning)
# Change to boolean mask
mask = mask.astype(np.bool)
if plot and plt is not None:
# debugging array
test = image.copy()
test[mask] = 0
mean = np.median(test)
stddev = test.std()
lower = mean - stddev
upper = mean + stddev
fig1, ax1 = plt.subplots()
ax1.imshow(test, vmin=lower, vmax=upper, cmap=plt.cm.gray)
ax1.set_title('Masked image')
fig2, ax2 = plt.subplots()
ax2.imshow(mask, cmap=plt.cm.gray)
ax2.set_title('DQ mask')
plt.draw()
if verbose:
t_end = time.time()
print('Run time: {0} s'.format(t_end - t_beg))
return mask
def fbistd(y):
"""Helper function of trendplot. Returns biweight std of input 1d-array (see astropy.stats)"""
return biweight_midvariance(y)
mr1 = (fluxdata[allgood]['ar'] < 0.25)
mr2 = np.logical_and(fluxdata[allgood]['ar'] >= 0.25,
fluxdata[allgood]['ar'] < 0.5)
mr3 = np.logical_and(fluxdata[allgood]['ar'] >= 0.5,
fluxdata[allgood]['ar'] < 0.75)
mr4 = (fluxdata[allgood]['ar'] >= 0.75)
med1 = np.median(fluxdata[allgood][mr1].av)
med2 = np.median(fluxdata[allgood][mr2].av)
med3 = np.median(fluxdata[allgood][mr3].av)
med4 = np.median(fluxdata[allgood][mr4].av)
med751 = np.percentile(fluxdata[allgood][mr1].av, 75)
med752 = np.percentile(fluxdata[allgood][mr2].av, 75)
med753 = np.percentile(fluxdata[allgood][mr3].av, 75)
med754 = np.percentile(fluxdata[allgood][mr4].av, 75)
emed1 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr1].av))
emed2 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr2].av))
emed3 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr3].av))
emed4 = np.sqrt(biweight_midvariance(fluxdata[allgood][mr4].av))
ms = 12
lwbw = 2
for ax in axarr:
ax2 = axarr2[c]
ax3 = axarr3[c]
ax4 = axarr4[c]
ax5 = axarr5[c]
if c == 0:
col = 'good'
filt = allgood
color = 'blue'
def trendplot(
xin,
yin,
crange=None,
nbin=None,
xycut=None,
prange=None,
ystat=None,
estat=None,
psize=None,
pcolor=None,
cmarker=None,
ccolor=None,
csize=None,
cls=None,
clw=None,
sigfact=None,
sigcolor=None,
sigls=None,
noscatter=False,
noplot=False,
):
"""
XY scatter plot with tendency curves and optional errorbars + sigma curves.
Parameters
----------
**xin,yin** : 1d-arrays.
Input data. Required.
**crange** : list of form [xlim1,xlim2].
x-axis range that the tendency curve will span. Defaults to data [min,max].
**nbin** : int.
Nr of bins of tendency curve.
**xycut** : list of form [xlim1,xlim2,ylim1,ylim2].
Rectangular window to cut input data by. Defaults to data [min,max].
**prange** : list of form [xlim1,xlim2,ylim1,ylim2].
Rectangular window for plotting. Defaults to data [min,max].
**ystat** : y-axis statistic. Default: "median".
* "median" : median of data in bin.
* "mean" : mean of data in bin.
* "bimean" : biweight mean of data in bin (from astropy.stats).
**estat** : y-axis error bar statistic.
* "std" : standard deviation.
* "bistd" : biweight standard deviation(from astropy.stats).
* "poisson" : poisson error.
* "boot" : boostrap sampling error, i.e. the error of each bin is bistd(bimean(bootsamples)). Defaults to 250 samples.
**psize** : point size of scatter plot. Default: 1
**pcolor** : point color of scatter plot. Default: "0.5" (gray)
**cmarker** : marker of tendency curve. Default: "o"
**ccolor** : color of tendency curve. Default: "red"
**csize** : size of tendency curve markers. Default: 4
**cls** : linestyle of tendency curve. Default: "-"
**clw** : line width of tendency curve. Default: 1
**sigfact** : float. Plot 1,2,3 or x sigma lines above and below tendency curve. Uses bistd().
**sigcolor** : color of sigma lines. Default: "k"
**sigls** : linestyle of sigma lines. Default: "--"
**noscatter** : bool. Do not plot scatter points. Default: False
**noplot** : bool. Do not plot anything and return curves instead. Default: False
Returns
-------
(only when noplot=True)
* **(px,py)** : coordinates of curve, if estat is not set
* **(px,py,ey)**: coordinates of curve + errors, if estat is set
Examples
--------
::
import trend as t
x = np.arange(0,100,0.1)
y = 1.2*x + np.random.randn(len(x))*15
y[300:600] = 55. + np.random.randn(300)*6
# Simple plot
t.trendplot(x,y)
# More bins, 1-sigma error lines
t.trendplot(x,y, nbin=20, sigfact=1, estat='std')
# Custom curve range, custom ystat, boostrap error bars
t.trendplot(x,y,crange=[30,80],ystat='mean',estat='boot',ccolor='blue')
# Don't plot but get the biweight mean tendency curve with boostrap errors
px,py,ey = t.trendplot(x,y,ystat='bimean',estat='boot',noplot=True)
"""
# Default plotting styles
if psize is None:
psize = 1 # scatter point size
if pcolor is None:
pcolor = "0.5" # scatter point color
if cmarker is None:
cmarker = "o" # t. curve point style
if csize is None:
csize = 4 # t. curve point size
if ccolor is None:
ccolor = "red" # t. curve color
if cls is None:
cls = "-" # t. curve line style
if clw is None:
clw = 1 # t. curve line width
if sigcolor is None:
sigcolor = "k" # sigma line color
if sigls is None:
sigls = "--" # sigma line style
# Default number of bins
if nbin is None:
nbin = 10
# Default y-axis statistic
if ystat is None:
ystat = "median"
# Default xrange that the curve will spawn
if crange is None:
crange = [np.min(xin), np.max(xin)]
# Cut input values. Add some tolerance
if xycut is None:
tol = 0.01
xycut = [0.0, 0.0, 0.0, 0.0]
xycut[0] = np.min(xin) - tol
xycut[1] = np.max(xin) + tol
xycut[2] = np.min(yin) - tol
xycut[3] = np.max(yin) + tol
idxin = np.where((xin > xycut[0]) & (xin < xycut[1]) & (yin > xycut[2]) & (yin < xycut[3]))
xin, yin = xin[idxin], yin[idxin]
# Default plot range for entire plot. Do this after cutting input?
if prange is None:
prange = [np.min(xin), np.max(xin), np.min(yin), np.max(yin)]
# Bin x-coordinates
xcounts, xedges = np.histogram(xin, bins=nbin, range=crange)
# Find curve x-axis (mid-point of bins)
hbsiz = abs(xedges[0] - xedges[1]) * 0.5
px = xedges[:-1] + hbsiz # drop last
# Find curve y-axis (mean, median, etc.)
py = np.zeros(nbin)
ey = np.zeros(nbin)
sigmav = np.zeros(nbin)
ystatoptions = {"median": fmed, "mean": fmean, "bimean": fbimean}
estatoptions = {"std": fstd, "poisson": fpoisson, "boot": fboot, "bistd": fbistd}
for l, r, i in zip(xedges[:-1], xedges[1:], range(nbin)):
ydat = yin[(xin > l) & (xin < r)]
py[i] = ystatoptions[ystat](ydat)
if estat is not None:
ey[i] = estatoptions[estat](ydat)
if sigfact is not None:
sigmav[i] = biweight_midvariance(ydat)
# print(i,'--',l,'--',r,'--',elm,'--',py[i])
# Do plot or get results =================================================
if noplot == False:
if noscatter == False:
plt.scatter(xin, yin, marker=".", s=psize, color=pcolor) # scatter points
print cls
plt.plot(px, py, color=ccolor, linestyle=cls, linewidth=clw) # curve line
plt.plot(
px, py, marker=cmarker, markersize=csize, color=ccolor, markeredgecolor=ccolor, linestyle="none"
) # curve points
if estat is not None:
plt.errorbar(px, py, yerr=ey, fmt="none", ecolor=ccolor) # curve errorbars
if sigfact is not None:
plt.plot(px, py + sigfact * sigmav, color=sigcolor, linestyle=sigls) # sigma lines
plt.plot(px, py - sigfact * sigmav, color=sigcolor, linestyle=sigls)
# Reset plot limits to desired range
plt.xlim(prange[0], prange[1])
plt.ylim(prange[2], prange[3])
else:
if estat is not None:
res = (px, py, ey)
else:
res = (px, py)
return res
def _gaussian_kernel(self, rvalues, vvalues, r200, normalization=100.0, scale=10.0, rres=200, vres=220, rmax=6.0,
vmax=5000.0):
"""
Uses a 2D gaussian kernel to estimate the density of the phase space.
As of now, the maximum radius extends to 6Mpc and the maximum velocity allowed is 5000km/s
The "q" parameter is termed "scale" here which we have set to 10 as default, but can go as high as 50.
"normalization" is simply H0
"r/vres" can be any value, but are recommended to be above 150
"adj" is a custom value and changes the size of uniform filters when used (not normally needed)
Parameters
----------
rvalues : r-coordinates of points in phase space
vvalues : v-coordinates of points in phase space
r200 : Required estimate of r200 to calculate a rough dispersion
normalization = 100 : This is equivalent to H0. Default is H0=100
scale = 10 : "q" parameter in Diaferio 99. Literature says this can be between 10-50
rres = 200 : r-grid resolution
vres = 220 : v-grid resolution
rmax = 6.0 : Maximum r-grid value. If data points exceed this amount either increase
this value or cut sample to be within this value.
vmax = 5000 : Maximum/minimum v-grid value. If data points exceed this amount either increase
this value or cut sample to be within this value.
Returns
-------
self.r_range : array of r-grid values
self.v_range : array of v-grid values
self.img : smoothed density image
self.img_grad : first derivative of img
self.img_inf : second derivative of img
"""
if np.max(rvalues) >= rmax:
raise Exception(
'Bounding Error: Please either increase your rmax value or trim your sample to be r < ' + str(rmax))
if np.max(np.abs(vvalues)) >= vmax:
raise Exception(
'Bounding Error: Please either increase your vmax value or trim your sample to be v < ' + str(vmax))
vvalues = vvalues / (normalization * scale)
self.r_range = np.arange(0, rmax, 0.05)
r_range_bin = np.arange(0, rmax + 0.05, 0.05)
rres = self.r_range.size
self.v_range = np.arange(-vmax / (normalization * scale), vmax / (normalization * scale),
0.05) * normalization * scale
v_range_bin = np.arange(-1 * vmax / (normalization * scale), (vmax / (normalization * scale)) + 0.05, 0.05)
# vres = self.v_range.size
r_scale = (rvalues / rmax) * rres
v_scale = ((vvalues * (normalization * scale) + vmax) / (vmax * 2.0)) * self.v_range.size
# self.ksize_r = (4.0/(3.0*rvalues.size))**(1/5.0)*np.std(r_scale[rvalues<r200])
ksize_r = (4.0 / (3.0 * rvalues.size)) ** (1 / 5.0) * \
np.sqrt((
biweight_midvariance((r_scale[rvalues < r200]).copy(), 9.0) ** 2 +
biweight_midvariance((v_scale[rvalues < r200]).copy(), 9.0) ** 2) / 2.0
)
ksize_r *= 1.0
ksize_v = ksize_r # (4.0/(3.0*rvalues.size))**(1/5.0)*np.std(v_scale[rvalues<r200])
imgr, redge, vedge = np.histogram2d(rvalues, vvalues, bins=(r_range_bin, v_range_bin))
self.img = ndi.gaussian_filter(imgr, (ksize_r, ksize_v), mode='reflect')
self.img_grad = ndi.gaussian_gradient_magnitude(imgr, (ksize_r, ksize_v))
self.img_inf = ndi.gaussian_gradient_magnitude(ndi.gaussian_gradient_magnitude(imgr, (ksize_r, ksize_v)),
(ksize_r, ksize_v))
clus_dec = DEC[i] clus_z = Z[i] rvir = RVIR[i] ang_d,lum_d = C.zdistance(clus_z,H0) angles = C.findangle(gal_ra,gal_dec,clus_ra,clus_dec) rdata = angles * ang_d vdata = c * (gal_z - clus_z) / (1 + clus_z) # Calculate HVD rlim = rvir * 1.25 # Mpc rad_cut = np.where(rdata < rlim)[0] if len(rad_cut) == 0: continue elif len(rad_cut) > 2: hvd = astats.biweight_midvariance(vdata[np.where((rdata < rlim)&(np.abs(vdata)<3000))]) else: hvd = np.std(vdata[np.where(rdata < rlim)]) Nspec.append( np.where((rdata < rvir) & (np.abs(vdata) < hvd * 1.5))[0].size ) HVD.append(hvd) good2.append(i) if i % 200 == 0: print i Nspec,HVD,good2 = np.array(Nspec),np.array(HVD),np.array(good2) N200 = [] for i in good2: N200.append(new[i][2,1,4,9] - newb[i][2,1,4,1]) N200 = np.array(N200) new_halos = halos[good2]
ones[~np.isfinite(ffskysub)] = np.nan
wave_here = (def_wave > 4550) & (def_wave <= 4650)
for star in stars:
detectid = star["detectid"]
ra, dec = star["ra_1"], star["dec_1"]
rsqs = distsq(ra, dec, shot_tab["ra"], shot_tab["dec"])
mask_here = rsqs < 10**2
N = np.nansum(ones[mask_here][:, wave_here], axis=1)
flux_here = ffskysub[mask_here][:, wave_here]
weights_here = weights[mask_here][:, wave_here]
these_fibers_b = biweight_location(flux_here, ignore_nan=True, axis=1)
these_errs_b = np.sqrt(
biweight_midvariance(flux_here, ignore_nan=True,
axis=1)) / np.sqrt(N - 1)
weight_sum = np.nansum(weights_here, axis=1)
mean_flux_weighted = np.nansum(flux_here * weights_here,
axis=1) / weight_sum
diff_squared = (flux_here.T - mean_flux_weighted)**2
diff_squared = diff_squared.T
mean_error_weighted = np.sqrt(
np.nansum(diff_squared * weights_here, axis=1) /
((N - 1) * weight_sum))
rs = np.sqrt(rsqs[mask_here])
p0 = [np.nanmax(mean_flux_weighted), 1.3]
mask_finite = np.isfinite(mean_flux_weighted) & np.isfinite(
mean_error_weighted)
def _cal_contrast(image, mask, pixel_scale, zeropoint, sigma=1.0, scale_arcsec=60,
minfrac=0.8, minback=6, verbose=True, logger=None):
"""
Calculate the surface brightness detection limit on a given angular scale.
The error of SB limit is calculated according to https://stats.stackexchange.com/questions/631/standard-deviation-of-standard-deviation.
Parameters:
image (numpy 2-D array): input image.
mask (numpy 2-D array): if you want to mask out a pixel, set its value to 1; otherwise set to zero.
pixel_scale (float): pixel scale of the input image, in the unit of ``arcsec/pixel``.
zeropoint (float): photometric zeropoint of the input image.
sigma (float): indicates the detection threshold on which the SB limit will be calculated.
scale_arcsec (float): on which scale we calculate SB limit, in the unit of ``arcsec``.
If ``scale_arcsec=60``, this function prints out SB limit on the scale of 60 arcsec * 60 arcsec square.
minfrac (float): Must be less than 1.0. We discard super-pixels in which less than ``minfrac`` fraction of pixels are available.
Hence super-pixels with too many pixels masked out are discarded.
minback (int): Given a super-pixel, we discard it (set to zero) if there are less than ``minback`` non-zero super-pixels surrounding it.
verbose (bool): whether print out results.
logger (``logging.logger`` object): logger for this function. Default is ``None``.
"""
import copy
from astropy.stats import biweight_midvariance
from astropy.stats import biweight_location
## read image
ny, nx = image.shape
scale_pix = scale_arcsec / pixel_scale # scale in pixel
scale_x = np.array([scale_pix, int(scale_pix), int(scale_pix), int(scale_pix) + 1])
scale_y = np.array([scale_pix, int(scale_pix), int(scale_pix) + 1, int(scale_pix) + 1])
area = scale_x * scale_y
area -= area[0]
area = abs(area)[1:] # d1, d2, d3
bin_x = int(scale_x[np.argmin(area) + 1])
bin_y = int(scale_y[np.argmin(area) + 1])
area_ratio = bin_x * bin_y / scale_pix**2
if verbose:
if logger is not None:
logger.info("Determine surface brightness detection limit")
logger.info(' - Binning factors: dx = {0}, dy = {1}'.format(bin_x, bin_y))
logger.info(' - Used bin area / True bin area = {:.5f}'.format(area_ratio))
else:
print('# Determine surface brightness detection limit')
print(' - Binning factors: dx = {0}, dy = {1}'.format(bin_x, bin_y))
print(' - Used bin area / True bin area = {:.5f}'.format(area_ratio))
nbins_x = np.int(nx / bin_x)
nbins_y = np.int(ny / bin_y)
im_var = np.zeros((nbins_y, nbins_x))
im_loc = np.zeros((nbins_y, nbins_x))
im_frac = np.zeros((nbins_y, nbins_x))
im_fluct = np.zeros((nbins_y, nbins_x))
for i in range(nbins_x - 1):
for j in range(nbins_y - 1):
x1, x2, y1, y2 = i * bin_x, (i + 1) * bin_x, j * bin_y, (j + 1) * bin_y
im_sec = image[y1:y2, x1:x2]
im_mask_sec = mask[y1:y2, x1:x2]
im_sec_in = im_sec[(im_mask_sec == 0)]
if im_sec_in.size > 0:
im_var[j, i] = biweight_midvariance(im_sec_in)
im_loc[j, i] = biweight_location(im_sec_in)
im_frac[j, i] = 1 - np.float(im_sec_in.size) / np.float(im_sec.size)
'''
temp = copy.deepcopy(image)
temp[mask==1] = np.nan
# var
im_var = block_reduce(temp, (bin_y, bin_x), func=np.nanvar, cval=np.nan)
# loc
im_loc = block_reduce(temp, (bin_y, bin_x), func=np.nanmedian, cval=np.nan)
# frac
im_frac = block_reduce(mask, (bin_y, bin_x), func=np.sum, cval=np.nan) / (bin_x * bin_y)
'''
# calculate fluctuation
for i in range(1, nbins_x - 1):
for j in range(1, nbins_y - 1):
backvec = im_loc[j-1:j+2, i-1:i+2]
backvec = np.delete(backvec.flatten(), 4)
maskvec = im_frac[j-1:j+2, i-1:i+2]
maskvec = np.delete(maskvec.flatten(), 4) # fraction of being masked out
backvec_in = backvec[(maskvec < 1 - minfrac)]
if len(backvec_in) > minback:
im_fluct[j,i] = im_loc[j,i] - biweight_location(backvec_in)
im_fluct_in = im_fluct[im_fluct != 0]
sig_adu = np.sqrt(biweight_midvariance(im_fluct_in)) * 0.80 # 8/9 is area correction
sig_adu *= sigma
dsig_adu = sig_adu / np.sqrt(2 * (im_fluct_in.size - 1))
dsig_adu *= sigma
# For the standard deviation of standard deviation, see this: https://stats.stackexchange.com/questions/631/standard-deviation-of-standard-deviation
# convert to magnitudes
sb_lim = zeropoint - 2.5 * np.log10(sig_adu / pixel_scale**2)
dsb_lim = 2.5 * np.log10(1 + 1/np.sqrt(im_fluct_in.size))
#2.5 / np.log(10) / sig_adu * dsig_adu
if verbose:
if logger is not None:
logger.info(' - {0:.1f}-sigma variation in counts = {1:.4f} +- {2:.4f}'.format(sigma, sig_adu, dsig_adu))
logger.info(' - {0:.1f}-sigma surface brightness limit on {1} arcsec scale is {2:.4f} +- {3:.4f}'.format(sigma, scale_arcsec, sb_lim, dsb_lim))
else:
print(' - {0:.1f}-sigma variation in counts = {1:.4f} +- {2:.4f}'.format(sigma, sig_adu, dsig_adu))
print(' - {0:.1f}-sigma surface brightness limit on {1} arcsec scale is {2:.4f} +- {3:.4f}'.format(sigma, scale_arcsec, sb_lim, dsb_lim))
return (sb_lim, dsb_lim), (sig_adu, dsig_adu), [im_fluct, im_loc, im_var, im_frac]
