def get_shear_weighted(data, shape_noise=True):
"""
If shape_noise==False then it is assumed there is
no shape noise (e.g. a ring) so only the errors are used
"""
from esutil.stat import wmom
wt = data["weight"]
e1mean, e1err = wmom(data["e1"], wt, calcerr=True)
e2mean, e2err = wmom(data["e2"], wt, calcerr=True)
R, Rerr = wmom(data["R"], wt, calcerr=True)
ssh, ssherr = wmom(data["ssh"], wt, calcerr=True)
if not shape_noise:
err2 = data["err"] ** 2
e1ivar = (1.0 / err2).sum()
e1err = numpy.sqrt(1.0 / e1ivar)
e2err = e1err
sh1 = 0.5 * e1mean / ssh
sh2 = 0.5 * e2mean / ssh
sh1err = 0.5 * e1err / ssh
sh2err = 0.5 * e2err / ssh
shear = numpy.array([sh1, sh2], dtype="f8")
shear_err = numpy.array([sh1err, sh2err], dtype="f8")
return shear, shear_err, ssh, R
def get_mean_shear(g, gsens, weight):
g_mean,g_err=wmom(g, weight, calcerr=True)
gsens_mean,gsens_err=wmom(gsens, weight, calcerr=True)
g_mean /= gsens_mean
g_err /= gsens_mean
return {'g':g_mean,
'g_err':g_err,
'gsens':gsens_mean,
'gsens_err':gsens_err}
def get_shear(data, **keys):
"""
parameters
----------
data:
Outputs from running regauss.
include_shape_noise_err: bool, optional
If True, just use the weighted variance of shapes, otherwise
use the errors only.
shape_noise_type: string, optional
What type of fits from cosmos to use for shape noise.
"""
from esutil.stat import wmom
include_shape_noise_err=keys.get('include_shape_noise_err',True)
wt, ssh = get_weight_and_ssh(data['e1'], data['e2'], data['err'],
**keys)
e1mean,e1err = wmom(data['e1'], wt, calcerr=True)
e2mean,e2err = wmom(data['e2'], wt, calcerr=True)
R,Rerr = wmom(data['R'], wt, calcerr=True)
ssh,ssherr = wmom(ssh, wt, calcerr=True)
if not include_shape_noise_err:
err2=err**2
e1ivar = ( 1.0/err2 ).sum()
e1err = numpy.sqrt( 1.0/e1ivar )
e2err = e1err
g1=0.5*e1mean/ssh
g2=0.5*e2mean/ssh
g1err = 0.5*e1err/ssh
g2err = 0.5*e2err/ssh
shear=numpy.array([g1,g2])
shear_err=numpy.array([g1err,g2err])
shear_cov=numpy.zeros( (2,2) )
shear_cov[0,0]=g1err**2
shear_cov[1,1]=g2err**2
out={'shear':shear,
'shear_err':shear_err,
'shear_cov':shear_cov,
'ssh':ssh,
'R':R,
'Rerr':Rerr}
return out
def bin_shear_data(data, bin_field, **keys):
"""
median or wmedian are not an improvement
"""
use_median=keys.get('use_median',False)
use_wmedian=keys.get('use_wmedian',False)
h,rev=histogram(data[bin_field], rev=True, **keys)
nbin=len(h)
dt,fields=get_binned_dtype(bin_field)
bindata=zeros(nbin, dtype=dt)
for i in xrange(nbin):
if rev[i] != rev[i+1]:
w=rev[ rev[i]:rev[i+1] ]
bindata['n'][i] = w.size
wts=get_weights(data['gcov'], ind=w)
for field in fields:
if field == 'g1':
fdata=data['g'][w,0]
elif field=='g2':
fdata=data['g'][w,1]
elif field=='g1sens':
fdata=data['gsens'][w,0]
elif field=='g2sens':
fdata=data['gsens'][w,1]
else:
fdata=data[field][w]
err_field=field+'_err'
wmean,werr=wmom(fdata, wts, calcerr=True)
if use_median:
bindata[field][i] = median(fdata)
elif use_wmedian:
bindata[field][i] = wmedian(fdata,wts)
else:
bindata[field][i] = wmean
bindata[err_field][i] = werr
# for when we use a binsize instead of nperbni
w,=where(h > 0)
bindata=bindata[w]
bindata['g1'] /= bindata['g1sens']
bindata['g1_err'] /= bindata['g1sens']
bindata['g2'] /= bindata['g2sens']
bindata['g2_err'] /= bindata['g2sens']
return bindata
def logbin_shear_data(data, bin_field, **keys):
"""
Send nbin
"""
mindata=keys.get('min',data[bin_field].min())
maxdata=keys.get('max',data[bin_field].max())
keys['min']=log10(mindata)
keys['max']=log10(maxdata)
keys['rev']=True
logdata=log10(data[bin_field])
h,rev=histogram(logdata, **keys)
nbin=len(h)
dt,fields=get_binned_dtype(bin_field)
bindata=zeros(nbin, dtype=dt)
for i in xrange(nbin):
if rev[i] != rev[i+1]:
w=rev[ rev[i]:rev[i+1] ]
bindata['n'][i] = w.size
wts=get_weights(data['gcov'], ind=w)
for field in fields:
if field == 'g1':
fdata=data['g'][w,0]
elif field=='g2':
fdata=data['g'][w,1]
elif field=='g1sens':
fdata=data['gsens'][w,0]
elif field=='g2sens':
fdata=data['gsens'][w,1]
else:
fdata=data[field][w]
err_field=field+'_err'
wmean,werr=wmom(fdata, wts, calcerr=True)
bindata[field][i] = wmean
bindata[err_field][i] = werr
w,=where(h > 0)
bindata=bindata[w]
bindata['g1'] /= bindata['g1sens']
bindata['g1_err'] /= bindata['g1sens']
bindata['g2'] /= bindata['g2sens']
bindata['g2_err'] /= bindata['g2sens']
return bindata
def set_averages(self):
nsh=len(files.SHNUMS)
dt=[('s2n','f8'),
('g1true','f8'),
('g2true','f8'),
('g1','f8'),('g1_err','f8'),
('g2','f8'),('g2_err','f8'),
('g1sens','f8'),('g1sens_err','f8'),
('g2sens','f8'),('g2sens_err','f8')]
out=zeros(nsh, dtype=dt)
for i,shnum in enumerate(files.SHNUMS):
data=self.datadict[shnum]
shres=stats.get_mean_shear_shstruct(data)
wts=stats.get_weights(data['gcov'])
out['s2n'][shnum-1],err = wmom(data[self.s2n_field], wts)
out['g1true'][shnum-1] = cluster_step.sh1exp[shnum]
out['g2true'][shnum-1] = cluster_step.sh2exp[shnum]
for n in shres:
out[n][shnum-1] = shres[n]
self.avg=out
def bhist_vs(data, *fields, **keys):
"""
Plot data from an array with fields or dictionary.
If only xfield is sent, a histogram of that field is the only plot. If
other arguments are sent, these name other fields in the input data to
plot vs x in the same bins.
parameters
----------
data: numpy array with fields or dict
Must have field names. This can be a recarray or ordinary
array with field names, or even a dict as long as the arrays
all have the same length.
field1, field2, ...: string
A set of fields names to plot. The first is the "x" variable The data
are binned according to this variable. If only a single field is sent,
a simple histogram is shown. If multiple are given, the average as a
function of x is shown in separate plots.
Note if nperbin= is given, no histogram is shown unless binsize is
*also* given (to be implemented). In that case a histogram of "x" is
also shown in light grey on the background.
stype: string
The type of statistic to plot
if 'mean', plot the mean with errors as a function of the
binning field.
if 'sdev', plot the standard deviation as a function of the
binning field.
names: dict
Dictionary with names for plotting, e.g. if a field name is 'x'
this could be {'x':'new name for x'}.
clip: bool
If clip=true and weights are not sent for the histogram, the
data are sigma clipped at 4 sigma with 4 iterations.
extra keywords:
Extra keywords for the histogram program and for plotting.
"""
import biggles
from itertools import izip
import copy
from esutil.stat import wmom
if len(fields) == 0:
raise ValueError("Send at least one field name")
fields=list(fields)
stype=keys.get('stype','mean')
# names for the fields in the plots
knames=keys.get('names',{})
plabels = {}
for k in fields:
if k in knames:
plabels[k] = knames[k]
else:
plabels[k] = k
xfield = fields.pop(0)
x=data[xfield]
keys['more'] = True
hout = esutil.stat.histogram(x, **keys)
plots=[]
if 'nperbin' not in keys:
if 'weights' in keys:
hcurve = make_hist_curve(hout['low'],hout['high'],hout['whist'])
else:
hcurve = make_hist_curve(hout['low'],hout['high'],hout['hist'])
hplt = biggles.FramedPlot()
hplt.add(hcurve)
hplt.xlabel = plabels[xfield]
hplt.show()
plots.append(hplt)
nfields=len(fields)
if nfields == 0:
return
nx=len(x)
bindata=[]
nbin = hout['hist'].size
nonempty,=numpy.where(hout['hist'] > 0)
# now make a data set for each argument
for f in fields:
if len(data[f]) != nx:
raise ValueError("field %s is not same size as field %s" % (f,xfield))
d = {'name':f,
'plabel':plabels[f]}
if stype == 'mean':
d['mean'] = numpy.zeros(nbin)
d['err'] = numpy.zeros(nbin)
else:
d['sdev'] = numpy.zeros(nbin)
bindata.append(d)
# get averages for each argument in each bin
rev=hout['rev']
weights=keys.get('weights',None)
# this only applies if weights are None
clip=keys.get('clip',False)
for i in xrange(nbin):
if rev[i] != rev[i+1]:
w=rev[ rev[i]:rev[i+1] ]
for bd in bindata:
ydata = data[bd['name']][w]
if weights is not None:
mn,err,sdev = wmom(ydata, weights[w], sdev=True)
else:
if clip:
mn,sdev=esutil.stat.sigma_clip(ydata)
else:
mn = ydata.mean()
sdev = ydata.std()
err = sdev/numpy.sqrt(w.size)
if stype == 'mean':
bd['mean'][i] = mn
bd['err'][i] = err
else:
bd['sdev'][i] = sdev
# now run through and make all the plots
keys['xlabel'] = plabels[xfield]
for bd in bindata:
keys['ylabel'] = bd['plabel']
if 'mean' in hout:
xh = hout['mean'][nonempty]
else:
xh = hout['center'][nonempty]
if stype == 'mean':
plt=bscatter(xh, bd['mean'][nonempty], yerr=bd['err'][nonempty], **keys)
else:
plt=bscatter(xh, bd['sdev'][nonempty], **keys)
plots.append(plt)
return plots
def bhist_vs(data, *fields, **keys):
"""
Plot data from an array with fields or dictionary.
If only xfield is sent, a histogram of that field is the only plot. If
other arguments are sent, these name other fields in the input data to
plot vs x in the same bins.
parameters
----------
data: numpy array with fields or dict
Must have field names. This can be a recarray or ordinary
array with field names, or even a dict as long as the arrays
all have the same length.
field1, field2, ...: string
A set of fields names to plot. The first is the "x" variable The data
are binned according to this variable. If only a single field is sent,
a simple histogram is shown. If multiple are given, the average as a
function of x is shown in separate plots.
Note if nperbin= is given, no histogram is shown unless binsize is
*also* given (to be implemented). In that case a histogram of "x" is
also shown in light grey on the background.
stype: string
The type of statistic to plot
if 'mean', plot the mean with errors as a function of the
binning field.
if 'sdev', plot the standard deviation as a function of the
binning field.
names: dict
Dictionary with names for plotting, e.g. if a field name is 'x'
this could be {'x':'new name for x'}.
clip: bool
If clip=true and weights are not sent for the histogram, the
data are sigma clipped at 4 sigma with 4 iterations.
extra keywords:
Extra keywords for the histogram program and for plotting.
"""
import biggles
from itertools import izip
import copy
from esutil.stat import wmom
if len(fields) == 0:
raise ValueError("Send at least one field name")
fields = list(fields)
stype = keys.get('stype', 'mean')
# names for the fields in the plots
knames = keys.get('names', {})
plabels = {}
for k in fields:
if k in knames:
plabels[k] = knames[k]
else:
plabels[k] = k
xfield = fields.pop(0)
x = data[xfield]
keys['more'] = True
hout = esutil.stat.histogram(x, **keys)
plots = []
if 'nperbin' not in keys:
if 'weights' in keys:
hcurve = make_hist_curve(hout['low'], hout['high'], hout['whist'])
else:
hcurve = make_hist_curve(hout['low'], hout['high'], hout['hist'])
hplt = biggles.FramedPlot()
hplt.add(hcurve)
hplt.xlabel = plabels[xfield]
hplt.show()
plots.append(hplt)
nfields = len(fields)
if nfields == 0:
return
nx = len(x)
bindata = []
nbin = hout['hist'].size
nonempty, = numpy.where(hout['hist'] > 0)
# now make a data set for each argument
for f in fields:
if len(data[f]) != nx:
raise ValueError("field %s is not same size as field %s" %
(f, xfield))
d = {'name': f, 'plabel': plabels[f]}
if stype == 'mean':
d['mean'] = numpy.zeros(nbin)
d['err'] = numpy.zeros(nbin)
else:
d['sdev'] = numpy.zeros(nbin)
bindata.append(d)
# get averages for each argument in each bin
rev = hout['rev']
weights = keys.get('weights', None)
# this only applies if weights are None
clip = keys.get('clip', False)
for i in xrange(nbin):
if rev[i] != rev[i + 1]:
w = rev[rev[i]:rev[i + 1]]
for bd in bindata:
ydata = data[bd['name']][w]
if weights is not None:
mn, err, sdev = wmom(ydata, weights[w], sdev=True)
else:
if clip:
mn, sdev = esutil.stat.sigma_clip(ydata)
else:
mn = ydata.mean()
sdev = ydata.std()
err = sdev / numpy.sqrt(w.size)
if stype == 'mean':
bd['mean'][i] = mn
bd['err'][i] = err
else:
bd['sdev'][i] = sdev
# now run through and make all the plots
keys['xlabel'] = plabels[xfield]
for bd in bindata:
keys['ylabel'] = bd['plabel']
if 'mean' in hout:
xh = hout['mean'][nonempty]
else:
xh = hout['center'][nonempty]
if stype == 'mean':
plt = bscatter(xh,
bd['mean'][nonempty],
yerr=bd['err'][nonempty],
**keys)
else:
plt = bscatter(xh, bd['sdev'][nonempty], **keys)
plots.append(plt)
return plots
