def select_gals(*, data, rmagmin, rmagmax, rmag_index):
w, = np.where(
(data['ext_combo'] == 1) &
between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax)
)
mn, std = eu.stat.sigma_clip(data['conc'][w])
minconc, maxconc = [mn - 5*std, mn + 5*std]
if rmagmin < 20:
minconc = 0.004
elif rmagmin < 20.5:
minconc = 0.003
elif rmagmin < 21:
minconc = 0.0015
elif rmagmin < 21.5:
minconc = 0.001
if minconc < -0.014:
minconc = -0.014
ww, = np.where(
between(data['conc'][w], minconc, maxconc)
)
w = w[ww]
return w
def get_radec_range_logic(self, ra, dec):
"""
not currently used
apply cuts on ra,dec range if set
"""
ra_range=self.get('ra_range',None)
dec_range=self.get('dec_range',None)
if ra_range is not None:
print(" Cutting ra to [%g, %g]" % tuple(ra_range))
print(" Cutting dec to [%g, %g]" % tuple(dec_range))
logic = ( between(ra, ra_range[0], ra_range[1])
& between(dec, dec_range[0], dec_range[1]) )
w,=numpy.where(logic)
print(" remaining %d/%d" % (w.size,ra.size))
if w.size == 0:
raise ValueError("No objects passed ra,dec range cut")
else:
logic = numpy.ones(ra.size, dtype='bool')
return logic
def sample(self, size=None):
"""
get [r50, flux] or [:, r50_flux]
"""
if size is None:
size=1
is_scalar=True
else:
is_scalar=False
r50min,r50max=self.r50_range
fmin,fmax=self.flux_range
data=numpy.zeros( (size,2) )
ngood=0
nleft=data.shape[0]
while nleft > 0:
r=self.kde.resample(size=nleft).T
w,=numpy.where(
between(r[:,0], r50min, r50max) &
between(r[:,1], fmin, fmax)
)
if w.size > 0:
data[ngood:ngood+w.size,:] = r[w,:]
ngood += w.size
nleft -= w.size
if is_scalar:
data=data[0,:]
return data
def _load_data_old(self):
fname='real_galaxy_catalog_25.2_fits.fits'
"""
fname=os.path.join(
sys.exec_prefix,
'share',
'galsim',
'COSMOS_25.2_training_sample',
fname,
)
"""
fname=os.path.join(
galsim.meta_data.share_dir,
'COSMOS_25.2_training_sample',
fname,
)
r50min,r50max=self.r50_sanity_range
fmin,fmax=self.flux_sanity_range
logger.info("reading cosmos file: %s" % fname)
alldata=fitsio.read(fname, lower=True)
w,=numpy.where(
(alldata['viable_sersic']==1) &
between(alldata['hlr'][:,0], r50min, r50max) &
between(alldata['flux'][:,0], fmin, fmax)
)
logger.debug("kept %d/%d" % (w.size, alldata.size))
self.alldata=alldata[w]
def fit_joint_TF_fracdev_run(run,
model,
fracdev_min=-1.0,
fracdev_max=1.1,
fracdev_err_max=0.1,
**keys):
"""
Fit a joint prior to the fields from the given run
Calls more generic stuff such as fit_joint_noshape
"""
import fitsio
from esutil.numpy_util import between
conf=files.read_config(run)
keys['noshape']=True
keys['dolog']=True
keys['min_covar']=keys.get('min_covar',1.0e-4)
#keys['ngauss']=keys.get('ngauss',5)
data=read_all(run, **keys)
if model=='cm' and 'composite_g' in data.dtype.names:
n=Namer('composite')
else:
n=Namer(model)
T=data[n('pars')][:,4]
F=data[n('pars')][:,5]
fracdev = data[ n('fracdev') ]
fracdev_err = data[ n('fracdev_err') ]
w,=where(
between(T, -7.0, 2.0)
& between(F, -1.0,4.0)
& between(fracdev, fracdev_min, fracdev_max)
& (fracdev != 0.0)
& (fracdev != 1.0)
& (fracdev_err < fracdev_err_max)
)
data=data[w]
pars = zeros( (w.size, 3) )
pars[:,0] = data[n('pars')][:,4]
pars[:,1] = data[n('pars')][:,5]
pars[:,2] = data[n('fracdev')]
conf['partype']='TF-fracdev'
fits_name=files.get_prior_file(ext='fits', **conf)
eps_name=files.get_prior_file(ext='eps', **conf)
print(fits_name)
print(eps_name)
fit_joint_noshape(pars,
model,
fname=fits_name,
eps=eps_name,
**keys)
def select_stars(*, data, rmagmin, rmagmax, rmag_index):
w, = np.where(
(data['ext_combo'] == 0) &
between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax)
)
mn, std = eu.stat.sigma_clip(data['conc'][w])
minconc, maxconc = [mn - 4*std, mn + 6*std]
ww, = np.where(
between(data['conc'][w], minconc, maxconc)
)
w = w[ww]
return w
def _load_data(self):
fname=files.get_cosmos_file()
r50min,r50max=self.r50_sanity_range
fmin,fmax=self.flux_sanity_range
logger.info("reading cosmos file: %s" % fname)
alldata=fitsio.read(fname)
w,=numpy.where(
between(alldata['hlr'][:,0], r50min, r50max) &
between(alldata['flux'][:,0], fmin, fmax)
)
logger.debug("kept %d/%d" % (w.size, alldata.size))
self.alldata=alldata[w]
def sample(self, n=None, sigma_factor=1):
if n is not None:
scalar = False
else:
n = 1
scalar = True
rand = np.zeros(n)
ngood = 0
nleft = n
while nleft > 0:
tmp = self.rng.normal(
size=n,
loc=self.mean,
scale=self.sigma * sigma_factor,
)
w, = np.where(between(tmp, self.bounds[0], self.bounds[1]))
if w.size > 0:
rand[ngood:ngood + w.size] = tmp
ngood += w.size
nleft -= w.size
if scalar:
rand = rand[0]
return rand
def _preselect(self, data):
"""
sub-classes might make a pre-selection, e.g. of some flags
"""
w, = np.where(
np.isfinite(data['mcal_g'][:, 0])
& np.isfinite(data['mcal_g_1p'][:, 0])
& np.isfinite(data['mcal_g_1m'][:, 0])
& np.isfinite(data['mcal_g_2p'][:, 0])
& np.isfinite(data['mcal_g_2m'][:, 0]))
data = data[w]
if self.args.preselect:
print('pre-selecting')
R11 = (data['mcal_g_1p'][:, 0] - data['mcal_g_1m'][:, 0]) / 0.02
R22 = (data['mcal_g_2p'][:, 1] - data['mcal_g_2m'][:, 1]) / 0.02
w, = where(between(R11, -7, 9) & between(R22, -7, 9))
data = data[w]
return data
def trim_radec(ra, dec, des_region, get_index=False):
"""
trim the data to the region of intereste
"""
from esutil.numpy_util import between
print("trimming to region:", des_region)
conf = read_config("constants")
reginfo = conf["regions"][des_region]
ra_range = reginfo["ra_range"]
dec_range = reginfo["dec_range"]
w, = numpy.where(between(ra, ra_range[0], ra_range[1]) & between(dec, dec_range[0], dec_range[1]))
if get_index:
return w
else:
return ra[w], dec[w]
def __init__(self, *, fname, rng, max_inject_rmag):
self.rng = rng
data = fitsio.read(fname)
gmr = data['psf_mag'][:, 0] - data['psf_mag'][:, 1]
rmi = data['psf_mag'][:, 1] - data['psf_mag'][:, 2]
imz = data['psf_mag'][:, 2] - data['psf_mag'][:, 3]
w, = np.where(
between(gmr, -1, 3) & between(rmi, -1, 3) & between(imz, -1, 2))
self.data = data[w]
self.gmr = gmr[w]
self.rmi = rmi[w]
self.imz = imz[w]
self.bright, = np.where(self.data['psf_mag'][:, 1] < 21)
mrng = [staramp.MAGOFF, max_inject_rmag]
self.generator = eu.random.Generator(
lambda x: (x - staramp.MAGOFF)**staramp.SLOPE,
xrange=mrng,
nx=100,
)
def get_logic(data, model, prior_name):
from esutil.numpy_util import between
pconf=files.read_prior_config(prior_name)
#pprint(pconf)
n=Namer(model)
T=data[n('pars')][:,4]
F=data[n('pars')][:,5]
g=sqrt( data[n('pars')][:,2]**2
+ data[n('pars')][:,3]**2 )
Tr = pconf['T_range']
Fr = pconf['F_range']
logic = (
between(T, Tr[0], Tr[1])
& between(F, Fr[0], Fr[1])
& (g < GMAX )
)
if n('fracdev') in data.dtype.names:
fracdev = data[ n('fracdev') ]
fracdev_err = data[ n('fracdev_err') ]
fdr = pconf['fracdev_range']
logic = (
logic
& between(fracdev, fdr[0], fdr[1])
& (fracdev != 0.0)
& (fracdev != 1.0)
& (fracdev_err < pconf['fracdev_err_max'])
)
return logic
def plot_results(trials, **keys):
"""
Plot the points and histograms of trials from an MCMC chain.
"""
import biggles
import esutil
from esutil.numpy_util import where1, between
npars = trials.shape[1]
fontsize_min = keys.get("fontsize_min", 1)
biggles.configure("default", "fontsize_min", fontsize_min)
weights = keys.get("weights", None)
nbin = keys.get("nbin", 35)
names = keys.get("names", None)
show = keys.get("show", True)
nsigma = keys.get("nsigma", None)
means, cov = extract_stats(trials, weights=weights)
errs = sqrt(diag(cov))
plt = biggles.Table(npars, 2)
ind = numpy.arange(trials.shape[0])
for i in xrange(npars):
if names is not None:
name = names[i]
else:
name = r"$p_{%d}$" % i
use_trials = trials[:, i]
use_ind = ind
use_wts = weights
if nsigma is not None:
w = where1(between(trials[:, i], means[i] - nsigma * errs[i], means[i] + nsigma * errs[i]))
use_trials = trials[w, i]
use_ind = ind[w]
if weights is not None:
use_wts = weights[w]
burn_plot_i = biggles.plot(use_ind, use_trials, type="solid", xlabel="step", ylabel=name, visible=False)
plt[i, 0] = burn_plot_i
hcurve, bin_edges, harray = biggles.make_histc(use_trials, nbin=nbin, weights=use_wts, get_hdata=True)
plti = biggles.FramedPlot()
plti.xlabel = name
hmax = harray.max()
plti.yrange = [-0.05 * hmax, 1.2 * hmax]
plti.add(hcurve)
lab = r"$<%s> = %0.4g \pm %0.4g$" % (name, means[i], errs[i])
plab = biggles.PlotLabel(0.1, 0.8, lab, halign="left", color="blue")
plti.add(plab)
plt[i, 1] = plti
plt.title = keys.get("title", None)
if show:
plt.show()
return plt
def get_amp(*, rmag, conc, show=False, output=None, get_plot=False):
"""
get the amplitude of the stellar locus assuming a
power law distribution A (mag - 13.5)**1.5 The data are binned
and clipped, and sent to the calculate_amp function.
Parameters
----------
rmag: array
psf magnitude, should be r band
conc: array
concentration parameters for stars
show: bool
If True, show a plot
output: string, optional
If sent, write a plot file with that name
Returns
-------
amp, amp_err: float, float
Amplitude and uncertainty
"""
w, = np.where(between(rmag, MAGMIN, MAGMAX) & between(conc, CMIN, CMAX))
hd = eu.stat.histogram(
rmag[w],
min=MAGMIN,
max=MAGMAX,
nbin=NBIN,
more=True,
)
herr = np.sqrt(hd['hist'])
amp, amp_err = calculate_amp(rmag=hd['center'], conc=hd['hist'])
print('amp: %g +/- %g' % (amp, amp_err))
if show or output is not None or get_plot:
import hickory
# alpha = 0.5
plt = hickory.Table(2, 1, figsize=(8, 7))
plt[0].plot(
rmag,
conc,
marker='.',
alpha=0.5,
)
plt[0].plot(
rmag[w],
conc[w],
marker='.',
markeredgecolor='black',
alpha=0.5,
)
plt[0].set(xlim=(16, 20), ylim=(-0.002, 0.005))
plt[0].set(
xlabel='psf mag r',
ylabel='conc',
)
predicted = predict(rmag=hd['center'], amp=amp)
plt[1].errorbar(
hd['center'],
hd['hist'],
yerr=herr,
)
plt[1].curve(hd['center'], predicted)
xvals = np.linspace(13.9, 21)
plt[1].curve(
xvals,
predict(rmag=xvals, amp=amp),
)
plt[1].set(
xlabel='psf mag r',
ylabel='Number',
xlim=(13.5, 21),
)
if show:
plt.show()
if output is not None:
print('writing:', output)
plt.savefig(output)
if get_plot:
return amp, amp_err, plt
else:
return amp, amp_err
def fit_exp(*, rmag, conc, show=False, output=None):
"""
get the amplitude of the stellar locus assuming a
power law distribution A (mag - 13.5)**1.5 The data are binned
and clipped, and sent to the calculate_amp function.
Parameters
----------
rmag: array
psf magnitude, should be r band
conc: array
concentration parameters for stars
show: bool
If True, show a plot
output: string, optional
If sent, write a plot file with that name
Returns
-------
amp, amp_err: float, float
Amplitude and uncertainty
"""
w, = np.where(between(rmag, MAGMIN, MAGMAX) & between(conc, CMIN, CMAX))
hd = eu.stat.histogram(
rmag[w],
min=MAGMIN,
max=MAGMAX,
nbin=NBIN,
more=True,
)
num = hd['hist']
num_err = np.sqrt(num)
guess = [1000, 18, 4, num[:4].mean()]
res = fit_exp_binned(hd['center'], num, num_err, guess)
print_pars(res['pars'], front='pars: ')
print_pars(res['pars_err'], front='pars: ')
if res['flags'] != 0:
raise RuntimeError('fit failed')
if show or output is not None:
import hickory
# alpha = 0.5
plt = hickory.Table(2, 1, figsize=(8, 7))
plt[0].plot(
rmag,
conc,
marker='.',
alpha=0.5,
)
plt[0].plot(
rmag[w],
conc[w],
marker='.',
markeredgecolor='black',
alpha=0.5,
)
plt[0].set(
xlim=(MAGMIN - 0.5, MAGMAX + 0.5),
ylim=(-0.0005, CMAX),
)
plt[0].set(
xlabel='psf mag r',
ylabel='conc',
)
plt[1].errorbar(
hd['center'],
num,
yerr=num_err,
)
predicted = exp_func(res['pars'], hd['center'])
eu.misc.colprint(num,
predicted,
num_err,
format='%20s',
names=['num', 'pred', 'num_err'])
plt[1].curve(hd['center'], predicted)
xvals = np.linspace(13.9, MAGMAX + 0.5)
plt[1].curve(
xvals,
exp_func(res['pars'], xvals),
)
plt[1].set(
xlabel='psf mag r',
ylabel='Number',
xlim=(MAGMIN - 0.5, MAGMAX + 0.5),
yscale='log',
# xlim=(13.5, MAGMAX+0.5),
)
if show:
plt.show()
if output is not None:
print('writing:', output)
plt.savefig(output)
return res['pars']
def fit_conc_pdf(
*,
data,
prior_file,
rmag_index,
seed,
output,
training,
show=False,
):
rng = np.random.RandomState(seed)
print('will write to:', output)
pdf_file = replace_ext(output, '.npy', '-plots.pdf')
print('writing to pdf file:', pdf_file)
pdf = PdfPages(pdf_file)
data = data[data['flags'] == 0]
prior_data = fitsio.read(prior_file)
edges = [
(16.5, 17.0),
(17.0, 17.5),
(17.5, 18.0),
(18.0, 18.5),
(18.5, 19.0),
(19, 19.5),
(19.5, 20.0),
(20, 20.5),
(20.5, 21.0),
(21, 21.5),
(21.5, 22.0),
(22, 22.5),
(22.5, 23),
(23, 23.5),
(23.5, 24.0),
(24, 24.5),
]
outdata = make_output(len(edges))
rmag_centers = np.array(
[0.5 * (rmagmin + rmagmax) for rmagmin, rmagmax in edges])
# initial estimate of N(mag) for stars
initial_amp, initial_amp_err, plt = staramp.get_amp(
rmag=data['psf_mag'][:, rmag_index],
conc=data['conc'],
show=show,
get_plot=True,
)
pdf.savefig(figure=plt)
init_nstar, init_nstar_err = staramp.predict(
rmag=rmag_centers,
amp=initial_amp,
amp_err=initial_amp_err,
)
# gal_num_pars = galamp.fit_exp(
# rmag=data['psf_mag'][:, rmag_index],
# conc=data['conc'],
# show=show,
# )
#
# init_ngal = galamp.exp_func(gal_num_pars, rmag_centers)
# init_ngal_err = np.sqrt(init_ngal)
# init_ngal_err *= 10
# init_nstar_err *= 10
ngal_list = np.zeros(len(edges))
for i in range(len(edges)):
rmagmin, rmagmax = edges[i]
rmag = 0.5 * (rmagmin + rmagmax)
label = r'$%.2f < r < %.2f$' % (rmagmin, rmagmax)
print('-' * 70)
print(label)
minconc, maxconc = -0.01, 0.025
w, = np.where(
between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax)
& between(data['conc'], minconc, maxconc))
print('number in bin:', w.size)
nstar_predicted = init_nstar[i]
star_priors = get_star_priors(
rng=rng,
data=prior_data,
rmag=rmag,
nstar=nstar_predicted,
nstar_err=init_nstar_err[i],
)
# if rmag < 18.0:
# gal_ngauss = 1
# else:
# gal_ngauss = 2
gal_ngauss = 2
ngal_predicted = w.size - nstar_predicted
if ngal_predicted < 3:
ngal_predicted = 3
gal_priors = get_gal_priors(
rng=rng,
data=prior_data,
rmag=rmag,
ngal=ngal_predicted,
ngal_err=np.sqrt(ngal_predicted),
ngauss=gal_ngauss,
training=training,
)
fitter = Fitter(
data=data['conc'][w],
star_priors=star_priors,
gal_priors=gal_priors,
rng=rng,
)
fitter.go()
res = fitter.result
print('final')
cem.gmix_print(fitter.gmix)
print(res)
assert res['flags'] == 0, 'failed to converge'
nstar_meas = fitter.gmix['num'][:2].sum()
ngal_meas = fitter.gmix['num'][2:].sum()
nobj_meas = fitter.gmix['num'].sum()
ngal_list[i] = ngal_meas
print('true num: %g meas num: %g' % (w.size, nobj_meas))
print('nstar pred: %g nstar meas: %g' % (nstar_predicted, nstar_meas))
print('ngal pred: %g ngal meas: %g' % (ngal_predicted, ngal_meas))
if rmag > 23.5:
plt = fitter.plot(title=label, legend=True, show=show)
else:
plt = fitter.plot3(label=label, show=show)
pdf.savefig(figure=plt)
outdata['rmagmin'][i] = rmagmin
outdata['rmagmax'][i] = rmagmax
outdata['rmag'][i] = rmag
outdata['gmix'][i] = fitter.gmix
plt = plot_fits_vs_rmag(outdata, 'star', show=show)
pdf.savefig(plt)
plt = plot_fits_vs_rmag(outdata, 'gal', show=show)
pdf.savefig(plt)
print('closing pdf file:', pdf_file)
pdf.close()
print('writing:', output)
np.save(output, outdata)
def _run_sep(self):
import sep
# THRESH=1.2 # in sky sigma
# DETECT_THRESH=1.6 # in sky sigma
# DEBLEND_MINCONT=0.005
# DETECT_MINAREA = 6 # minimum number of pixels above threshold
objs, seg = sep.extract(
self.detim,
self.detect_thresh,
err=self.detnoise,
segmentation_map=True,
**self.sx_config
)
logger.debug('found %d objects' % objs.size)
if objs.size == 0:
self.cat = objs
return None
flux_auto = np.zeros(objs.size)-9999.0
fluxerr_auto = np.zeros(objs.size)-9999.0
flux_radius = np.zeros(objs.size)-9999.0
kron_radius = np.zeros(objs.size)-9999.0
w, = np.where(
(objs['a'] >= 0.0) &
(objs['b'] >= 0.0) &
between(objs['theta'], -pi/2., pi/2., type='[]')
)
if w.size > 0:
kron_radius[w], krflag = sep.kron_radius(
self.detim,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
6.0,
)
objs['flag'][w] |= krflag
aper_rad = 2.5*kron_radius
flux_auto[w], fluxerr_auto[w], flag_auto = \
sep.sum_ellipse(
self.detim,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
aper_rad[w],
subpix=1,
)
objs['flag'][w] |= flag_auto
# what we did in DES, but note threshold above
# is 1 as opposed to wide survey. deep survey
# was even lower, 0.8?
# used half light radius
PHOT_FLUXFRAC = 0.5
flux_radius[w], frflag = sep.flux_radius(
self.detim,
objs['x'][w],
objs['y'][w],
6.*objs['a'][w],
PHOT_FLUXFRAC,
normflux=flux_auto[w],
subpix=5,
)
objs['flag'][w] |= frflag # combine flags into 'flag'
ncut = 2 # need this to make sure array
new_dt = [
('id', 'i8'),
('number', 'i4'),
('ncutout', 'i4'),
('kron_radius', 'f4'),
('flux_auto', 'f4'),
('fluxerr_auto', 'f4'),
('flux_radius', 'f4'),
('isoarea_image', 'f4'),
('iso_radius', 'f4'),
('box_size', 'i4'),
('file_id', 'i8', ncut),
('orig_row', 'f4', ncut),
('orig_col', 'f4', ncut),
('orig_start_row', 'i8', ncut),
('orig_start_col', 'i8', ncut),
('orig_end_row', 'i8', ncut),
('orig_end_col', 'i8', ncut),
('cutout_row', 'f4', ncut),
('cutout_col', 'f4', ncut),
('dudrow', 'f8', ncut),
('dudcol', 'f8', ncut),
('dvdrow', 'f8', ncut),
('dvdcol', 'f8', ncut),
]
cat = eu.numpy_util.add_fields(objs, new_dt)
cat['id'] = np.arange(cat.size)
cat['number'] = np.arange(1, cat.size+1)
cat['ncutout'] = 1
cat['flux_auto'] = flux_auto
cat['fluxerr_auto'] = fluxerr_auto
cat['flux_radius'] = flux_radius
wcs = self.datalist[0]['wcs']
cat['dudrow'][:, 0] = wcs.dudy
cat['dudcol'][:, 0] = wcs.dudx
cat['dvdrow'][:, 0] = wcs.dvdy
cat['dvdcol'][:, 0] = wcs.dvdx
# use the number of pixels in the seg map as the iso area
for i in range(objs.size):
w = np.where(seg == (i+1))
cat['isoarea_image'][i] = w[0].size
cat['iso_radius'] = np.sqrt(cat['isoarea_image'].clip(min=1)/np.pi)
box_size = self._get_box_sizes(cat)
half_box_size = box_size//2
maxrow, maxcol = self.detim.shape
cat['box_size'] = box_size
cat['orig_row'][:, 0] = cat['y']
cat['orig_col'][:, 0] = cat['x']
orow = cat['orig_row'][:, 0].astype('i4')
ocol = cat['orig_col'][:, 0].astype('i4')
ostart_row = orow - half_box_size + 1
ostart_col = ocol - half_box_size + 1
oend_row = orow + half_box_size + 1 # plus one for slices
oend_col = ocol + half_box_size + 1
ostart_row.clip(min=0, out=ostart_row)
ostart_col.clip(min=0, out=ostart_col)
oend_row.clip(max=maxrow, out=oend_row)
oend_col.clip(max=maxcol, out=oend_col)
# could result in smaller than box_size above
cat['orig_start_row'][:, 0] = ostart_row
cat['orig_start_col'][:, 0] = ostart_col
cat['orig_end_row'][:, 0] = oend_row
cat['orig_end_col'][:, 0] = oend_col
cat['cutout_row'][:, 0] = \
cat['orig_row'][:, 0] - cat['orig_start_row'][:, 0]
cat['cutout_col'][:, 0] = \
cat['orig_col'][:, 0] - cat['orig_start_col'][:, 0]
self.seg = seg
self.bmask = np.zeros(seg.shape, dtype='i4')
self.cat = cat
def select_good(data,
g_field='g',
min_T=None,
T_range=None,
flux_range=None,
min_arate=None,
max_arate=None,
min_s2n=None,
min_s2n_r=None,
min_Ts2n=None,
min_Ts2n_r=None,
min_efficiency=None,
max_chi2per=None,
s2n_r_range=None,
mcal_s2n_r_range=None,
Ts2n_r_range=None,
fracdev_err_max=None,
cut_fracdev_exact=False,
fracdev_range=None,
min_Trat_r=None,
max_g=1.0):
"""
apply standard selection
"""
logic = ones(data.size, dtype=bool)
elogic = (data['flags']==0)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from flags" % (w.size, data.size))
logic = logic & elogic
if g_field in data.dtype.names:
gv = data[g_field]
g = numpy.sqrt( gv[:,0]**2 + gv[:,1]**2 )
elogic = (g < max_g)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from %s < %g" % (w.size, data.size, g_field, max_g))
logic = logic & elogic
if min_s2n is not None:
elogic = (data['s2n_w'] > min_s2n)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n > %g" % (w.size, data.size, min_s2n))
logic = logic & elogic
if min_s2n_r is not None:
if 's2n_r' in data.dtype.names:
elogic = (data['s2n_r'] > min_s2n_r)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n_r > %g" % (w.size, data.size, min_s2n_r))
logic = logic & elogic
if s2n_r_range is not None:
if 's2n_r' in data.dtype.names:
rng=s2n_r_range
elogic = (data['s2n_r'] > rng[0]) & (data['s2n_r'] < rng[1])
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n_r in [%g,%g]" % (w.size, data.size, rng[0],rng[1]))
logic = logic & elogic
if mcal_s2n_r_range is not None:
field='mcal_s2n_r'
if field in data.dtype.names:
rng=mcal_s2n_r_range
elogic = between(data[field], rng[0], rng[1])
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from %s in [%g,%g]" % (w.size, data.size, field, rng[0],rng[1]))
logic = logic & elogic
if min_Ts2n is not None:
elogic = (data['T_s2n'] > min_Ts2n)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n > %g" % (w.size, data.size, min_Ts2n))
logic = logic & elogic
if min_Ts2n_r is not None:
if 'T_s2n_r' in data.dtype.names:
elogic = (data['T_s2n_r'] > min_Ts2n_r)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n_r > %g" % (w.size, data.size, min_Ts2n_r))
logic = logic & elogic
if Ts2n_r_range is not None:
if 'T_s2n_r' in data.dtype.names:
rng=Ts2n_r_range
elogic = (data['T_s2n_r'] > rng[0]) & (data['T_s2n_r'] < rng[1])
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n_r in [%g,%g]" % (w.size, data.size, rng[0],rng[1]))
logic = logic & elogic
if min_T is not None:
elogic = (data['log_T'] > min_T)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from log_T > %g" % (w.size, data.size, min_T))
logic = logic & elogic
if T_range is not None:
elogic = (data['log_T'] > T_range[0]) & (data['log_T'] < T_range[1])
w,=where(elogic)
if w.size != data.size:
tup=(w.size, data.size, T_range[0], T_range[1])
print(" kept %d/%d from log_T in [%g,%g]" % tup)
logic = logic & elogic
if flux_range is not None:
elogic = (data['log_flux'] > flux_range[0]) & (data['log_flux'] < flux_range[1])
w,=where(elogic)
if w.size != data.size:
tup=(w.size, data.size, flux_range[0], flux_range[1])
print(" kept %d/%d from log_flux in [%g,%g]" % tup)
logic = logic & elogic
if 'arate' in data.dtype.names:
elogic = (data['arate'] > min_arate) & (data['arate'] < max_arate)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from arate [%g,%g]" % (w.size, data.size,
min_arate,max_arate))
logic = logic & elogic
if 'fracdev' in data.dtype.names:
if fracdev_range is not None:
elogic = ( (data['fracdev'] > fracdev_range[0])
& (data['fracdev'] < fracdev_range[1]) )
w,=where(elogic)
if w.size != data.size:
mess=" kept %d/%d from fracdev [%g,%g]"
print(mess % (w.size, data.size, fracdev_range[0],fracdev_range[1]))
logic = logic & elogic
if fracdev_err_max is not None:
elogic = (data['fracdev_err'] < fracdev_err_max)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from fracdev_err < %g" % (w.size, data.size, fracdev_err_max))
logic = logic & elogic
if cut_fracdev_exact:
elogic = (data['fracdev'] != 0.0) & (data['fracdev'] != 1.0)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from fracdev exact" % (w.size, data.size))
logic = logic & elogic
if min_efficiency is not None:
elogic = (data['efficiency'] > min_efficiency)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from eff > %g" % (w.size, data.size, min_efficiency))
logic = logic & elogic
if max_chi2per is not None:
elogic = (data['chi2per'] > max_chi2per)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from chi2per < %g" % (w.size, data.size, max_chi2per))
logic = logic & elogic
if min_Trat_r is not None:
Trat = data['Trat_r']
elogic = (Trat > min_Trat_r)
w,=where(elogic)
if w.size != data.size:
print(" kept %d/%d from round Tg/Tpsf > %g" % (w.size, data.size, min_Trat_r))
logic = logic & elogic
w,=where(logic)
print(" kept %d/%d" % (w.size, data.size))
data=data[w]
return data
def fit_conc_hists(*, data, prior_file, rmag_index, seed):
rng = np.random.RandomState(seed)
data = data[data['flags'] == 0]
prior_data = fitsio.read(prior_file)
edges = [
(16.5, 17.0),
(17.0, 17.5),
(17.5, 18.0),
(18.0, 18.5),
# (16, 18.5),
(18.5, 19.0),
(19, 19.5),
(19.5, 20.0),
(20, 20.5),
(20.5, 21.0),
(21, 21.5),
(21.5, 22.0),
(22, 22.5),
(22.5, 23),
(23, 23.5),
(23.5, 24.0),
(24, 24.5),
]
# edges = [
# (23, 23.5),
# (23.5, 24.0),
# (24, 24.5),
# ]
#
rmag_centers = np.array(
[0.5 * (rmagmin + rmagmax) for rmagmin, rmagmax in edges])
# initial estimate of N(mag) for stars
initial_amp, initial_amp_err = staramp.get_amp(
rmag=data['psf_mag'][:, rmag_index],
conc=data['conc'],
show=True,
)
gal_num_pars = galamp.fit_exp(
rmag=data['psf_mag'][:, rmag_index],
conc=data['conc'],
show=True,
)
init_nstar, init_nstar_err = staramp.predict(
rmag=rmag_centers,
amp=initial_amp,
amp_err=initial_amp_err,
)
init_ngal = galamp.exp_func(gal_num_pars, rmag_centers)
# init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5)
init_ngal_err = np.sqrt(init_ngal)
# init_ngal_err *= 10
# init_nstar_err *= 10
ngal_list = np.zeros(len(edges))
for i in range(len(edges)):
rmagmin, rmagmax = edges[i]
rmag = 0.5 * (rmagmin + rmagmax)
label = 'rmag: [%.2f, %.2f]' % (rmagmin, rmagmax)
binsize = 0.00005
binsize_coarse = 0.0004
binsize_coarser = 0.0004 * 2
# off = 0.007
# power = 0.5
minconc, maxconc = -0.01, 0.025
# minconc, maxconc = -0.0005, 0.01
w, = np.where(
between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax)
& between(data['conc'], minconc, maxconc))
print('number in bin:', w.size)
nstar_predicted = init_nstar[i]
star_priors = get_star_priors(
rng=rng,
data=prior_data,
rmag=rmag,
nstar=nstar_predicted,
nstar_err=init_nstar_err[i],
)
if rmag < 18.0:
gal_ngauss = 1
else:
gal_ngauss = 2
# print_pars(ngal_list[0:i], front='ngal_list: ')
# gal_num_res = galamp.fit_exp_binned(
# rmag_centers[0:i],
# ngal_list[0:i],
# np.sqrt(ngal_list[0:i]),
# gal_num_pars,
# )
# print_pars(gal_num_pars, front='orig: ')
# gal_num_pars = gal_num_res['pars']
# print_pars(gal_num_pars, front='new: ')
# init_ngal = galamp.exp_func(gal_num_pars, rmag_centers)
# # init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5)
#
# init_ngal_err = np.sqrt(init_ngal)
# gal_ngauss = 2
ngal_predicted = w.size - nstar_predicted
if ngal_predicted < 3:
ngal_predicted = init_ngal[i]
if ngal_predicted < 3:
ngal_predicted = 3
gal_priors = get_gal_priors(
rng=rng,
data=prior_data,
rmag=rmag,
ngal=ngal_predicted,
ngal_err=init_ngal_err[i],
ngauss=gal_ngauss,
)
hd = eu.stat.histogram(
data['conc'][w],
min=minconc,
max=maxconc,
binsize=binsize,
more=True,
)
plt = hickory.Plot(title=label, xlabel='concentration')
plt.bar(hd['center'], hd['hist'], width=binsize)
plt.show()
hd_coarse = eu.stat.histogram(
data['conc'][w],
min=minconc,
max=maxconc,
binsize=binsize_coarse,
more=True,
)
hd_coarser = eu.stat.histogram(
data['conc'][w],
min=minconc,
max=maxconc,
binsize=binsize_coarser,
more=True,
)
x = hd['center']
y = hd['hist']
yerr = np.sqrt(hd['hist'])
fitter = Fitter(
x=x,
y=y,
yerr=yerr,
star_priors=star_priors,
gal_priors=gal_priors,
rng=rng,
)
fitter.go()
res = fitter.result
print_pars(res['pars'], front='pars: ')
print_pars(res['pars_err'], front='perr: ')
pars = res['pars']
nstar_meas = pars[2] + pars[5]
if gal_ngauss == 1:
nobj_meas = pars[2] + pars[5] + pars[8]
ngal_meas = res['pars'][-1]
else:
nobj_meas = pars[2] + pars[5] + pars[8] + pars[11]
ngal_meas = res['pars'][-1] + res['pars'][-4]
ngal_list[i] = ngal_meas
print('true num: %g meas num: %g' % (w.size, nobj_meas))
print('nstar pred: %g nstar meas: %g' % (nstar_predicted, nstar_meas))
print('ngal pred: %g ngal meas: %g' % (ngal_predicted, ngal_meas))
if res['flags'] == 0:
fitter.plot(
figsize=(10, 7.5),
x=hd_coarse['center'],
y=hd_coarse['hist'],
xlabel='concentration',
legend=True,
show=True,
title=label,
xlim=(-0.005, 0.025),
)
splt = fitter.plot(
figsize=(10, 7.5),
legend=True,
show=False,
title=label,
xlim=(-0.003, 0.003),
)
splt.show()
cmin, cmax = 0.003, 0.025
w, = np.where(between(hd_coarser['center'], cmin, cmax))
ylim = [0, 1.1 * hd_coarser['hist'][w].max()]
fitter.plot(
figsize=(10, 7.5),
x=hd_coarser['center'],
y=hd_coarser['hist'],
xlabel='concentration',
legend=True,
show=True,
title=label,
xlim=(0, cmax),
ylim=ylim,
)
def select_good(data,
g_field='g',
min_T=None,
T_range=None,
flux_range=None,
min_arate=None,
max_arate=None,
min_s2n=None,
min_s2n_r=None,
min_Ts2n=None,
min_Ts2n_r=None,
min_efficiency=None,
max_chi2per=None,
s2n_r_range=None,
mcal_s2n_r_range=None,
Ts2n_r_range=None,
fracdev_err_max=None,
cut_fracdev_exact=False,
fracdev_range=None,
min_Trat_r=None,
max_g=1.0):
"""
apply standard selection
"""
logic = ones(data.size, dtype=bool)
elogic = (data['flags'] == 0)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from flags" % (w.size, data.size))
logic = logic & elogic
if g_field in data.dtype.names:
gv = data[g_field]
g = numpy.sqrt(gv[:, 0]**2 + gv[:, 1]**2)
elogic = (g < max_g)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from %s < %g" %
(w.size, data.size, g_field, max_g))
logic = logic & elogic
if min_s2n is not None:
elogic = (data['s2n_w'] > min_s2n)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n > %g" %
(w.size, data.size, min_s2n))
logic = logic & elogic
if min_s2n_r is not None:
if 's2n_r' in data.dtype.names:
elogic = (data['s2n_r'] > min_s2n_r)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n_r > %g" %
(w.size, data.size, min_s2n_r))
logic = logic & elogic
if s2n_r_range is not None:
if 's2n_r' in data.dtype.names:
rng = s2n_r_range
elogic = (data['s2n_r'] > rng[0]) & (data['s2n_r'] < rng[1])
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from s2n_r in [%g,%g]" %
(w.size, data.size, rng[0], rng[1]))
logic = logic & elogic
if mcal_s2n_r_range is not None:
field = 'mcal_s2n_r'
if field in data.dtype.names:
rng = mcal_s2n_r_range
elogic = between(data[field], rng[0], rng[1])
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from %s in [%g,%g]" %
(w.size, data.size, field, rng[0], rng[1]))
logic = logic & elogic
if min_Ts2n is not None:
elogic = (data['T_s2n'] > min_Ts2n)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n > %g" %
(w.size, data.size, min_Ts2n))
logic = logic & elogic
if min_Ts2n_r is not None:
if 'T_s2n_r' in data.dtype.names:
elogic = (data['T_s2n_r'] > min_Ts2n_r)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n_r > %g" %
(w.size, data.size, min_Ts2n_r))
logic = logic & elogic
if Ts2n_r_range is not None:
if 'T_s2n_r' in data.dtype.names:
rng = Ts2n_r_range
elogic = (data['T_s2n_r'] > rng[0]) & (data['T_s2n_r'] < rng[1])
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from Ts2n_r in [%g,%g]" %
(w.size, data.size, rng[0], rng[1]))
logic = logic & elogic
if min_T is not None:
elogic = (data['log_T'] > min_T)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from log_T > %g" %
(w.size, data.size, min_T))
logic = logic & elogic
if T_range is not None:
elogic = (data['log_T'] > T_range[0]) & (data['log_T'] < T_range[1])
w, = where(elogic)
if w.size != data.size:
tup = (w.size, data.size, T_range[0], T_range[1])
print(" kept %d/%d from log_T in [%g,%g]" % tup)
logic = logic & elogic
if flux_range is not None:
elogic = (data['log_flux'] > flux_range[0]) & (data['log_flux'] <
flux_range[1])
w, = where(elogic)
if w.size != data.size:
tup = (w.size, data.size, flux_range[0], flux_range[1])
print(" kept %d/%d from log_flux in [%g,%g]" % tup)
logic = logic & elogic
if 'arate' in data.dtype.names:
elogic = (data['arate'] > min_arate) & (data['arate'] < max_arate)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from arate [%g,%g]" %
(w.size, data.size, min_arate, max_arate))
logic = logic & elogic
if 'fracdev' in data.dtype.names:
if fracdev_range is not None:
elogic = ((data['fracdev'] > fracdev_range[0])
& (data['fracdev'] < fracdev_range[1]))
w, = where(elogic)
if w.size != data.size:
mess = " kept %d/%d from fracdev [%g,%g]"
print(mess %
(w.size, data.size, fracdev_range[0], fracdev_range[1]))
logic = logic & elogic
if fracdev_err_max is not None:
elogic = (data['fracdev_err'] < fracdev_err_max)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from fracdev_err < %g" %
(w.size, data.size, fracdev_err_max))
logic = logic & elogic
if cut_fracdev_exact:
elogic = (data['fracdev'] != 0.0) & (data['fracdev'] != 1.0)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from fracdev exact" %
(w.size, data.size))
logic = logic & elogic
if min_efficiency is not None:
elogic = (data['efficiency'] > min_efficiency)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from eff > %g" %
(w.size, data.size, min_efficiency))
logic = logic & elogic
if max_chi2per is not None:
elogic = (data['chi2per'] > max_chi2per)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from chi2per < %g" %
(w.size, data.size, max_chi2per))
logic = logic & elogic
if min_Trat_r is not None:
Trat = data['Trat_r']
elogic = (Trat > min_Trat_r)
w, = where(elogic)
if w.size != data.size:
print(" kept %d/%d from round Tg/Tpsf > %g" %
(w.size, data.size, min_Trat_r))
logic = logic & elogic
w, = where(logic)
print(" kept %d/%d" % (w.size, data.size))
data = data[w]
return data
