def select_gals(*, data, rmagmin, rmagmax, rmag_index): w, = np.where( (data['ext_combo'] == 1) & between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax) ) mn, std = eu.stat.sigma_clip(data['conc'][w]) minconc, maxconc = [mn - 5*std, mn + 5*std] if rmagmin < 20: minconc = 0.004 elif rmagmin < 20.5: minconc = 0.003 elif rmagmin < 21: minconc = 0.0015 elif rmagmin < 21.5: minconc = 0.001 if minconc < -0.014: minconc = -0.014 ww, = np.where( between(data['conc'][w], minconc, maxconc) ) w = w[ww] return w
def get_radec_range_logic(self, ra, dec): """ not currently used apply cuts on ra,dec range if set """ ra_range=self.get('ra_range',None) dec_range=self.get('dec_range',None) if ra_range is not None: print(" Cutting ra to [%g, %g]" % tuple(ra_range)) print(" Cutting dec to [%g, %g]" % tuple(dec_range)) logic = ( between(ra, ra_range[0], ra_range[1]) & between(dec, dec_range[0], dec_range[1]) ) w,=numpy.where(logic) print(" remaining %d/%d" % (w.size,ra.size)) if w.size == 0: raise ValueError("No objects passed ra,dec range cut") else: logic = numpy.ones(ra.size, dtype='bool') return logic
def sample(self, size=None): """ get [r50, flux] or [:, r50_flux] """ if size is None: size=1 is_scalar=True else: is_scalar=False r50min,r50max=self.r50_range fmin,fmax=self.flux_range data=numpy.zeros( (size,2) ) ngood=0 nleft=data.shape[0] while nleft > 0: r=self.kde.resample(size=nleft).T w,=numpy.where( between(r[:,0], r50min, r50max) & between(r[:,1], fmin, fmax) ) if w.size > 0: data[ngood:ngood+w.size,:] = r[w,:] ngood += w.size nleft -= w.size if is_scalar: data=data[0,:] return data
def _load_data_old(self): fname='real_galaxy_catalog_25.2_fits.fits' """ fname=os.path.join( sys.exec_prefix, 'share', 'galsim', 'COSMOS_25.2_training_sample', fname, ) """ fname=os.path.join( galsim.meta_data.share_dir, 'COSMOS_25.2_training_sample', fname, ) r50min,r50max=self.r50_sanity_range fmin,fmax=self.flux_sanity_range logger.info("reading cosmos file: %s" % fname) alldata=fitsio.read(fname, lower=True) w,=numpy.where( (alldata['viable_sersic']==1) & between(alldata['hlr'][:,0], r50min, r50max) & between(alldata['flux'][:,0], fmin, fmax) ) logger.debug("kept %d/%d" % (w.size, alldata.size)) self.alldata=alldata[w]
def fit_joint_TF_fracdev_run(run, model, fracdev_min=-1.0, fracdev_max=1.1, fracdev_err_max=0.1, **keys): """ Fit a joint prior to the fields from the given run Calls more generic stuff such as fit_joint_noshape """ import fitsio from esutil.numpy_util import between conf=files.read_config(run) keys['noshape']=True keys['dolog']=True keys['min_covar']=keys.get('min_covar',1.0e-4) #keys['ngauss']=keys.get('ngauss',5) data=read_all(run, **keys) if model=='cm' and 'composite_g' in data.dtype.names: n=Namer('composite') else: n=Namer(model) T=data[n('pars')][:,4] F=data[n('pars')][:,5] fracdev = data[ n('fracdev') ] fracdev_err = data[ n('fracdev_err') ] w,=where( between(T, -7.0, 2.0) & between(F, -1.0,4.0) & between(fracdev, fracdev_min, fracdev_max) & (fracdev != 0.0) & (fracdev != 1.0) & (fracdev_err < fracdev_err_max) ) data=data[w] pars = zeros( (w.size, 3) ) pars[:,0] = data[n('pars')][:,4] pars[:,1] = data[n('pars')][:,5] pars[:,2] = data[n('fracdev')] conf['partype']='TF-fracdev' fits_name=files.get_prior_file(ext='fits', **conf) eps_name=files.get_prior_file(ext='eps', **conf) print(fits_name) print(eps_name) fit_joint_noshape(pars, model, fname=fits_name, eps=eps_name, **keys)
def select_stars(*, data, rmagmin, rmagmax, rmag_index): w, = np.where( (data['ext_combo'] == 0) & between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax) ) mn, std = eu.stat.sigma_clip(data['conc'][w]) minconc, maxconc = [mn - 4*std, mn + 6*std] ww, = np.where( between(data['conc'][w], minconc, maxconc) ) w = w[ww] return w
def _load_data(self): fname=files.get_cosmos_file() r50min,r50max=self.r50_sanity_range fmin,fmax=self.flux_sanity_range logger.info("reading cosmos file: %s" % fname) alldata=fitsio.read(fname) w,=numpy.where( between(alldata['hlr'][:,0], r50min, r50max) & between(alldata['flux'][:,0], fmin, fmax) ) logger.debug("kept %d/%d" % (w.size, alldata.size)) self.alldata=alldata[w]
def sample(self, n=None, sigma_factor=1): if n is not None: scalar = False else: n = 1 scalar = True rand = np.zeros(n) ngood = 0 nleft = n while nleft > 0: tmp = self.rng.normal( size=n, loc=self.mean, scale=self.sigma * sigma_factor, ) w, = np.where(between(tmp, self.bounds[0], self.bounds[1])) if w.size > 0: rand[ngood:ngood + w.size] = tmp ngood += w.size nleft -= w.size if scalar: rand = rand[0] return rand
def _preselect(self, data): """ sub-classes might make a pre-selection, e.g. of some flags """ w, = np.where( np.isfinite(data['mcal_g'][:, 0]) & np.isfinite(data['mcal_g_1p'][:, 0]) & np.isfinite(data['mcal_g_1m'][:, 0]) & np.isfinite(data['mcal_g_2p'][:, 0]) & np.isfinite(data['mcal_g_2m'][:, 0])) data = data[w] if self.args.preselect: print('pre-selecting') R11 = (data['mcal_g_1p'][:, 0] - data['mcal_g_1m'][:, 0]) / 0.02 R22 = (data['mcal_g_2p'][:, 1] - data['mcal_g_2m'][:, 1]) / 0.02 w, = where(between(R11, -7, 9) & between(R22, -7, 9)) data = data[w] return data
def trim_radec(ra, dec, des_region, get_index=False): """ trim the data to the region of intereste """ from esutil.numpy_util import between print("trimming to region:", des_region) conf = read_config("constants") reginfo = conf["regions"][des_region] ra_range = reginfo["ra_range"] dec_range = reginfo["dec_range"] w, = numpy.where(between(ra, ra_range[0], ra_range[1]) & between(dec, dec_range[0], dec_range[1])) if get_index: return w else: return ra[w], dec[w]
def __init__(self, *, fname, rng, max_inject_rmag): self.rng = rng data = fitsio.read(fname) gmr = data['psf_mag'][:, 0] - data['psf_mag'][:, 1] rmi = data['psf_mag'][:, 1] - data['psf_mag'][:, 2] imz = data['psf_mag'][:, 2] - data['psf_mag'][:, 3] w, = np.where( between(gmr, -1, 3) & between(rmi, -1, 3) & between(imz, -1, 2)) self.data = data[w] self.gmr = gmr[w] self.rmi = rmi[w] self.imz = imz[w] self.bright, = np.where(self.data['psf_mag'][:, 1] < 21) mrng = [staramp.MAGOFF, max_inject_rmag] self.generator = eu.random.Generator( lambda x: (x - staramp.MAGOFF)**staramp.SLOPE, xrange=mrng, nx=100, )
def get_logic(data, model, prior_name): from esutil.numpy_util import between pconf=files.read_prior_config(prior_name) #pprint(pconf) n=Namer(model) T=data[n('pars')][:,4] F=data[n('pars')][:,5] g=sqrt( data[n('pars')][:,2]**2 + data[n('pars')][:,3]**2 ) Tr = pconf['T_range'] Fr = pconf['F_range'] logic = ( between(T, Tr[0], Tr[1]) & between(F, Fr[0], Fr[1]) & (g < GMAX ) ) if n('fracdev') in data.dtype.names: fracdev = data[ n('fracdev') ] fracdev_err = data[ n('fracdev_err') ] fdr = pconf['fracdev_range'] logic = ( logic & between(fracdev, fdr[0], fdr[1]) & (fracdev != 0.0) & (fracdev != 1.0) & (fracdev_err < pconf['fracdev_err_max']) ) return logic
def plot_results(trials, **keys): """ Plot the points and histograms of trials from an MCMC chain. """ import biggles import esutil from esutil.numpy_util import where1, between npars = trials.shape[1] fontsize_min = keys.get("fontsize_min", 1) biggles.configure("default", "fontsize_min", fontsize_min) weights = keys.get("weights", None) nbin = keys.get("nbin", 35) names = keys.get("names", None) show = keys.get("show", True) nsigma = keys.get("nsigma", None) means, cov = extract_stats(trials, weights=weights) errs = sqrt(diag(cov)) plt = biggles.Table(npars, 2) ind = numpy.arange(trials.shape[0]) for i in xrange(npars): if names is not None: name = names[i] else: name = r"$p_{%d}$" % i use_trials = trials[:, i] use_ind = ind use_wts = weights if nsigma is not None: w = where1(between(trials[:, i], means[i] - nsigma * errs[i], means[i] + nsigma * errs[i])) use_trials = trials[w, i] use_ind = ind[w] if weights is not None: use_wts = weights[w] burn_plot_i = biggles.plot(use_ind, use_trials, type="solid", xlabel="step", ylabel=name, visible=False) plt[i, 0] = burn_plot_i hcurve, bin_edges, harray = biggles.make_histc(use_trials, nbin=nbin, weights=use_wts, get_hdata=True) plti = biggles.FramedPlot() plti.xlabel = name hmax = harray.max() plti.yrange = [-0.05 * hmax, 1.2 * hmax] plti.add(hcurve) lab = r"$<%s> = %0.4g \pm %0.4g$" % (name, means[i], errs[i]) plab = biggles.PlotLabel(0.1, 0.8, lab, halign="left", color="blue") plti.add(plab) plt[i, 1] = plti plt.title = keys.get("title", None) if show: plt.show() return plt
def get_amp(*, rmag, conc, show=False, output=None, get_plot=False): """ get the amplitude of the stellar locus assuming a power law distribution A (mag - 13.5)**1.5 The data are binned and clipped, and sent to the calculate_amp function. Parameters ---------- rmag: array psf magnitude, should be r band conc: array concentration parameters for stars show: bool If True, show a plot output: string, optional If sent, write a plot file with that name Returns ------- amp, amp_err: float, float Amplitude and uncertainty """ w, = np.where(between(rmag, MAGMIN, MAGMAX) & between(conc, CMIN, CMAX)) hd = eu.stat.histogram( rmag[w], min=MAGMIN, max=MAGMAX, nbin=NBIN, more=True, ) herr = np.sqrt(hd['hist']) amp, amp_err = calculate_amp(rmag=hd['center'], conc=hd['hist']) print('amp: %g +/- %g' % (amp, amp_err)) if show or output is not None or get_plot: import hickory # alpha = 0.5 plt = hickory.Table(2, 1, figsize=(8, 7)) plt[0].plot( rmag, conc, marker='.', alpha=0.5, ) plt[0].plot( rmag[w], conc[w], marker='.', markeredgecolor='black', alpha=0.5, ) plt[0].set(xlim=(16, 20), ylim=(-0.002, 0.005)) plt[0].set( xlabel='psf mag r', ylabel='conc', ) predicted = predict(rmag=hd['center'], amp=amp) plt[1].errorbar( hd['center'], hd['hist'], yerr=herr, ) plt[1].curve(hd['center'], predicted) xvals = np.linspace(13.9, 21) plt[1].curve( xvals, predict(rmag=xvals, amp=amp), ) plt[1].set( xlabel='psf mag r', ylabel='Number', xlim=(13.5, 21), ) if show: plt.show() if output is not None: print('writing:', output) plt.savefig(output) if get_plot: return amp, amp_err, plt else: return amp, amp_err
def fit_exp(*, rmag, conc, show=False, output=None): """ get the amplitude of the stellar locus assuming a power law distribution A (mag - 13.5)**1.5 The data are binned and clipped, and sent to the calculate_amp function. Parameters ---------- rmag: array psf magnitude, should be r band conc: array concentration parameters for stars show: bool If True, show a plot output: string, optional If sent, write a plot file with that name Returns ------- amp, amp_err: float, float Amplitude and uncertainty """ w, = np.where(between(rmag, MAGMIN, MAGMAX) & between(conc, CMIN, CMAX)) hd = eu.stat.histogram( rmag[w], min=MAGMIN, max=MAGMAX, nbin=NBIN, more=True, ) num = hd['hist'] num_err = np.sqrt(num) guess = [1000, 18, 4, num[:4].mean()] res = fit_exp_binned(hd['center'], num, num_err, guess) print_pars(res['pars'], front='pars: ') print_pars(res['pars_err'], front='pars: ') if res['flags'] != 0: raise RuntimeError('fit failed') if show or output is not None: import hickory # alpha = 0.5 plt = hickory.Table(2, 1, figsize=(8, 7)) plt[0].plot( rmag, conc, marker='.', alpha=0.5, ) plt[0].plot( rmag[w], conc[w], marker='.', markeredgecolor='black', alpha=0.5, ) plt[0].set( xlim=(MAGMIN - 0.5, MAGMAX + 0.5), ylim=(-0.0005, CMAX), ) plt[0].set( xlabel='psf mag r', ylabel='conc', ) plt[1].errorbar( hd['center'], num, yerr=num_err, ) predicted = exp_func(res['pars'], hd['center']) eu.misc.colprint(num, predicted, num_err, format='%20s', names=['num', 'pred', 'num_err']) plt[1].curve(hd['center'], predicted) xvals = np.linspace(13.9, MAGMAX + 0.5) plt[1].curve( xvals, exp_func(res['pars'], xvals), ) plt[1].set( xlabel='psf mag r', ylabel='Number', xlim=(MAGMIN - 0.5, MAGMAX + 0.5), yscale='log', # xlim=(13.5, MAGMAX+0.5), ) if show: plt.show() if output is not None: print('writing:', output) plt.savefig(output) return res['pars']
def fit_conc_pdf( *, data, prior_file, rmag_index, seed, output, training, show=False, ): rng = np.random.RandomState(seed) print('will write to:', output) pdf_file = replace_ext(output, '.npy', '-plots.pdf') print('writing to pdf file:', pdf_file) pdf = PdfPages(pdf_file) data = data[data['flags'] == 0] prior_data = fitsio.read(prior_file) edges = [ (16.5, 17.0), (17.0, 17.5), (17.5, 18.0), (18.0, 18.5), (18.5, 19.0), (19, 19.5), (19.5, 20.0), (20, 20.5), (20.5, 21.0), (21, 21.5), (21.5, 22.0), (22, 22.5), (22.5, 23), (23, 23.5), (23.5, 24.0), (24, 24.5), ] outdata = make_output(len(edges)) rmag_centers = np.array( [0.5 * (rmagmin + rmagmax) for rmagmin, rmagmax in edges]) # initial estimate of N(mag) for stars initial_amp, initial_amp_err, plt = staramp.get_amp( rmag=data['psf_mag'][:, rmag_index], conc=data['conc'], show=show, get_plot=True, ) pdf.savefig(figure=plt) init_nstar, init_nstar_err = staramp.predict( rmag=rmag_centers, amp=initial_amp, amp_err=initial_amp_err, ) # gal_num_pars = galamp.fit_exp( # rmag=data['psf_mag'][:, rmag_index], # conc=data['conc'], # show=show, # ) # # init_ngal = galamp.exp_func(gal_num_pars, rmag_centers) # init_ngal_err = np.sqrt(init_ngal) # init_ngal_err *= 10 # init_nstar_err *= 10 ngal_list = np.zeros(len(edges)) for i in range(len(edges)): rmagmin, rmagmax = edges[i] rmag = 0.5 * (rmagmin + rmagmax) label = r'$%.2f < r < %.2f$' % (rmagmin, rmagmax) print('-' * 70) print(label) minconc, maxconc = -0.01, 0.025 w, = np.where( between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax) & between(data['conc'], minconc, maxconc)) print('number in bin:', w.size) nstar_predicted = init_nstar[i] star_priors = get_star_priors( rng=rng, data=prior_data, rmag=rmag, nstar=nstar_predicted, nstar_err=init_nstar_err[i], ) # if rmag < 18.0: # gal_ngauss = 1 # else: # gal_ngauss = 2 gal_ngauss = 2 ngal_predicted = w.size - nstar_predicted if ngal_predicted < 3: ngal_predicted = 3 gal_priors = get_gal_priors( rng=rng, data=prior_data, rmag=rmag, ngal=ngal_predicted, ngal_err=np.sqrt(ngal_predicted), ngauss=gal_ngauss, training=training, ) fitter = Fitter( data=data['conc'][w], star_priors=star_priors, gal_priors=gal_priors, rng=rng, ) fitter.go() res = fitter.result print('final') cem.gmix_print(fitter.gmix) print(res) assert res['flags'] == 0, 'failed to converge' nstar_meas = fitter.gmix['num'][:2].sum() ngal_meas = fitter.gmix['num'][2:].sum() nobj_meas = fitter.gmix['num'].sum() ngal_list[i] = ngal_meas print('true num: %g meas num: %g' % (w.size, nobj_meas)) print('nstar pred: %g nstar meas: %g' % (nstar_predicted, nstar_meas)) print('ngal pred: %g ngal meas: %g' % (ngal_predicted, ngal_meas)) if rmag > 23.5: plt = fitter.plot(title=label, legend=True, show=show) else: plt = fitter.plot3(label=label, show=show) pdf.savefig(figure=plt) outdata['rmagmin'][i] = rmagmin outdata['rmagmax'][i] = rmagmax outdata['rmag'][i] = rmag outdata['gmix'][i] = fitter.gmix plt = plot_fits_vs_rmag(outdata, 'star', show=show) pdf.savefig(plt) plt = plot_fits_vs_rmag(outdata, 'gal', show=show) pdf.savefig(plt) print('closing pdf file:', pdf_file) pdf.close() print('writing:', output) np.save(output, outdata)
def _run_sep(self): import sep # THRESH=1.2 # in sky sigma # DETECT_THRESH=1.6 # in sky sigma # DEBLEND_MINCONT=0.005 # DETECT_MINAREA = 6 # minimum number of pixels above threshold objs, seg = sep.extract( self.detim, self.detect_thresh, err=self.detnoise, segmentation_map=True, **self.sx_config ) logger.debug('found %d objects' % objs.size) if objs.size == 0: self.cat = objs return None flux_auto = np.zeros(objs.size)-9999.0 fluxerr_auto = np.zeros(objs.size)-9999.0 flux_radius = np.zeros(objs.size)-9999.0 kron_radius = np.zeros(objs.size)-9999.0 w, = np.where( (objs['a'] >= 0.0) & (objs['b'] >= 0.0) & between(objs['theta'], -pi/2., pi/2., type='[]') ) if w.size > 0: kron_radius[w], krflag = sep.kron_radius( self.detim, objs['x'][w], objs['y'][w], objs['a'][w], objs['b'][w], objs['theta'][w], 6.0, ) objs['flag'][w] |= krflag aper_rad = 2.5*kron_radius flux_auto[w], fluxerr_auto[w], flag_auto = \ sep.sum_ellipse( self.detim, objs['x'][w], objs['y'][w], objs['a'][w], objs['b'][w], objs['theta'][w], aper_rad[w], subpix=1, ) objs['flag'][w] |= flag_auto # what we did in DES, but note threshold above # is 1 as opposed to wide survey. deep survey # was even lower, 0.8? # used half light radius PHOT_FLUXFRAC = 0.5 flux_radius[w], frflag = sep.flux_radius( self.detim, objs['x'][w], objs['y'][w], 6.*objs['a'][w], PHOT_FLUXFRAC, normflux=flux_auto[w], subpix=5, ) objs['flag'][w] |= frflag # combine flags into 'flag' ncut = 2 # need this to make sure array new_dt = [ ('id', 'i8'), ('number', 'i4'), ('ncutout', 'i4'), ('kron_radius', 'f4'), ('flux_auto', 'f4'), ('fluxerr_auto', 'f4'), ('flux_radius', 'f4'), ('isoarea_image', 'f4'), ('iso_radius', 'f4'), ('box_size', 'i4'), ('file_id', 'i8', ncut), ('orig_row', 'f4', ncut), ('orig_col', 'f4', ncut), ('orig_start_row', 'i8', ncut), ('orig_start_col', 'i8', ncut), ('orig_end_row', 'i8', ncut), ('orig_end_col', 'i8', ncut), ('cutout_row', 'f4', ncut), ('cutout_col', 'f4', ncut), ('dudrow', 'f8', ncut), ('dudcol', 'f8', ncut), ('dvdrow', 'f8', ncut), ('dvdcol', 'f8', ncut), ] cat = eu.numpy_util.add_fields(objs, new_dt) cat['id'] = np.arange(cat.size) cat['number'] = np.arange(1, cat.size+1) cat['ncutout'] = 1 cat['flux_auto'] = flux_auto cat['fluxerr_auto'] = fluxerr_auto cat['flux_radius'] = flux_radius wcs = self.datalist[0]['wcs'] cat['dudrow'][:, 0] = wcs.dudy cat['dudcol'][:, 0] = wcs.dudx cat['dvdrow'][:, 0] = wcs.dvdy cat['dvdcol'][:, 0] = wcs.dvdx # use the number of pixels in the seg map as the iso area for i in range(objs.size): w = np.where(seg == (i+1)) cat['isoarea_image'][i] = w[0].size cat['iso_radius'] = np.sqrt(cat['isoarea_image'].clip(min=1)/np.pi) box_size = self._get_box_sizes(cat) half_box_size = box_size//2 maxrow, maxcol = self.detim.shape cat['box_size'] = box_size cat['orig_row'][:, 0] = cat['y'] cat['orig_col'][:, 0] = cat['x'] orow = cat['orig_row'][:, 0].astype('i4') ocol = cat['orig_col'][:, 0].astype('i4') ostart_row = orow - half_box_size + 1 ostart_col = ocol - half_box_size + 1 oend_row = orow + half_box_size + 1 # plus one for slices oend_col = ocol + half_box_size + 1 ostart_row.clip(min=0, out=ostart_row) ostart_col.clip(min=0, out=ostart_col) oend_row.clip(max=maxrow, out=oend_row) oend_col.clip(max=maxcol, out=oend_col) # could result in smaller than box_size above cat['orig_start_row'][:, 0] = ostart_row cat['orig_start_col'][:, 0] = ostart_col cat['orig_end_row'][:, 0] = oend_row cat['orig_end_col'][:, 0] = oend_col cat['cutout_row'][:, 0] = \ cat['orig_row'][:, 0] - cat['orig_start_row'][:, 0] cat['cutout_col'][:, 0] = \ cat['orig_col'][:, 0] - cat['orig_start_col'][:, 0] self.seg = seg self.bmask = np.zeros(seg.shape, dtype='i4') self.cat = cat
def select_good(data, g_field='g', min_T=None, T_range=None, flux_range=None, min_arate=None, max_arate=None, min_s2n=None, min_s2n_r=None, min_Ts2n=None, min_Ts2n_r=None, min_efficiency=None, max_chi2per=None, s2n_r_range=None, mcal_s2n_r_range=None, Ts2n_r_range=None, fracdev_err_max=None, cut_fracdev_exact=False, fracdev_range=None, min_Trat_r=None, max_g=1.0): """ apply standard selection """ logic = ones(data.size, dtype=bool) elogic = (data['flags']==0) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from flags" % (w.size, data.size)) logic = logic & elogic if g_field in data.dtype.names: gv = data[g_field] g = numpy.sqrt( gv[:,0]**2 + gv[:,1]**2 ) elogic = (g < max_g) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from %s < %g" % (w.size, data.size, g_field, max_g)) logic = logic & elogic if min_s2n is not None: elogic = (data['s2n_w'] > min_s2n) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from s2n > %g" % (w.size, data.size, min_s2n)) logic = logic & elogic if min_s2n_r is not None: if 's2n_r' in data.dtype.names: elogic = (data['s2n_r'] > min_s2n_r) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from s2n_r > %g" % (w.size, data.size, min_s2n_r)) logic = logic & elogic if s2n_r_range is not None: if 's2n_r' in data.dtype.names: rng=s2n_r_range elogic = (data['s2n_r'] > rng[0]) & (data['s2n_r'] < rng[1]) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from s2n_r in [%g,%g]" % (w.size, data.size, rng[0],rng[1])) logic = logic & elogic if mcal_s2n_r_range is not None: field='mcal_s2n_r' if field in data.dtype.names: rng=mcal_s2n_r_range elogic = between(data[field], rng[0], rng[1]) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from %s in [%g,%g]" % (w.size, data.size, field, rng[0],rng[1])) logic = logic & elogic if min_Ts2n is not None: elogic = (data['T_s2n'] > min_Ts2n) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n > %g" % (w.size, data.size, min_Ts2n)) logic = logic & elogic if min_Ts2n_r is not None: if 'T_s2n_r' in data.dtype.names: elogic = (data['T_s2n_r'] > min_Ts2n_r) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n_r > %g" % (w.size, data.size, min_Ts2n_r)) logic = logic & elogic if Ts2n_r_range is not None: if 'T_s2n_r' in data.dtype.names: rng=Ts2n_r_range elogic = (data['T_s2n_r'] > rng[0]) & (data['T_s2n_r'] < rng[1]) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n_r in [%g,%g]" % (w.size, data.size, rng[0],rng[1])) logic = logic & elogic if min_T is not None: elogic = (data['log_T'] > min_T) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from log_T > %g" % (w.size, data.size, min_T)) logic = logic & elogic if T_range is not None: elogic = (data['log_T'] > T_range[0]) & (data['log_T'] < T_range[1]) w,=where(elogic) if w.size != data.size: tup=(w.size, data.size, T_range[0], T_range[1]) print(" kept %d/%d from log_T in [%g,%g]" % tup) logic = logic & elogic if flux_range is not None: elogic = (data['log_flux'] > flux_range[0]) & (data['log_flux'] < flux_range[1]) w,=where(elogic) if w.size != data.size: tup=(w.size, data.size, flux_range[0], flux_range[1]) print(" kept %d/%d from log_flux in [%g,%g]" % tup) logic = logic & elogic if 'arate' in data.dtype.names: elogic = (data['arate'] > min_arate) & (data['arate'] < max_arate) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from arate [%g,%g]" % (w.size, data.size, min_arate,max_arate)) logic = logic & elogic if 'fracdev' in data.dtype.names: if fracdev_range is not None: elogic = ( (data['fracdev'] > fracdev_range[0]) & (data['fracdev'] < fracdev_range[1]) ) w,=where(elogic) if w.size != data.size: mess=" kept %d/%d from fracdev [%g,%g]" print(mess % (w.size, data.size, fracdev_range[0],fracdev_range[1])) logic = logic & elogic if fracdev_err_max is not None: elogic = (data['fracdev_err'] < fracdev_err_max) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from fracdev_err < %g" % (w.size, data.size, fracdev_err_max)) logic = logic & elogic if cut_fracdev_exact: elogic = (data['fracdev'] != 0.0) & (data['fracdev'] != 1.0) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from fracdev exact" % (w.size, data.size)) logic = logic & elogic if min_efficiency is not None: elogic = (data['efficiency'] > min_efficiency) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from eff > %g" % (w.size, data.size, min_efficiency)) logic = logic & elogic if max_chi2per is not None: elogic = (data['chi2per'] > max_chi2per) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from chi2per < %g" % (w.size, data.size, max_chi2per)) logic = logic & elogic if min_Trat_r is not None: Trat = data['Trat_r'] elogic = (Trat > min_Trat_r) w,=where(elogic) if w.size != data.size: print(" kept %d/%d from round Tg/Tpsf > %g" % (w.size, data.size, min_Trat_r)) logic = logic & elogic w,=where(logic) print(" kept %d/%d" % (w.size, data.size)) data=data[w] return data
def fit_conc_hists(*, data, prior_file, rmag_index, seed): rng = np.random.RandomState(seed) data = data[data['flags'] == 0] prior_data = fitsio.read(prior_file) edges = [ (16.5, 17.0), (17.0, 17.5), (17.5, 18.0), (18.0, 18.5), # (16, 18.5), (18.5, 19.0), (19, 19.5), (19.5, 20.0), (20, 20.5), (20.5, 21.0), (21, 21.5), (21.5, 22.0), (22, 22.5), (22.5, 23), (23, 23.5), (23.5, 24.0), (24, 24.5), ] # edges = [ # (23, 23.5), # (23.5, 24.0), # (24, 24.5), # ] # rmag_centers = np.array( [0.5 * (rmagmin + rmagmax) for rmagmin, rmagmax in edges]) # initial estimate of N(mag) for stars initial_amp, initial_amp_err = staramp.get_amp( rmag=data['psf_mag'][:, rmag_index], conc=data['conc'], show=True, ) gal_num_pars = galamp.fit_exp( rmag=data['psf_mag'][:, rmag_index], conc=data['conc'], show=True, ) init_nstar, init_nstar_err = staramp.predict( rmag=rmag_centers, amp=initial_amp, amp_err=initial_amp_err, ) init_ngal = galamp.exp_func(gal_num_pars, rmag_centers) # init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5) init_ngal_err = np.sqrt(init_ngal) # init_ngal_err *= 10 # init_nstar_err *= 10 ngal_list = np.zeros(len(edges)) for i in range(len(edges)): rmagmin, rmagmax = edges[i] rmag = 0.5 * (rmagmin + rmagmax) label = 'rmag: [%.2f, %.2f]' % (rmagmin, rmagmax) binsize = 0.00005 binsize_coarse = 0.0004 binsize_coarser = 0.0004 * 2 # off = 0.007 # power = 0.5 minconc, maxconc = -0.01, 0.025 # minconc, maxconc = -0.0005, 0.01 w, = np.where( between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax) & between(data['conc'], minconc, maxconc)) print('number in bin:', w.size) nstar_predicted = init_nstar[i] star_priors = get_star_priors( rng=rng, data=prior_data, rmag=rmag, nstar=nstar_predicted, nstar_err=init_nstar_err[i], ) if rmag < 18.0: gal_ngauss = 1 else: gal_ngauss = 2 # print_pars(ngal_list[0:i], front='ngal_list: ') # gal_num_res = galamp.fit_exp_binned( # rmag_centers[0:i], # ngal_list[0:i], # np.sqrt(ngal_list[0:i]), # gal_num_pars, # ) # print_pars(gal_num_pars, front='orig: ') # gal_num_pars = gal_num_res['pars'] # print_pars(gal_num_pars, front='new: ') # init_ngal = galamp.exp_func(gal_num_pars, rmag_centers) # # init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5) # # init_ngal_err = np.sqrt(init_ngal) # gal_ngauss = 2 ngal_predicted = w.size - nstar_predicted if ngal_predicted < 3: ngal_predicted = init_ngal[i] if ngal_predicted < 3: ngal_predicted = 3 gal_priors = get_gal_priors( rng=rng, data=prior_data, rmag=rmag, ngal=ngal_predicted, ngal_err=init_ngal_err[i], ngauss=gal_ngauss, ) hd = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize, more=True, ) plt = hickory.Plot(title=label, xlabel='concentration') plt.bar(hd['center'], hd['hist'], width=binsize) plt.show() hd_coarse = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize_coarse, more=True, ) hd_coarser = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize_coarser, more=True, ) x = hd['center'] y = hd['hist'] yerr = np.sqrt(hd['hist']) fitter = Fitter( x=x, y=y, yerr=yerr, star_priors=star_priors, gal_priors=gal_priors, rng=rng, ) fitter.go() res = fitter.result print_pars(res['pars'], front='pars: ') print_pars(res['pars_err'], front='perr: ') pars = res['pars'] nstar_meas = pars[2] + pars[5] if gal_ngauss == 1: nobj_meas = pars[2] + pars[5] + pars[8] ngal_meas = res['pars'][-1] else: nobj_meas = pars[2] + pars[5] + pars[8] + pars[11] ngal_meas = res['pars'][-1] + res['pars'][-4] ngal_list[i] = ngal_meas print('true num: %g meas num: %g' % (w.size, nobj_meas)) print('nstar pred: %g nstar meas: %g' % (nstar_predicted, nstar_meas)) print('ngal pred: %g ngal meas: %g' % (ngal_predicted, ngal_meas)) if res['flags'] == 0: fitter.plot( figsize=(10, 7.5), x=hd_coarse['center'], y=hd_coarse['hist'], xlabel='concentration', legend=True, show=True, title=label, xlim=(-0.005, 0.025), ) splt = fitter.plot( figsize=(10, 7.5), legend=True, show=False, title=label, xlim=(-0.003, 0.003), ) splt.show() cmin, cmax = 0.003, 0.025 w, = np.where(between(hd_coarser['center'], cmin, cmax)) ylim = [0, 1.1 * hd_coarser['hist'][w].max()] fitter.plot( figsize=(10, 7.5), x=hd_coarser['center'], y=hd_coarser['hist'], xlabel='concentration', legend=True, show=True, title=label, xlim=(0, cmax), ylim=ylim, )
def select_good(data, g_field='g', min_T=None, T_range=None, flux_range=None, min_arate=None, max_arate=None, min_s2n=None, min_s2n_r=None, min_Ts2n=None, min_Ts2n_r=None, min_efficiency=None, max_chi2per=None, s2n_r_range=None, mcal_s2n_r_range=None, Ts2n_r_range=None, fracdev_err_max=None, cut_fracdev_exact=False, fracdev_range=None, min_Trat_r=None, max_g=1.0): """ apply standard selection """ logic = ones(data.size, dtype=bool) elogic = (data['flags'] == 0) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from flags" % (w.size, data.size)) logic = logic & elogic if g_field in data.dtype.names: gv = data[g_field] g = numpy.sqrt(gv[:, 0]**2 + gv[:, 1]**2) elogic = (g < max_g) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from %s < %g" % (w.size, data.size, g_field, max_g)) logic = logic & elogic if min_s2n is not None: elogic = (data['s2n_w'] > min_s2n) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from s2n > %g" % (w.size, data.size, min_s2n)) logic = logic & elogic if min_s2n_r is not None: if 's2n_r' in data.dtype.names: elogic = (data['s2n_r'] > min_s2n_r) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from s2n_r > %g" % (w.size, data.size, min_s2n_r)) logic = logic & elogic if s2n_r_range is not None: if 's2n_r' in data.dtype.names: rng = s2n_r_range elogic = (data['s2n_r'] > rng[0]) & (data['s2n_r'] < rng[1]) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from s2n_r in [%g,%g]" % (w.size, data.size, rng[0], rng[1])) logic = logic & elogic if mcal_s2n_r_range is not None: field = 'mcal_s2n_r' if field in data.dtype.names: rng = mcal_s2n_r_range elogic = between(data[field], rng[0], rng[1]) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from %s in [%g,%g]" % (w.size, data.size, field, rng[0], rng[1])) logic = logic & elogic if min_Ts2n is not None: elogic = (data['T_s2n'] > min_Ts2n) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n > %g" % (w.size, data.size, min_Ts2n)) logic = logic & elogic if min_Ts2n_r is not None: if 'T_s2n_r' in data.dtype.names: elogic = (data['T_s2n_r'] > min_Ts2n_r) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n_r > %g" % (w.size, data.size, min_Ts2n_r)) logic = logic & elogic if Ts2n_r_range is not None: if 'T_s2n_r' in data.dtype.names: rng = Ts2n_r_range elogic = (data['T_s2n_r'] > rng[0]) & (data['T_s2n_r'] < rng[1]) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from Ts2n_r in [%g,%g]" % (w.size, data.size, rng[0], rng[1])) logic = logic & elogic if min_T is not None: elogic = (data['log_T'] > min_T) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from log_T > %g" % (w.size, data.size, min_T)) logic = logic & elogic if T_range is not None: elogic = (data['log_T'] > T_range[0]) & (data['log_T'] < T_range[1]) w, = where(elogic) if w.size != data.size: tup = (w.size, data.size, T_range[0], T_range[1]) print(" kept %d/%d from log_T in [%g,%g]" % tup) logic = logic & elogic if flux_range is not None: elogic = (data['log_flux'] > flux_range[0]) & (data['log_flux'] < flux_range[1]) w, = where(elogic) if w.size != data.size: tup = (w.size, data.size, flux_range[0], flux_range[1]) print(" kept %d/%d from log_flux in [%g,%g]" % tup) logic = logic & elogic if 'arate' in data.dtype.names: elogic = (data['arate'] > min_arate) & (data['arate'] < max_arate) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from arate [%g,%g]" % (w.size, data.size, min_arate, max_arate)) logic = logic & elogic if 'fracdev' in data.dtype.names: if fracdev_range is not None: elogic = ((data['fracdev'] > fracdev_range[0]) & (data['fracdev'] < fracdev_range[1])) w, = where(elogic) if w.size != data.size: mess = " kept %d/%d from fracdev [%g,%g]" print(mess % (w.size, data.size, fracdev_range[0], fracdev_range[1])) logic = logic & elogic if fracdev_err_max is not None: elogic = (data['fracdev_err'] < fracdev_err_max) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from fracdev_err < %g" % (w.size, data.size, fracdev_err_max)) logic = logic & elogic if cut_fracdev_exact: elogic = (data['fracdev'] != 0.0) & (data['fracdev'] != 1.0) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from fracdev exact" % (w.size, data.size)) logic = logic & elogic if min_efficiency is not None: elogic = (data['efficiency'] > min_efficiency) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from eff > %g" % (w.size, data.size, min_efficiency)) logic = logic & elogic if max_chi2per is not None: elogic = (data['chi2per'] > max_chi2per) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from chi2per < %g" % (w.size, data.size, max_chi2per)) logic = logic & elogic if min_Trat_r is not None: Trat = data['Trat_r'] elogic = (Trat > min_Trat_r) w, = where(elogic) if w.size != data.size: print(" kept %d/%d from round Tg/Tpsf > %g" % (w.size, data.size, min_Trat_r)) logic = logic & elogic w, = where(logic) print(" kept %d/%d" % (w.size, data.size)) data = data[w] return data