def _fix_guess(self, guess, prior, ntry=4): """ just fix flux """ n=guess.shape[0] for j in xrange(n): for itry in xrange(ntry): try: lnp=prior.get_lnprob_scalar(guess[j,:]) if lnp <= LOWVAL: dosample=True else: dosample=False except GMixRangeError as err: dosample=True if dosample: print_pars(guess[j,:], front="bad guess:") if itry < ntry: tguess = prior.sample() guess[j, 5:] = tguess[5:] else: raise UtterFailure("could not find a good guess") else: break
def get_mof_model(self): obs, coords, dims = self._get_mof_obs() prior = self._get_mof_prior(coords) nobj = len(coords) mm = minimof.MOF(obs, "bdf", nobj, prior=prior) for i in range(2): guess = self._get_mof_guess(coords) ngmix.print_pars(guess, front=" guess:") mm.go(guess) res = mm.get_result() if res['flags'] == 0: break if res['flags'] == 0: ngmix.print_pars(res['pars'], front=" fit:") center_obs = mm.make_corrected_obs(0, recenter=True) center_obs.noise = obs.noise else: center_obs = None return center_obs, coords, dims
def _fix_guess(self, guess, prior, ntry=4): """ Fix a guess for out-of-bounds values according the the input prior Bad guesses are replaced by a sample from the prior """ #guess[:,2]=-9999 n=guess.shape[0] for j in xrange(n): for itry in xrange(ntry): try: lnp=prior.get_lnprob_scalar(guess[j,:]) if lnp <= LOWVAL: dosample=True else: dosample=False except GMixRangeError as err: dosample=True if dosample: print_pars(guess[j,:], front="bad guess:") if itry < ntry: guess[j,:] = prior.sample() else: raise UtterFailure("could not find a good guess") else: break
def _get_psf_obs_fake(self, obs, file_id, meta, row, col): """ psfex specific code here for psfex we need to add 0.5 to get an offset that is the same as used for the object """ import ngmix assert self['psf']['type']=='psfex',"only psfex for now" pmeta={} if self['dither_psfs']: rowget=row+0.5 colget=col+0.5 else: #print("not dithering psfs") rowget=int(row) colget=int(col) pim, pcen = self._get_psf_im(file_id, rowget, colget) psf_weight=np.zeros(pim.shape) + 1.0/0.001**2 ccen=(np.array(pim.shape)-1.0)/2.0 # for psfex we assume the jacobian is the same, not # quite right pjac = obs.jacobian.copy() #pjac.set_cen(row=pcen[0], col=pcen[1]) pjac.set_cen(row=ccen[0], col=ccen[1]) # now fake the image import ngmix fac=pjac.get_scale()**2 T = 4.0*fac gm = ngmix.GMixModel([0.0,0.0,0.00,0.0,T,1.0],"gauss") pim = gm.make_image(pim.shape, jacobian=pjac) noise = 0.0001 pim += self.rng.normal(scale=noise, size=pim.shape) psf_weight = psf_weight*0 + 1.0/noise**2 pmeta['offset_pixels']=dict( row_offset=pjac.row0-ccen[0], col_offset=pjac.col0-ccen[1], ) #print("psf offsets:",pmeta['offset_pixels']) psf_obs = ngmix.Observation( pim, weight=psf_weight, jacobian=pjac, meta=pmeta, ) Tguess=T fitter=ngmix.admom.run_admom(psf_obs, Tguess) res=fitter.get_result() ngmix.print_pars(res['e'],front=' best e: ') print(' T frac:',res['T']/T-1) return psf_obs
def _print_galaxy_result(self): res=self.gal_fitter.get_result() if 'pars' in res: print_pars(res['pars'], front=' gal_pars: ') if 'pars_err' in res: print_pars(res['pars_err'],front=' gal_perr: ')
def get_mof_model(self): mb, coords = self._get_mof_obs() prior = self._get_mof_prior(coords, len(mb)) nobj = len(coords) mm = minimof.MOF(mb, "bdf", nobj, prior=prior) print(mm) for i in range(2): guess = self._get_mof_guess(coords, mb) ngmix.print_pars(guess, front=" guess:") mm.go(guess) res = mm.get_result() if res['flags'] == 0: break if res['flags'] == 0: ngmix.print_pars(res['pars'], front=" fit:") center_obs = mm.make_corrected_obs(0, band=None, obsnum=None, recenter=True) for i in range(len(center_obs)): center_obs[i][0].noise = mb[i][0].noise if self.show: import images cim = center_obs.image s = cim.shape tim = np.zeros((s[0], 2 * s[1])) tim[:, 0:s[1]] = obs.image tim[:, s[1]:] = cim tim *= 1.0 / tim.max() tim = np.log10(tim - tim.min() + 1.0) images.view( #tim-tim.min(), tim, title='MOF', ) if 'q' == input('hit a key: (q to quit) '): stop else: center_obs = None return center_obs
def go(self, guess=None): if guess is None: guess=self._get_guess() else: guess=self._transform_guess(guess) ngmix.print_pars(guess,front=" guess: ") obs=ngmix.Observation(self.p, jacobian=self.jacobian) fitter=ngmix.fitting.LMSimple(obs, 'gauss') fitter.go(guess) res=fitter.get_result() ngmix.print_pars(res['pars'],front=" pars: ") self._add_shear_info(res) self._result=res self._fitter=fitter
def go(self, ntry=100): ngauss = self.ngauss # ysum = self.y.sum() npars_per = 3 for i in range(ntry): self.guess = np.zeros(npars_per * ngauss) for ip, prior in enumerate(self.priors): self.guess[ip] = prior.sample(sigma_factor=1) # self.guess[ip] = prior.sample(sigma_factor=0.1) print_pars(self.guess, front='guess: ') self.result = run_leastsq( self._errfunc, self.guess, self.n_prior_pars, bounds=self.bounds, maxfev=4000, ) if self.result['flags'] == 0: break
def testExp(self): """ test fitting and exponential """ import ngmix rng=self.rng print('\n') for noise in [0.001, 0.1, 1.0]: print('='*10) print('noise:',noise) mdict=self._get_obs_data('exp',noise) obs=mdict['obs'] obs.set_psf(mdict['psf_obs']) pars=mdict['pars'].copy() pars[0] += rng.uniform(low=-0.1,high=0.1) pars[1] += rng.uniform(low=-0.1,high=0.1) pars[2] += rng.uniform(low=-0.1,high=0.1) pars[3] += rng.uniform(low=-0.1,high=0.1) pars[4] *= (1.0 + rng.uniform(low=-0.1,high=0.1)) pars[5] *= (1.0 + rng.uniform(low=-0.1,high=0.1)) max_pars={'method':'lm', 'lm_pars':{'maxfev':4000}} prior=ngmix.joint_prior.make_uniform_simple_sep( [0.0,0.0], # cen [0.1,0.1], # g [-10.0,3500.], # T [-0.97,1.0e9], # flux ) boot=ngmix.bootstrap.Bootstrapper(obs) boot.fit_psfs('gauss', 4.0) boot.fit_max('exp', max_pars, pars, prior=prior) res=boot.get_max_fitter().get_result() ngmix.print_pars(mdict['pars'], front='pars true: ') ngmix.print_pars(res['pars'], front='pars meas: ') ngmix.print_pars(res['pars_err'], front='pars err: ') print('s2n:',res['s2n_w'])
def fit_conc_hists(*, data, prior_file, rmag_index, seed): rng = np.random.RandomState(seed) data = data[data['flags'] == 0] prior_data = fitsio.read(prior_file) edges = [ (16.5, 17.0), (17.0, 17.5), (17.5, 18.0), (18.0, 18.5), # (16, 18.5), (18.5, 19.0), (19, 19.5), (19.5, 20.0), (20, 20.5), (20.5, 21.0), (21, 21.5), (21.5, 22.0), (22, 22.5), (22.5, 23), (23, 23.5), (23.5, 24.0), (24, 24.5), ] # edges = [ # (23, 23.5), # (23.5, 24.0), # (24, 24.5), # ] # rmag_centers = np.array( [0.5 * (rmagmin + rmagmax) for rmagmin, rmagmax in edges]) # initial estimate of N(mag) for stars initial_amp, initial_amp_err = staramp.get_amp( rmag=data['psf_mag'][:, rmag_index], conc=data['conc'], show=True, ) gal_num_pars = galamp.fit_exp( rmag=data['psf_mag'][:, rmag_index], conc=data['conc'], show=True, ) init_nstar, init_nstar_err = staramp.predict( rmag=rmag_centers, amp=initial_amp, amp_err=initial_amp_err, ) init_ngal = galamp.exp_func(gal_num_pars, rmag_centers) # init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5) init_ngal_err = np.sqrt(init_ngal) # init_ngal_err *= 10 # init_nstar_err *= 10 ngal_list = np.zeros(len(edges)) for i in range(len(edges)): rmagmin, rmagmax = edges[i] rmag = 0.5 * (rmagmin + rmagmax) label = 'rmag: [%.2f, %.2f]' % (rmagmin, rmagmax) binsize = 0.00005 binsize_coarse = 0.0004 binsize_coarser = 0.0004 * 2 # off = 0.007 # power = 0.5 minconc, maxconc = -0.01, 0.025 # minconc, maxconc = -0.0005, 0.01 w, = np.where( between(data['psf_mag'][:, rmag_index], rmagmin, rmagmax) & between(data['conc'], minconc, maxconc)) print('number in bin:', w.size) nstar_predicted = init_nstar[i] star_priors = get_star_priors( rng=rng, data=prior_data, rmag=rmag, nstar=nstar_predicted, nstar_err=init_nstar_err[i], ) if rmag < 18.0: gal_ngauss = 1 else: gal_ngauss = 2 # print_pars(ngal_list[0:i], front='ngal_list: ') # gal_num_res = galamp.fit_exp_binned( # rmag_centers[0:i], # ngal_list[0:i], # np.sqrt(ngal_list[0:i]), # gal_num_pars, # ) # print_pars(gal_num_pars, front='orig: ') # gal_num_pars = gal_num_res['pars'] # print_pars(gal_num_pars, front='new: ') # init_ngal = galamp.exp_func(gal_num_pars, rmag_centers) # # init_ngal[0] = galamp.exp_func(gal_num_pars, 18.5) # # init_ngal_err = np.sqrt(init_ngal) # gal_ngauss = 2 ngal_predicted = w.size - nstar_predicted if ngal_predicted < 3: ngal_predicted = init_ngal[i] if ngal_predicted < 3: ngal_predicted = 3 gal_priors = get_gal_priors( rng=rng, data=prior_data, rmag=rmag, ngal=ngal_predicted, ngal_err=init_ngal_err[i], ngauss=gal_ngauss, ) hd = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize, more=True, ) plt = hickory.Plot(title=label, xlabel='concentration') plt.bar(hd['center'], hd['hist'], width=binsize) plt.show() hd_coarse = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize_coarse, more=True, ) hd_coarser = eu.stat.histogram( data['conc'][w], min=minconc, max=maxconc, binsize=binsize_coarser, more=True, ) x = hd['center'] y = hd['hist'] yerr = np.sqrt(hd['hist']) fitter = Fitter( x=x, y=y, yerr=yerr, star_priors=star_priors, gal_priors=gal_priors, rng=rng, ) fitter.go() res = fitter.result print_pars(res['pars'], front='pars: ') print_pars(res['pars_err'], front='perr: ') pars = res['pars'] nstar_meas = pars[2] + pars[5] if gal_ngauss == 1: nobj_meas = pars[2] + pars[5] + pars[8] ngal_meas = res['pars'][-1] else: nobj_meas = pars[2] + pars[5] + pars[8] + pars[11] ngal_meas = res['pars'][-1] + res['pars'][-4] ngal_list[i] = ngal_meas print('true num: %g meas num: %g' % (w.size, nobj_meas)) print('nstar pred: %g nstar meas: %g' % (nstar_predicted, nstar_meas)) print('ngal pred: %g ngal meas: %g' % (ngal_predicted, ngal_meas)) if res['flags'] == 0: fitter.plot( figsize=(10, 7.5), x=hd_coarse['center'], y=hd_coarse['hist'], xlabel='concentration', legend=True, show=True, title=label, xlim=(-0.005, 0.025), ) splt = fitter.plot( figsize=(10, 7.5), legend=True, show=False, title=label, xlim=(-0.003, 0.003), ) splt.show() cmin, cmax = 0.003, 0.025 w, = np.where(between(hd_coarser['center'], cmin, cmax)) ylim = [0, 1.1 * hd_coarser['hist'][w].max()] fitter.plot( figsize=(10, 7.5), x=hd_coarser['center'], y=hd_coarser['hist'], xlabel='concentration', legend=True, show=True, title=label, xlim=(0, cmax), ylim=ylim, )
def _test_nobj(nobj, seed, show=False): """ simulate one object """ detband = 0 conf = _get_conf(nobj) sim = Sim(conf, seed) sim.make_obs() if show: import images images.view(sim.obs[0][0].image) if raw_input('hit a key: (q to quit): ') == 'q': raise KeyboardInterrupt('stopping') fitrng = np.random.RandomState(sim.rng.randint(0, 2**15)) # this runs sextractor medser = sim.get_medsifier() m2 = medser.get_meds(detband) objects = m2.get_cat() nobj = len(objects) if nobj == 0: print('found no objects') return m = medser.get_multiband_meds() list_of_obs = [] for iobj in range(objects.size): mbo = m.get_mbobs(iobj, weight_type='weight') for olist in mbo: for o in olist: o.set_psf(sim.psf_obs) list_of_obs.append(mbo) # first do a fit prior = get_mof_stamps_prior( list_of_obs, conf['fit_model'], fitrng, ) fitter = MOFStamps( list_of_obs, conf['fit_model'], prior=prior, lm_pars=LM_PARS, ) guess = get_stamp_guesses( list_of_obs, detband, conf['fit_model'], fitrng, ) for itry in range(2): fitter.go(guess) res = fitter.get_result() if res['flags'] == 0: break if res['flags'] != 0: raise RuntimeError('didnt find a fit') print('-' * 70) print('best fit pars for all objects') print('object pars') for i in range(nobj): ores = fitter.get_object_result(i) s2n = ores['s2n'] ngmix.print_pars(ores['pars'], front='%d s2n: %.1f pars: ' % (i + 1, s2n)) ffitter = MOFFlux(list_of_obs, conf['fit_model'], res['pars']) ffitter.go() fres = ffitter.get_result() if np.any(fres['flags'] != 0): print('didnt find a linear fit') return np.zeros(0), np.zeros(0) print('-' * 70) print('flux comparison') print('object flux flux_err fflux fflux_err') fluxes = np.zeros(nobj) flux_errs = np.zeros(nobj) for i in range(nobj): ores = fitter.get_object_result(i) fres = ffitter.get_object_result(i) print( 'object:', i + 1, 'flux:', ores['flux'][0], 'flux_err:', ores['flux_err'][0], 'fflux:', fres['flux'][0], 'fflux_err:', fres['flux_err'][0], ) fluxes[i] = fres['flux'][0] flux_errs[i] = fres['flux_err'][0] return fluxes, flux_errs
def plot_all(*, struct, type, show=False, output=None): if type == 'star': label = 'stars' else: label = 'galaxies' tab = hickory.Table( nrows=2, ncols=2, ) tab[1, 1].axis('off') tab[0, 0].ntext(0.1, 0.5, label, verticalalignment='center') tab[0, 0].set( xlabel='r mag', ylabel='weight', ) tab[0, 1].set( xlabel='r mag', ylabel='mean', ) tab[1, 0].set( xlabel='r mag', ylabel=r'$\sigma$', ) centers = struct['rmag_centers'] ngauss = struct['%s_weights' % type].shape[1] for igauss in range(ngauss): weights = struct['%s_weights' % type][:, igauss] means = struct['%s_means' % type][:, igauss] sigmas = struct['%s_sigmas' % type][:, igauss] wmean = weights.mean() print(type, igauss, wmean) tab[0, 0].axhline(wmean, color='black') tab[0, 0].curve(centers, weights, marker='o', markersize=2) if type == 'gal': poly = np.poly1d(np.polyfit(centers, means, 2)) print(poly) tab[0, 1].curve(centers, poly(centers), color='black') else: from .fitting import fit_exp, exp_func if igauss == 0: guess = [-1.0e-6, 16, 1] else: guess = [+1.0e-6, 16, 1] res = fit_exp(centers, means, guess) assert res['flags'] == 0 print_pars( res['pars'], front='star mean exp pars: ', fmt='%.6g', ) tab[0, 1].curve(centers, exp_func(res['pars'], centers), color='black') tab[0, 1].curve(centers, means, marker='o', markersize=1.5) tab[1, 0].curve(centers, sigmas, marker='o', markersize=1.5) if show: tab.show() if output is not None: print('writing:', output) tab.savefig(output, dpi=100)
def get_mof_ngal_model(self): sim, nobj, nfail, objects, coord_list, fit_model, noise, dims = self.sim( ) ####Created simple simulation for testing scale = 0.263 psf = galsim.Gaussian(fwhm=0.9) psf_gsim = psf.drawImage(nx=25, ny=25, scale=scale) psf_im = psf_gsim.array psf_err = 0.0001 noise_psf = np.random.normal(scale=psf_err, size=psf_im.shape) psf_im += noise_psf psf_weight = psf_im * 0 + 1.0 / psf_err**2 obj = galsim.Gaussian( half_light_radius=.5, flux=6000., ) dx1 = np.random.uniform(low=-0.5, high=0.5) dy1 = np.random.uniform(low=-0.5, high=0.5) obj = obj.shift(dx=dx1 * scale, dy=dy1 * scale) obj = galsim.Convolve(obj, psf) gsim = obj.drawImage( nx=dims[0], ny=dims[1], scale=scale, ) im = gsim.array err = 0.001 noise = np.random.normal(scale=err, size=im.shape) im += noise weight = im * 0 + 1.0 / err**2 psf_cen = (np.array(psf_im.shape) - 1.0) / 2.0 psf_jac = ngmix.DiagonalJacobian(scale=scale, row=psf_cen[0], col=psf_cen[1]) psf_obs = ngmix.Observation( psf_im, weight=psf_weight, jacobian=psf_jac, ) cen = (np.array(dims) - 1.0) / 2.0 coord1 = (dy1 + cen[1], dx1 + cen[0]) jac = ngmix.DiagonalJacobian(scale=scale, row=cen[0] + dy1, col=cen[1] + dx1) obs = ngmix.Observation( im, weight=weight, jacobian=jac, psf=psf_obs, ) noise = np.random.normal(scale=err, size=im.shape) obs.noise = noise #coord2 = (dy2+cen[1],dx2+cen[0]) coord_list = [coord1] # obs = observation(im,0.001,coord1[0],coord1[1],0.0001,psf_im) # obs.noise = noise #noise = np.random.normal(scale=0.001,size=(dims[0],dims[1])) return [obs], coord_list jac = sim.obs.jacobian jac.set_cen(row=objects['y'][0], col=objects['x'][0]) sim.obs.jacobian = jac return [sim.obs], coord_list ###Orginal start of get ngal model prior = self._get_mof_prior(coord_list, 1, sim.obs.jacobian, objects) fitter = minimof.MOF( sim.obs, fit_model, len(coord_list), prior=prior, ) nobs = [] if len(coord_list) != 0: for i in range(2): guess = self._get_mof_guess(coord_list, sim.obs, sim.obs.jacobian, objects) fitter.go(guess) res = fitter.get_result() if res['flags'] == 0: break if res['flags'] == 0: ngmix.print_pars(res['pars'], front=" fit:") for i in range(len(coord_list)): obs = fitter.make_corrected_obs(i, band=None, obsnum=None, recenter=True) obs[0][0].noise = sim.obs.noise nobs.append(obs) return nobs, coord_list
def test_mom(): from ngmix import print_pars from numpy import linspace, array e1=array([-0.2, -0.2, 0.2, 0.2, 0.1, 0.0, 0.1, 0.0]) e2=array([ 0.1, 0.0, 0.1, 0.0, -0.2, -0.2, 0.2, 0.2]) T=array([16.0]*e1.size) M1 = T*e1 M2 = T*e2 md=Deriv(M1, M2, T) print_pars(M1, front="M1: ") print_pars(M2, front="M2: ") print_pars(T, front="T: ") print_pars(e1, front="e1: ") print_pars(e2, front="e2: ") print() print_pars(md.dTds1z(), front="dTds1z: ") print_pars(md.dTds2z(), front="dTds2z: ") print_pars(md.dM1ds1z(), front="dM1ds1z: ") print_pars(md.dM1ds2z(), front="dM1ds2z: ") print_pars(md.dM2ds1z(), front="dM2ds1z: ") print_pars(md.dM2ds2z(), front="dM2ds2z: ") print() print_pars(md.d2Tds1ds1z(), front="d2Tds1ds1z: ") print_pars(md.d2Tds1ds2z(), front="d2Tds1ds2z: ") print_pars(md.d2Tds2ds2z(), front="d2Tds2ds2z: ") print() print_pars(md.d2M1ds1ds1z(), front="d2M1ds1ds1z:") print_pars(md.d2M1ds1ds2z(), front="d2M1ds1ds2z:") print_pars(md.d2M1ds2ds2z(), front="d2M1ds2ds2z:") print() print_pars(md.d2M2ds1ds1z(), front="d2M2ds1ds1z:") print_pars(md.d2M2ds1ds2z(), front="d2M2ds1ds2z:") print_pars(md.d2M2ds2ds2z(), front="d2M2ds2ds2z:")
def test_mom(): from ngmix import print_pars from numpy import linspace, array e1 = array([-0.2, -0.2, 0.2, 0.2, 0.1, 0.0, 0.1, 0.0]) e2 = array([0.1, 0.0, 0.1, 0.0, -0.2, -0.2, 0.2, 0.2]) T = array([16.0] * e1.size) M1 = T * e1 M2 = T * e2 md = Deriv(M1, M2, T) print_pars(M1, front="M1: ") print_pars(M2, front="M2: ") print_pars(T, front="T: ") print_pars(e1, front="e1: ") print_pars(e2, front="e2: ") print() print_pars(md.dTds1z(), front="dTds1z: ") print_pars(md.dTds2z(), front="dTds2z: ") print_pars(md.dM1ds1z(), front="dM1ds1z: ") print_pars(md.dM1ds2z(), front="dM1ds2z: ") print_pars(md.dM2ds1z(), front="dM2ds1z: ") print_pars(md.dM2ds2z(), front="dM2ds2z: ") print() print_pars(md.d2Tds1ds1z(), front="d2Tds1ds1z: ") print_pars(md.d2Tds1ds2z(), front="d2Tds1ds2z: ") print_pars(md.d2Tds2ds2z(), front="d2Tds2ds2z: ") print() print_pars(md.d2M1ds1ds1z(), front="d2M1ds1ds1z:") print_pars(md.d2M1ds1ds2z(), front="d2M1ds1ds2z:") print_pars(md.d2M1ds2ds2z(), front="d2M1ds2ds2z:") print() print_pars(md.d2M2ds1ds1z(), front="d2M2ds1ds1z:") print_pars(md.d2M2ds1ds2z(), front="d2M2ds1ds2z:") print_pars(md.d2M2ds2ds2z(), front="d2M2ds2ds2z:")