def test_metacal_fixnoise_smoke(otype, set_noise_image): rng = np.random.RandomState(seed=100) obs = _get_obs(rng, noise=0.005, set_noise_image=set_noise_image) if otype == 'obslist': oobs = obs obs = ngmix.ObsList() obs.append(oobs) check_type = ngmix.ObsList elif otype == 'mbobs': oobs = obs obslist = ngmix.ObsList() obslist.append(oobs) obs = ngmix.MultiBandObsList() obs.append(obslist) check_type = ngmix.MultiBandObsList else: check_type = ngmix.Observation resdict = ngmix.metacal.get_all_metacal( obs, rng=rng, use_noise_image=set_noise_image, ) assert isinstance(resdict['noshear'], check_type)
def _make_obs(self): mbobs = ngmix.MultiBandObsList() for im in self.imlist: jacobian = ngmix.DiagonalJacobian(row=0, col=0, scale=self['pixel_scale']) if self['coadd']: wt = np.zeros(im.shape) + 1.0 / self['coadd_noise_sigma']**2 else: wt = np.zeros(im.shape) + 1.0 / self['noise_sigma']**2 obs = ngmix.Observation( im, weight=wt, jacobian=jacobian, psf=self.get_psf_obs(), ) olist = ngmix.ObsList() olist.append(obs) mbobs.append(olist) self.obs = mbobs
def get_obslist(self, iobj, weight_type='weight'): """ get an ngmix ObsList for all observations parameters ---------- iobj: Index of the object weight_type: string, optional Weight type. can be one of 'weight': the actual weight map 'uberseg': uberseg modified weight map Default is 'weight' returns ------- an ngmix ObsList """ import ngmix obslist = ngmix.ObsList() for icut in range(self._cat['ncutout'][iobj]): obs = self.get_obs(iobj, icut, weight_type=weight_type) obslist.append(obs) obslist.meta['flux'] = obs.meta['flux'] obslist.meta['T'] = obs.meta['T'] return obslist
def zero_masked_weights(*, obslist): """ zero the weight image for pixels that have bad bits set, return a new ObsList Parameters ---------- obslist: ngmix.ObsList Observations to check Returns ------- new_obslist: ngmix.ObsList New version of obslist with weights set to zero. Those with no non-zero weight pixels are removed """ new_obslist = ngmix.ObsList() for obs in obslist: try: with obs.writeable(): zero_masked_weight( obs.bmask, obs.weight, FLAGVALS_TO_MASK, ) new_obslist.append(obs) except ngmix.GMixFatalError: pass return new_obslist
def _get_best_epochs(self, index, mbobs): """ just keep the best epoch if there are more than one this is good when using coadds and more than one epoch means overlap """ new_mbobs = ngmix.MultiBandObsList() new_mbobs.meta.update(mbobs.meta) for band, obslist in enumerate(mbobs): nepoch = len(obslist) if nepoch > 1: mess = ' obj %d band %d keeping best of %d epochs' logger.debug(mess % (index, band, nepoch)) wts = np.array([obs.weight.sum() for obs in obslist]) logger.debug(' weights: %s' % str(wts)) ibest = wts.argmax() keep_obs = obslist[ibest] new_obslist = ngmix.ObsList() new_obslist.meta.update(obslist.meta) new_obslist.append(keep_obs) else: new_obslist = obslist new_mbobs.append(new_obslist) return new_mbobs
def __call__(self): """ get a simulated ngmix.MultiBandObsList """ iconf = self['image'] band_images, band_weights, obj_data = self._get_noisy_images() jacobian = ngmix.DiagonalJacobian( row=0, col=0, scale=iconf['pixel_scale'], ) mbobs = ngmix.MultiBandObsList() for image, weight in zip(band_images, band_weights): obs = ngmix.Observation( image, weight=weight, jacobian=jacobian, psf=self.get_psf_obs(), ignore_zero_weight=False, ) obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) obj_data_with_cen = self._set_centers(obs, obj_data) mbobs.meta['obj_data'] = obj_data_with_cen return mbobs
def make_obs(self, add_noise=True): """ make a new MultiBandObsList, sampling from the catalog """ kw={} if 'nobj' in self: kw['nobj'] = self['nobj'] else: kw['area'] = self['area'] cat=self.catalog_sampler.sample(**kw) logger.debug('rendering: %d' % cat.size) # offsets in arcminutes pos_yx = self.position_sampler.sample(size=cat.size) mbobs = ngmix.MultiBandObsList() for band in self['bands']: dobs = self._make_band_dobs(cat, pos_yx, band, add_noise=add_noise) obs = self._convert_dobs_to_obs(dobs) #eu.stat.print_stats(obs.image.ravel()) obslist=ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) self.obs=mbobs self.cat=cat if 'background' in self: if self['background']['measure']: self._subtract_backgrounds()
def trim_obslist(*, obslist): """ trim the images to the default radius, returning a new MultiBandObsList Parameters ---------- obslist: ngmix.ObsList Observations to trim Returns ---------- new_obslist: ngmix.ObsList Trimmed observations """ new_obslist = ngmix.ObsList() for obs in obslist: try: new_obs = trim_obs(obs=obs) new_obslist.append(new_obs) except ngmix.GMixFatalError: pass return new_obslist
def check_blacklist(mbobs, blacklist): """ check the meta['file_path'] entry against the blacklist return a new mbobs without the blacklisted observations """ new_mbobs = ngmix.MultiBandObsList() new_mbobs.meta.update(mbobs.meta) for band, obslist in enumerate(mbobs): new_obslist = ngmix.ObsList() new_obslist.meta.update(obslist.meta) for epoch, obs in enumerate(obslist): file_path = obs.meta['file_path'] if file_path in blacklist: logger.debug('removing blacklisted obs from "%s"' % file_path) else: new_obslist.append(obs) if len(new_obslist) == 0: logger.debug('all epochs from band %s are blacklisted' % band) return None new_mbobs.append(new_obslist) return new_mbobs
def _cut_high_maskfrac(self, mbobs): new_mbobs = ngmix.MultiBandObsList() new_mbobs.meta.update(mbobs.meta) mf = self.config['max_maskfrac'] ok = True for band, obslist in enumerate(mbobs): new_obslist = ngmix.ObsList() new_obslist.meta.update(obslist.meta) for epoch, obs in enumerate(obslist): npix, nmasked = util.get_masked_frac_sums(obs) maskfrac = nmasked / npix if maskfrac < mf: new_obslist.append(obs) else: logger.info('cutting cutout band %d ' 'epoch %d for maskfrac: %g' % (band, epoch, maskfrac)) if len(new_obslist) == 0: logger.info('no cutouts left for band %d' % band) ok = False new_mbobs.append(obslist) return new_mbobs, ok
def _extract_mbobs(self, ranges, wout): """ extract an ngmix.MultiBandObsList for the given region, filling noise in the regions not assigned to the requested objects Parameters ---------- ranges: tuple (minrow, maxrow, mincol, maxcol) wout: tuple of arrays (wrows, wcols) indices for areas not assigned to the objects of interest Returns ------- mbobs: ngmix.MultiBandObsList """ minrow, maxrow, mincol, maxcol = ranges midrow = (minrow + maxrow)/2 midcol = (mincol + maxcol)/2 mbobs = ngmix.MultiBandObsList() for band in range(self.nband): image, weight, mask = self._get_image_data(band, ranges) jacob = self._get_jacobian(band, midrow, midcol) psf_obs = self._get_psf_obs(band, midrow, midcol) _replace_with_noise(image, weight, wout, self.rng) if self._rescale: scale = jacob.scale image *= 1.0/scale**2 weight *= scale**4 # zero the weight map for bad pixels. Note we are doing this after # replacing non-member objects pixels with noise, in case the # weight map ends up all zero if self._zero_weight_badpix is not None: _zero_weight_map_for_badpix( mask, weight, self._zero_weight_badpix, ) obs = ngmix.Observation( image, weight=weight, jacobian=jacob, psf=psf_obs, ignore_zero_weight=self._ignore_zero_weight, ) obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) return mbobs
def make_mbobs(*, band_data, rng): mbobs = ngmix.MultiBandObsList() for band, bdata in band_data.items(): obslist = ngmix.ObsList() for epoch_ind, se_obs in enumerate(bdata): ny, nx = se_obs.image.array.shape cenx = (nx - 1) / 2 ceny = (ny - 1) / 2 jacobian = get_jac(wcs=se_obs.wcs, cenx=cenx, ceny=ceny) psf_gsimage = se_obs.get_psf( cenx, ceny, center_psf=False, ) psf_image = psf_gsimage.array psf_cen = (np.array(psf_image.shape) - 1) / 2 psf_jac = jacobian.copy() psf_jac.set_cen(row=psf_cen[0], col=psf_cen[1]) psf_noise_fake = psf_image.max() / 50000 psf_image += rng.normal(scale=psf_noise_fake, size=psf_image.shape) psf_weight = psf_image * 0 + 1.0 / psf_noise_fake**2 psf_obs = ngmix.Observation( image=psf_image, weight=psf_weight, jacobian=psf_jac, ) obs = ngmix.Observation( image=se_obs.image.array, weight=se_obs.weight.array, jacobian=jacobian, psf=psf_obs, ) obslist.append(obs) mbobs.append(obslist) return mbobs
def zero_bitmask_in_weight(mbobs, flags2zero): """ check if the input flags are set in the bmask, if so zero the weight map """ new_mbobs = ngmix.MultiBandObsList() new_mbobs.meta.update(mbobs.meta) for band, obslist in enumerate(mbobs): new_obslist = ngmix.ObsList() new_obslist.meta.update(obslist.meta) for epoch, obs in enumerate(obslist): try: if obs.has_bmask(): bmask = obs.bmask w = np.where((bmask & flags2zero) != 0) if w[0].size > 0: with obs.writeable(): # update_pixels will be run on exiting the context # weight = obs.weight logging.debug('band %d epoch %d zeroing %d/%d in ' 'weight' % (band, epoch, w[0].size, bmask.size)) obs.weight[w] = 0.0 # trigger rebuild of pixels # obs.weight = weight new_obslist.append(obs) except ngmix.GMixFatalError: logging.info('band %d epoch %d all zero weight after ' 'bitmask' % (band, epoch)) if len(new_obslist) == 0: return None, procflags.HIGH_MASKFRAC new_mbobs.append(new_obslist) return new_mbobs, 0
def rewrap_obslist(obslist_in): """ rewrap an ObsList. When the obs list is sliced, it is converted to a regular list (python behavior). This rewraps to ObsList Parameters ---------- observation list: [Observation] List of Observations Returns ------- obslist: ngmix.ObsList converted list of Obs """ obslist = ngmix.ObsList() for obs in obslist_in: obslist.append(obs) return obslist
def redetect(mbobs, sx_config, detect_thresh): """ currently we take all detections, but should probably trim to some central region to avoid edges since taking all, will want to use big postage stamps """ import sep assert len(mbobs) == 1 assert len(mbobs[0]) == 1 obs = mbobs[0][0] noise = np.sqrt(1.0 / obs.weight[0, 0]) cat = sep.extract(obs.image, detect_thresh, err=noise, **sx_config) logger.debug(' redetect found %d' % cat.size) mbobs_list = [] if cat.size > 0: for i in range(cat.size): row = cat['y'][i] col = cat['x'][i] tobs = obs.copy() # makes a copy j = tobs.jacobian j.set_cen(row=row, col=col) tobs.jacobian = j tobslist = ngmix.ObsList() tobslist.append(tobs) tmbobs = ngmix.MultiBandObsList() tmbobs.append(tobslist) mbobs_list.append(tmbobs) return mbobs_list
def inject_star_into_obslist(*, rng, obslist, star_flux): """ inject a star into the observations Parameters ---------- rng: np.RandomState The random number generator obs: ngmix.ObsList Observations to be replaced with the injected star. Returns ------- new_obs_list: ngmix.ObsList The new observations """ new_obslist = ngmix.ObsList() for obs in obslist: new_obs = inject_star_into_obs(rng=rng, obs=obs, star_flux=star_flux) new_obslist.append(new_obs) return new_obslist
def make_sim(seed=42): scale = 0.25 flux = 10.0**(0.4 * (30 - 18)) noise = 10 ngals = 120 shape = 361 buff = 60 inner_shape = (shape - 2*buff) im_cen = (shape-1)/2 rng = np.random.RandomState(seed=seed) us = rng.uniform(low=-inner_shape/2, high=inner_shape/2, size=ngals) * scale vs = rng.uniform(low=-inner_shape/2, high=inner_shape/2, size=ngals) * scale wcs = galsim.PixelScale(scale) psf = galsim.Gaussian(fwhm=0.8) size_fac = rng.uniform(low=1.0, high=1.5, size=ngals) g1s = rng.normal(size=ngals) * 0.2 g2s = rng.normal(size=ngals) * 0.2 flux_fac = 10**rng.uniform(low=-2, high=0, size=ngals) # PSF image psf_img = psf.drawImage(nx=33, ny=33, wcs=wcs).array psf_cen = (psf_img.shape[0]-1)/2 psf_jac = ngmix.jacobian.DiagonalJacobian( row=psf_cen, col=psf_cen, scale=scale, ) target_s2n = 500.0 target_noise = np.sqrt(np.sum(psf_img ** 2)) / target_s2n psf_obs = ngmix.Observation( psf_img, weight=np.ones_like(psf_img)/target_noise**2, jacobian=psf_jac, ) # gals gals = [] for u, v, sf, g1, g2, ff in zip(us, vs, size_fac, g1s, g2s, flux_fac): gals.append(galsim.Convolve([ galsim.Exponential(half_light_radius=0.5 * sf), psf, ]).shear(g1=g1, g2=g2).withFlux(flux*ff).shift(u, v)) gals = galsim.Sum(gals) img = gals.drawImage(nx=shape, ny=shape, wcs=wcs).array img += rng.normal(size=img.shape) * noise nse = rng.normal(size=img.shape) * noise wgt = img*0 + 1.0/noise**2 msk = np.zeros(img.shape, dtype='i4') im_jac = ngmix.jacobian.DiagonalJacobian( row=im_cen, col=im_cen, scale=scale, ) obs = ngmix.Observation( img, weight=wgt, bmask=msk, ormask=msk.copy(), jacobian=im_jac, psf=psf_obs, noise=nse, ) mbobs = ngmix.MultiBandObsList() obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) return mbobs
def get_mbobs(self, return_truth_cat=False): """Make a simulated MultiBandObsList for metadetect. Parameters ---------- return_truth_cat : bool If True, return the truth catalog. Returns ------- mbobs : MultiBandObsList """ if self.gal_type == 'wldeblend': all_band_obj, positions, z_population, One_deg = self._get_band_objects() truth_cat = np.zeros(len(positions), dtype=[('x', 'f8'), ('y', 'f8'),('z', 'f8')]) truth_cat['z'] = One_deg else: all_band_obj, positions, z_population = self._get_band_objects() truth_cat = np.zeros(len(positions), dtype=[('x', 'f8'), ('y', 'f8'),('z_population', 'f8')]) truth_cat['z_population'] = z_population _, _, _, _, method = self._render_psf_image( x=self.im_cen, y=self.im_cen) mbobs = ngmix.MultiBandObsList() for band in range(self.n_bands): im = galsim.ImageD(nrow=self.dim, ncol=self.dim, xmin=0, ymin=0) band_objects = [o[band] for o in all_band_obj] for obj_ind, (obj, pos) in enumerate(zip(band_objects, positions)): truth_cat['x'][obj_ind] = pos.x truth_cat['y'][obj_ind] = pos.y # draw with setup_only to get the image size _im = obj.drawImage( wcs=self.wcs, method=method, setup_only=True).array assert _im.shape[0] == _im.shape[1] # now get location of the stamp x_ll = int(pos.x - (_im.shape[1] - 1)/2) y_ll = int(pos.y - (_im.shape[0] - 1)/2) # get the offset of the center dx = pos.x - (x_ll + (_im.shape[1] - 1)/2) dy = pos.y - (y_ll + (_im.shape[0] - 1)/2) dx *= self.scale dy *= self.scale # draw and set the proper origin stamp = obj.shift(dx=dx, dy=dy).drawImage( nx=_im.shape[1], ny=_im.shape[0], wcs=self.wcs, method=method) stamp.setOrigin(x_ll, y_ll) # intersect and add to total image overlap = stamp.bounds & im.bounds im[overlap] += stamp[overlap] im = im.array.copy() im += self.noise_rng.normal(scale=self.noise[band], size=im.shape) wt = im*0 + 1.0/self.noise[band]**2 bmask = np.zeros(im.shape, dtype='i4') noise = self.noise_rng.normal(size=im.shape) / np.sqrt(wt) if self.mask_and_interp: im, noise, bmask = self._mask_and_interp(im, noise) galsim_jac = self._get_local_jacobian(x=self.im_cen, y=self.im_cen) psf_obs = self.get_psf_obs( x=self.im_cen, y=self.im_cen, band=band) if self.homogenize_psf: im, noise, psf_img = self._homogenize_psf( im, noise, band) psf_obs.set_image(psf_img) jac = ngmix.jacobian.Jacobian( row=self.im_cen, col=self.im_cen, wcs=galsim_jac) obs = ngmix.Observation( im, weight=wt, bmask=bmask, ormask=bmask.copy(), jacobian=jac, psf=psf_obs, noise=noise) obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) if return_truth_cat: return mbobs, truth_cat else: return mbobs
def do_meta_mof_full_withsim_old(sim, fit_conf, fitter, show=False): """ not finding groups, just fitting everything. This means it won't work on bigger images with lots of empty space """ import mof assert fit_conf['fofs']['find_fofs'] == False tm0 = time.time() mofc = fit_conf['mof'] # create metacal versions of image metacal_pars = fit_conf['metacal']['metacal_pars'] if metacal_pars.get('symmetrize_psf', False): fitters._fit_all_psfs([sim.obs], fit_conf['mof']['psf']) # create the catalog based on original images # this will just run sx and create seg and # cat medser = sim.get_medsifier() cat = medser.cat mof_fitter, data = fitter.go( sim.obs, cat, ntry=mofc['ntry'], get_fitter=True, ) tobs = sim.obs[0][0] if show: gmix = mof_fitter.get_convolved_gmix() _plot_compare_model(gmix, tobs) sim_mbobs_after = ngmix.MultiBandObsList() sim_mbobs_before = ngmix.MultiBandObsList() for band, obslist in enumerate(sim.obs): sim_obslist_after = ngmix.ObsList() sim_obslist_before = ngmix.ObsList() for obsnum, obs in enumerate(obslist): gmix = mof_fitter.get_convolved_gmix( band=band, obsnum=obsnum, ) sobs_after = ngmix.simobs.simulate_obs(gmix, obs, add_noise=False) sobs_before = ngmix.simobs.simulate_obs(gmix, obs, add_noise=True) # get another noise field to be used in metacal fixnoise # also used for the after obs sobs_before2 = ngmix.simobs.simulate_obs(gmix, obs, add_noise=True) sobs_before.noise = sobs_before2.noise_image # meta data gets passed on by metacal, we can use the total # noise image later for the 'after' obs sobs_after.meta['total_noise'] = \ sobs_before.noise_image + sobs_before2.noise_image #import images #images.view(sobs_before.image) #stop sim_obslist_after.append(sobs_after) sim_obslist_before.append(sobs_before) sim_mbobs_after.append(sim_obslist_after) sim_mbobs_before.append(sim_obslist_before) odict = ngmix.metacal.get_all_metacal(sim.obs, **metacal_pars) after_metacal_pars = {} after_metacal_pars.update(metacal_pars) after_metacal_pars['fixnoise'] = False odict_after = ngmix.metacal.get_all_metacal(sim_mbobs_after, **after_metacal_pars) before_metacal_pars = {} before_metacal_pars.update(metacal_pars) before_metacal_pars['use_noise_image'] = True odict_before = ngmix.metacal.get_all_metacal(sim_mbobs_before, **before_metacal_pars) # now go and add noise after shearing for key in odict_after: mbobs = odict_after[key] for obslist in mbobs: for obs in obslist: #import images #images.view(obs.image) #stop # because with fixnoise we added extra noise obs.weight *= 0.5 obs.image += obs.meta['total_noise'] # now run fits on all these images allstuff = [ (None, odict), ('after', odict_after), ('before', odict_before), ] reslists = {} for name, todict in allstuff: n = util.Namer(front=name) for key, mbobs in todict.items(): data = fitter.go( mbobs, cat, ntry=mofc['ntry'], ) reslists[n(key)] = [data] nobj = cat.size tm_fit = time.time() - tm0 return reslists, nobj, tm_fit
def extract_single_obs(sim, mbobs_list): assert len(mbobs_list[0]) == 1, 'one band for now' box_size = max([max(m[0][0].image.shape) for m in mbobs_list]) half_box_size = box_size // 2 # get the mean position flux = np.array([m[0][0].meta['flux'] for m in mbobs_list]) rows = np.array([m[0][0].meta['orig_row'] for m in mbobs_list]) cols = np.array([m[0][0].meta['orig_col'] for m in mbobs_list]) flux_sum = flux.sum() if flux_sum == 0.0: logger.info('zero flux') return [] mean_row = (rows * flux).sum() / flux.sum() mean_col = (cols * flux).sum() / flux.sum() print('rows:', rows, 'row mean:', mean_row) print('cols:', cols, 'col mean:', mean_col) imax = flux.argmax() mbobs_imax = mbobs_list[imax] # again assuming single band obs = sim.obs[0][0].copy() #obs.meta.update(mbobs_list[0][0][0].meta) maxrow, maxcol = obs.image.shape # now get the adaptive moments center j = obs.jacobian j.set_cen(row=mean_row, col=mean_col) obs.jacobian = j shiftmax = 3.0 # arcsec Tguess = 4.0 * j.scale**2 runner = ngmix.bootstrap.EMRunner(obs, Tguess, 1, { 'maxiter': 1000, 'tol': 1.0e-4 }) runner.go(ntry=2) fitter = runner.fitter res = fitter.get_result() #try: # fitter=ngmix.admom.run_admom(obs, Tguess, shiftmax=shiftmax) # res=fitter.get_result() #except ngmix.GMixRangeError as err: # res={'flags':2} #if res['flags']==0: # break if res['flags'] != 0: print(" could not fit for center", res) return [] gm = fitter.get_gmix() v, u = gm.get_cen() newrow, newcol = j.get_rowcol(v, u) print("new cen:", newrow, newcol) start_row = int(newrow) - half_box_size + 1 start_col = int(newcol) - half_box_size + 1 end_row = int(newrow) + half_box_size + 1 # plus one for slices end_col = int(newcol) + half_box_size + 1 if start_row < 0: start_row = 0 if start_col < 0: start_col = 0 if start_row > maxrow: start_row = maxrow if start_col > maxcol: start_col = maxcol cutout_row = newrow - start_row cutout_col = newcol - start_col im = obs.image[start_row:end_row, start_col:end_col, ] wt = obs.weight[start_row:end_row, start_col:end_col, ] j = obs.jacobian j.set_cen(row=cutout_row, col=cutout_col) nobs = ngmix.Observation( im, weight=wt, jacobian=j, bmask=np.zeros(im.shape, dtype='i4'), meta=mbobs_list[0][0][0].meta, psf=obs.psf.copy(), ) mbobs = ngmix.MultiBandObsList() obslist = ngmix.ObsList() obslist.append(nobs) mbobs.append(obslist) return [mbobs]
def _trim_images(self, mbobs, index): """ trim the images down to a minimal size """ # hst_band can be None if we are only processing non-hst data hst_band = self.config['hst_band'] logger.debug('trimming') min_size = self.config['trim_images']['min_size'] max_size = self.config['trim_images']['max_size'] min_rad = min_size / 2.0 max_rad = max_size / 2.0 fwhm = 1.5 sigma = fwhm / 2.35 exrad = 3 * sigma new_mbobs = ngmix.MultiBandObsList() new_mbobs.meta.update(mbobs.meta) for band, obslist in enumerate(mbobs): m = self.mb_meds.mlist[band] rad = m['iso_radius_arcsec'][index] * 3.0 if band != hst_band: #rad = np.sqrt(rad**2 + 0.4**2) rad = np.sqrt(rad**2 + exrad**2) new_obslist = ngmix.ObsList() new_obslist.meta.update(obslist.meta) for obs in obslist: imshape = obs.image.shape if imshape[0] > min_size: meta = obs.meta jac = obs.jacobian cen = jac.get_cen() rowpix = int(round(cen[0])) colpix = int(round(cen[1])) scale = jac.scale radpix = rad / scale if radpix < min_rad: radpix = min_rad if radpix > max_rad: radpix = max_rad radpix = int(radpix) - 1 row_start = rowpix - radpix row_end = rowpix + radpix + 1 col_start = colpix - radpix col_end = colpix + radpix + 1 if row_start < 0: row_start = 0 if row_end > imshape[0]: row_end = imshape[0] if col_start < 0: col_start = 0 if col_end > imshape[1]: col_end = imshape[1] subim = obs.image[row_start:row_end, col_start:col_end, ] subwt = obs.weight[row_start:row_end, col_start:col_end, ] logger.debug('%s -> %s' % (str(obs.image.shape), str(subim.shape))) cen = (cen[0] - row_start, cen[1] - col_start) jac.set_cen(row=cen[0], col=cen[1]) meta['orig_start_row'] += row_start meta['orig_start_col'] += col_start new_obs = ngmix.Observation( subim, weight=subwt, jacobian=jac, meta=obs.meta, psf=obs.psf, ) else: new_obs = obs new_obslist.append(new_obs) new_mbobs.append(new_obslist) return new_mbobs
def do_meta_mof_full_withsim_old2(sim, fit_conf, fitter, show=False): """ not finding groups, just fitting everything. This means it won't work on bigger images with lots of empty space """ import mof assert fit_conf['fofs']['find_fofs'] == False tm0 = time.time() mofc = fit_conf['mof'] # create metacal versions of image metacal_pars = fit_conf['metacal']['metacal_pars'] if metacal_pars.get('symmetrize_psf', False): fitters._fit_all_psfs([sim.obs], fit_conf['mof']['psf']) # create the catalog based on original images # this will just run sx and create seg and # cat medser = sim.get_medsifier() cat = medser.cat mof_fitter, data = fitter.go( sim.obs, cat, ntry=mofc['ntry'], get_fitter=True, ) tobs = sim.obs[0][0] if show: gmix = mof_fitter.get_convolved_gmix() _plot_compare_model(gmix, tobs) sim_mbobs = ngmix.MultiBandObsList() for band, obslist in enumerate(sim.obs): sim_obslist = ngmix.ObsList() for obsnum, obs in enumerate(obslist): gmix = mof_fitter.get_convolved_gmix( band=band, obsnum=obsnum, ) psf_gmix = obs.psf.gmix psf_im = psf_gmix.make_image( obs.psf.image.shape, jacobian=obs.psf.jacobian, ) pn = psf_im.max() / 1000.0 psf_im += fitter.rng.normal(size=psf_im.shape, scale=pn) psf_wt = psf_im * 0 + 1.0 / pn**2 sobs = ngmix.simobs.simulate_obs(gmix, obs, add_noise=False) sobs.psf.image = psf_im sobs.psf.weight = psf_wt sobs_noisy = ngmix.simobs.simulate_obs(gmix, obs, add_noise=True, rng=fitter.rng) # this one for fixnoise sobs_noisy2 = ngmix.simobs.simulate_obs(gmix, obs, add_noise=True, rng=fitter.rng) # to be added after shearing sobs.meta['noise'] = sobs_noisy.noise_image sobs.meta['noise_for_fixnoise'] = sobs_noisy2.noise_image sim_obslist.append(sobs) sim_mbobs.append(sim_obslist) # for the measurement on real data odict = ngmix.metacal.get_all_metacal(sim.obs, rng=fitter.rng, **metacal_pars) # shear the noiseless sim sim_metacal_pars = {} sim_metacal_pars.update(metacal_pars) sim_metacal_pars['fixnoise'] = False sim_metacal_pars['types'] = ['1p', '1m'] sim_odict = ngmix.metacal.get_all_metacal(sim_mbobs, **sim_metacal_pars) # now add the noise; the noise will be the same # realization for each for key, mbobs in sim_odict.items(): for obslist in mbobs: for obs in obslist: #import images #images.view(obs.image) #stop obs.image += obs.meta['noise'] obs.noise = obs.meta['noise_for_fixnoise'] sim_metacal_pars2 = {} sim_metacal_pars2.update(metacal_pars) sim_metacal_pars2['use_noise_image'] = True sim_odict_1p = ngmix.metacal.get_all_metacal(sim_odict['1p'], **sim_metacal_pars2) sim_odict_1m = ngmix.metacal.get_all_metacal(sim_odict['1m'], **sim_metacal_pars2) reslists = {} reslists.update(_process_one_full_mof_metacal(mofc, odict, cat, fitter)) reslists.update( _process_one_full_mof_metacal(mofc, sim_odict_1p, cat, fitter, prefix='sim1p')) reslists.update( _process_one_full_mof_metacal(mofc, sim_odict_1m, cat, fitter, prefix='sim1m')) nobj = cat.size tm_fit = time.time() - tm0 return reslists, nobj, tm_fit
def make_mbobs_from_coadd_data( *, wcs, cdata, ): """Make an ngmix.MultiBandObsList from the coadd data. Parameters ---------- wcs : galsim.BaseWCS or similar The coadd WCS object representing the WCS of the final images. cdata : dict The dictionary of coadd data from `pizza_cutter_sims.pizza_cutter.run_des_pizza_cutter_coadding_on_sim` Returns ------- mbobs : ngmix.MultiBandObsList The coadd data in mbobs form. """ image = cdata["image"] bmask = cdata["bmask"] ormask = cdata["ormask"] noise = cdata["noise"] psf = cdata["psf"] weight = cdata["weight"] mfrac = cdata["mfrac"] psf_cen = (psf.shape[0] - 1)/2 im_cen = (image.shape[0] - 1)/2 # make the mbobs psf_jac = ngmix.jacobian.Jacobian( x=psf_cen, y=psf_cen, dudx=wcs.dudx, dudy=wcs.dudy, dvdx=wcs.dvdx, dvdy=wcs.dvdy, ) target_s2n = 500.0 target_noise = np.sqrt(np.sum(psf ** 2)) / target_s2n psf_obs = ngmix.Observation( psf.copy(), weight=np.ones_like(psf)/target_noise**2, jacobian=psf_jac, ) im_jac = ngmix.jacobian.Jacobian( x=im_cen, y=im_cen, dudx=wcs.dudx, dudy=wcs.dudy, dvdx=wcs.dvdx, dvdy=wcs.dvdy, ) obs = ngmix.Observation( image.copy(), weight=weight.copy(), bmask=bmask.copy(), ormask=ormask.copy(), jacobian=im_jac, psf=psf_obs, noise=noise.copy(), mfrac=np.clip(mfrac.copy(), 0, 1), ) mbobs = ngmix.MultiBandObsList() obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) return mbobs
def run_metacal(n_sims, stamp_size, psf_stamp_size, rng, jacobian_dict, gauss_psf): """Run metacal on an image composed of stamps w/ constant noise. Parameters ---------- n_sims : int The number of objects to run. stamp_size : int The size of each stamp. psf_stamp_size : int The size of each PSF stamp. rng : np.random.RandomState An RNG to use. jacobian_dict : dict A dictonary with the components of the image jacobian. gauss_psf : bool If True, test with a Gaussian PSF. Returns ------- result : dict A dictionary with each of the metacal catalogs. """ method = 'auto' cen = (stamp_size - 1) / 2 psf_cen = (psf_stamp_size - 1) / 2 noise = 1 flux = 64000 gal = galsim.Exponential(half_light_radius=0.5).withFlux(flux).shear( g1=0.02, g2=0.0) if not gauss_psf: piff_cats = np.loadtxt('piff_cat.txt', dtype=str) piff_file = rng.choice(piff_cats) psf_model = DES_Piff(piff_file) print('piff file:', piff_file) if jacobian_dict['dudy'] != 0 or jacobian_dict['dvdx'] != 0: print('jacobian:', jacobian_dict) galsim_jac = galsim.JacobianWCS(dudx=jacobian_dict['dudx'], dudy=jacobian_dict['dudy'], dvdx=jacobian_dict['dvdx'], dvdy=jacobian_dict['dvdy']) data = [] for ind in tqdm.trange(n_sims): ################################ # make the obs # first get PSF x = rng.uniform(low=1, high=2048) y = rng.uniform(low=1, high=2048) if gauss_psf: psf = galsim.Gaussian(fwhm=0.9).withFlux(1) else: psf = psf_model.getPSF(galsim.PositionD(x=x, y=y), galsim_jac) psf_im = psf.drawImage(nx=psf_stamp_size, ny=psf_stamp_size, wcs=galsim_jac, method=method).array psf_noise = np.sqrt(np.sum(psf_im**2)) / 500 wgt = 0.0 * psf_im + 1.0 / psf_noise**2 psf_jac = ngmix.Jacobian(x=psf_cen, y=psf_cen, dudx=jacobian_dict['dudx'], dudy=jacobian_dict['dudy'], dvdx=jacobian_dict['dvdx'], dvdy=jacobian_dict['dvdy']) psf_obs = ngmix.Observation(image=psf_im, weight=wgt, jacobian=psf_jac) # now render object obj = galsim.Convolve(gal, psf) offset = rng.uniform(low=-0.5, high=0.5, size=2) im = obj.drawImage(nx=stamp_size, ny=stamp_size, wcs=galsim_jac, offset=offset, method=method).array jac = ngmix.Jacobian(x=cen + offset[0], y=cen + offset[1], dudx=jacobian_dict['dudx'], dudy=jacobian_dict['dudy'], dvdx=jacobian_dict['dvdx'], dvdy=jacobian_dict['dvdy']) wgt = np.ones_like(im) / noise**2 nse = rng.normal(size=im.shape) * noise im += (rng.normal(size=im.shape) * noise) obs = ngmix.Observation(image=im, weight=wgt, noise=nse, bmask=np.zeros_like(im, dtype=np.int32), ormask=np.zeros_like(im, dtype=np.int32), jacobian=jac, psf=psf_obs) # build the mbobs mbobs = ngmix.MultiBandObsList() obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) mbobs.meta['id'] = ind + 1 # these settings do not matter that much I think mbobs[0].meta['Tsky'] = 1 mbobs[0].meta['magzp_ref'] = 26.5 mbobs[0][0].meta['orig_col'] = ind + 1 mbobs[0][0].meta['orig_row'] = ind + 1 ################################ # run the fitter mcal = MetacalFitter(CONFIG, 1, rng) mcal.go([mbobs]) res = mcal.result if res is not None: data.append(res) if len(data) > 0: res = eu.numpy_util.combine_arrlist(data) result = _result_to_dict(res) else: result = None return result
def do_meta_mof_full_withsim(sim, fit_conf, fitter, show=False): """ not finding groups, just fitting everything. This means it won't work on bigger images with lots of empty space """ import galsim import mof assert fit_conf['fofs']['find_fofs'] == False tm0 = time.time() mofc = fit_conf['mof'] # create metacal versions of image metacal_pars = fit_conf['metacal']['metacal_pars'] if metacal_pars.get('symmetrize_psf', False): fitters._fit_all_psfs([sim.obs], fit_conf['mof']['psf']) # create the catalog based on original images # this will just run sx and create seg and # cat medser = sim.get_medsifier() if medser.cat.size == 0: return {}, 0, 0.0 cat = medser.cat mof_fitter, data = fitter.go( sim.obs, cat, ntry=mofc['ntry'], get_fitter=True, ) tobs = sim.obs[0][0] if show: gmix = mof_fitter.get_convolved_gmix() _plot_compare_model(gmix, tobs) sim_mbobs_1p = ngmix.MultiBandObsList() sim_mbobs_1m = ngmix.MultiBandObsList() for band, obslist in enumerate(sim.obs): sim_obslist_1p = ngmix.ObsList() sim_obslist_1m = ngmix.ObsList() band_gmix0 = mof_fitter.get_gmix(band=band, ) theta = fitter.rng.uniform(low=0.0, high=np.pi) #gmix0 = gmix0.get_rotated(theta) for obsnum, obs in enumerate(obslist): jac = obs.jacobian scale = jac.scale gmix0 = band_gmix0.copy() gmix0.set_flux(gmix0.get_flux() / scale**2) # cheating on psf for now ny, nx = obs.image.shape # galsim does everything relative to the canonical center, but # for the mof fitter we had the origin a 0,0. Shift over by # the cen ccen = (np.array(obs.image.shape) - 1.0) / 2.0 gs0 = gmix0.make_galsim_object() gs0 = gs0.shift(dx=-ccen[1] * scale, dy=-ccen[1] * scale) if show and obsnum == 0: import images #gs = gmix0.make_galsim_object(psf=sim.psf) #gs = gs.shift(dx=-ccen[1]*scale, dy=-ccen[1]*scale) gs = galsim.Convolve(gs0, sim.psf) tim = gs.drawImage(nx=nx, ny=ny, scale=sim['pixel_scale']).array images.compare_images(sim.obs[0][0].image, tim) if 'q' == input('hit a key (q to quit): '): stop gs0_1p = gs0.shear(g1=0.01, g2=0.0) gs0_1m = gs0.shear(g1=-0.01, g2=0.0) gs_1p = galsim.Convolve(gs0_1p, sim.psf) gs_1m = galsim.Convolve(gs0_1m, sim.psf) im_1p = gs_1p.drawImage(nx=nx, ny=ny, scale=sim['pixel_scale']).array im_1m = gs_1m.drawImage(nx=nx, ny=ny, scale=sim['pixel_scale']).array # adding same noise to both noise_image = ngmix.simobs.get_noise_image(obs.weight, rng=fitter.rng) im_1p += noise_image im_1m += noise_image # this one will be used for fixnoise noise_image2 = ngmix.simobs.get_noise_image(obs.weight, rng=fitter.rng) sobs_1p = ngmix.Observation( im_1p, weight=obs.weight.copy(), jacobian=jac, psf=obs.psf.copy(), noise=noise_image2.copy(), ) sobs_1m = ngmix.Observation( im_1m, weight=obs.weight.copy(), jacobian=jac, psf=obs.psf.copy(), noise=noise_image2.copy(), ) sim_obslist_1p.append(sobs_1p) sim_obslist_1m.append(sobs_1m) sim_mbobs_1p.append(sim_obslist_1p) sim_mbobs_1m.append(sim_obslist_1m) # for the measurement on real data odict = ngmix.metacal.get_all_metacal(sim.obs, rng=fitter.rng, **metacal_pars) sim_metacal_pars = {} sim_metacal_pars.update(metacal_pars) sim_metacal_pars['use_noise_image'] = True sim_odict_1p = ngmix.metacal.get_all_metacal( sim_mbobs_1p, rng=fitter.rng, # not needed **sim_metacal_pars) sim_odict_1m = ngmix.metacal.get_all_metacal( sim_mbobs_1m, rng=fitter.rng, # not needed **sim_metacal_pars) reslists = {} reslists.update(_process_one_full_mof_metacal(mofc, odict, cat, fitter)) reslists.update( _process_one_full_mof_metacal(mofc, sim_odict_1p, cat, fitter, prefix='sim1p')) reslists.update( _process_one_full_mof_metacal(mofc, sim_odict_1m, cat, fitter, prefix='sim1m')) nobj = cat.size tm_fit = time.time() - tm0 return reslists, nobj, tm_fit
def make_obs( *, n_grid=6, dim=235, buff=20, scale=0.2, psf_fwhm=0.9, hlr=0.5, nse=1e-7, star_dxdy=117, star_rad=1, n_stars=5, seed=10, shear=(0.02, 0.0), mcal_shear=(0.0, 0.0) ): rng = np.random.RandomState(seed=seed) n_gals = n_grid**2 tot_dim = dim + 2*buff tot_cen = (tot_dim-1)/2 gloc = (np.arange(n_grid) + 0.5) * (dim / n_grid) - dim/2 gloc *= scale dx, dy = np.meshgrid(gloc, gloc) dx = dx.ravel() + rng.uniform(low=-0.5, high=0.5, size=n_gals) * scale dy = dy.ravel() + rng.uniform(low=-0.5, high=0.5, size=n_gals) * scale ds = np.arange(n_gals) / (n_gals-1) * 0 + 1 gals = galsim.Sum([ galsim.Exponential( half_light_radius=hlr * _ds ).shift( _dx, _dy ).shear( g1=shear[0], g2=shear[1] ).shear( g1=mcal_shear[0], g2=mcal_shear[1] ) for _ds, _dx, _dy in zip(ds, dx, dy) ]) psf = galsim.Gaussian(fwhm=psf_fwhm) objs = galsim.Convolve([gals, psf]) im = objs.drawImage(nx=tot_dim, ny=tot_dim, scale=scale).array im += rng.normal(size=im.shape, scale=nse) nim = rng.normal(size=im.shape, scale=nse) psf_dim = 53 psf_cen = (psf_dim-1)/2 psf_im = psf.drawImage(nx=psf_dim, ny=psf_dim, scale=scale).array # make bmask bmask = np.zeros_like(im, dtype=np.int32) x, y = np.meshgrid(np.arange(tot_dim), np.arange(tot_dim)) sdata = [] for _ in range(n_stars): sr2 = np.power(10.0, rng.uniform(low=star_rad, high=star_rad+0.2))**2 sx = rng.uniform(low=-star_dxdy, high=star_dxdy) + tot_cen sy = rng.uniform(low=-star_dxdy, high=star_dxdy) + tot_cen dr2 = (x - sx)**2 + (y - sy)**2 msk = dr2 < sr2 bmask[msk] |= 2**0 im[msk] = 0 sdata.append((sx, sy, np.sqrt(sr2))) psf_obs = ngmix.Observation( image=psf_im, weight=np.ones_like(psf_im) / nse**2, jacobian=ngmix.DiagonalJacobian(scale=scale, row=psf_cen, col=psf_cen) ) wgt = np.ones_like(im) / nse**2 msk = bmask != 0 wgt[msk] = 0.0 mfrac = np.zeros_like(im) mfrac[msk] = 1.0 obs = ngmix.Observation( image=im, noise=nim, weight=wgt, bmask=bmask, ormask=bmask, jacobian=ngmix.DiagonalJacobian(scale=scale, row=tot_cen, col=tot_cen), psf=psf_obs ) obs.mfrac = mfrac mbobs = ngmix.MultiBandObsList() obsl = ngmix.ObsList() obsl.append(obs) mbobs.append(obsl) mbobs.meta["sdata"] = sdata return mbobs
def _run_moments(*, n_sims, rng, dudx, dudy, dvdx, dvdy): """Run metacal on an image composed of stamps w/ constant noise. Parameters ---------- n_sims : int The number of objects to run. rng : np.random.RandomState An RNG to use. dudx : float The du/dx Jacobian component. dudy : float The du/dy Jacobian component. dydx : float The dv/dx Jacobian component. dvdy : float The dv/dy Jacobian component. Returns ------- result : dict A dictionary with each of the metacal catalogs. """ method = 'no_pixel' stamp_size = 33 psf_stamp_size = 33 cen = (stamp_size - 1) / 2 psf_cen = (psf_stamp_size - 1) / 2 s2n = 1e16 flux = 1e6 galsim_jac = galsim.JacobianWCS(dudx=dudx, dudy=dudy, dvdx=dvdx, dvdy=dvdy) gal = galsim.Exponential(half_light_radius=0.5).withFlux(flux) psf = galsim.Gaussian(fwhm=0.9).withFlux(1) obj = galsim.Convolve(gal, psf) obj_im = obj.drawImage(nx=111, ny=111).array noise = np.sqrt(np.sum(obj_im**2)) / s2n data = [] for ind in tqdm.trange(n_sims): ################################ # make the obs # psf psf_im = psf.drawImage(nx=psf_stamp_size, ny=psf_stamp_size, wcs=galsim_jac, method=method).array psf_noise = np.sqrt(np.sum(psf_im**2)) / 10000 wgt = np.ones_like(psf_im) / psf_noise**2 psf_im += (rng.normal(size=psf_im.shape) * psf_noise) psf_jac = ngmix.Jacobian(x=psf_cen, y=psf_cen, dudx=dudx, dudy=dudy, dvdx=dvdx, dvdy=dvdy) psf_obs = ngmix.Observation(image=psf_im, weight=wgt, jacobian=psf_jac) # now render object scale = psf_jac.scale shift = rng.uniform(low=-scale / 2, high=scale / 2, size=2) _obj = obj.shift(dx=shift[0], dy=shift[1]) xy = galsim_jac.toImage(galsim.PositionD(shift)) im = _obj.drawImage(nx=stamp_size, ny=stamp_size, wcs=galsim_jac, method=method).array jac = ngmix.Jacobian(x=cen + xy.x, y=cen + xy.y, dudx=dudx, dudy=dudy, dvdx=dvdx, dvdy=dvdy) wgt = np.ones_like(im) / noise**2 nse = rng.normal(size=im.shape) * noise im += (rng.normal(size=im.shape) * noise) obs = ngmix.Observation(image=im, weight=wgt, noise=nse, bmask=np.zeros_like(im, dtype=np.int32), ormask=np.zeros_like(im, dtype=np.int32), jacobian=jac, psf=psf_obs) # build the mbobs mbobs = ngmix.MultiBandObsList() obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) mbobs.meta['id'] = ind + 1 # these settings do not matter that much I think mbobs[0].meta['Tsky'] = 1 mbobs[0].meta['magzp_ref'] = 26.5 mbobs[0][0].meta['orig_col'] = ind + 1 mbobs[0][0].meta['orig_row'] = ind + 1 ################################ # run the fitters try: res = _run_moments_fitter(mbobs, rng) except Exception as e: print(e) res = None if res is not None: data.append(res) if len(data) > 0: res = data else: res = None return res
def get_mbobs(self, return_band_images=False): """Make a simulated MultiBandObsList for metadetect. The underlying simulation is done per epoch and then coadded. Parameters ---------- return_band_images : bool If True, return a list of list of numpy arrays holding the SE images in each band. Returns ------- mbobs : MultiBandObsList """ assert not self.called, "you can only call a sim object once!" self.called = True all_band_obj, uv_positions = self._get_band_objects() method = 'auto' LOGGER.debug("using draw method '%s'", method) mbobs = ngmix.MultiBandObsList() if return_band_images: band_images = [] for band in range(self.n_bands): # generate the data I need band_objects = [o[band] for o in all_band_obj] wcs_objs = self._get_all_epoch_wcs_objs(band) # draw the images se_images = [] for epoch, wcs in enumerate(wcs_objs): se_im = self._build_se_image(band=band, epoch=epoch, wcs=wcs, band_objects=band_objects, uv_positions=uv_positions, method=method) se_images.append(se_im.array) if return_band_images: band_images.append(se_images) # add noise, maybe mask them, get noise/wt images and coadd (coadd_im, coadd_noise, coadd_intp, coadd_bmask, coadd_wgts) = self._add_noise_and_coadd(band=band, wcs_objs=wcs_objs, se_images=se_images) # coadd the PSFs coadd_psf = self._coadd_psfs(band=band, wcs_objs=wcs_objs, coadd_wgts=coadd_wgts, method=method) # make the final obs obs_jac = ngmix.jacobian.Jacobian(row=self.im_cen, col=self.im_cen, wcs=self.coadd_wcs.jacobian()) psf_jac = ngmix.jacobian.Jacobian(row=self._psf_cen, col=self._psf_cen, wcs=self.coadd_wcs.jacobian()) psf_obs = ngmix.Observation(coadd_psf, weight=0.0 * coadd_psf + 1.0 / self.noise[band]**2, jacobian=psf_jac) obs = ngmix.Observation(coadd_im, weight=0.0 * coadd_im + 1.0 / np.var(coadd_noise), bmask=coadd_bmask, ormask=coadd_bmask.copy(), jacobian=obs_jac, psf=psf_obs, noise=coadd_noise) obs.meta['fmask'] = coadd_intp obslist = ngmix.ObsList() obslist.append(obs) mbobs.append(obslist) if return_band_images: return mbobs, band_images else: return mbobs