def obslist_data():
dims = (12, 12)
rng = np.random.RandomState(seed=10)
jac = DiagonalJacobian(x=5.5, y=5.5, scale=0.25)
obslist = ObsList()
obslist.append(
Observation(
image=rng.normal(size=dims),
jacobian=jac.copy(),
weight=np.exp(rng.normal(size=dims)),
meta={"blue": 10},
)
)
obslist.append(
Observation(
image=rng.normal(size=dims),
jacobian=jac.copy(),
weight=np.exp(rng.normal(size=dims)),
meta={"blue": 8},
)
)
psf = Observation(
image=rng.normal(size=dims),
weight=np.exp(rng.normal(size=dims)),
jacobian=jac.copy(),
meta={"blue": 5},
)
obslist[1].psf = psf
return obslist
def test_obslist_set():
rng = np.random.RandomState(seed=11)
meta = {'duh': 5}
obslist = ObsList(meta=meta)
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
assert obslist.meta == meta
new_meta = {'blah': 6}
obslist.meta = new_meta
assert obslist.meta == new_meta
obslist.meta = None
assert len(obslist.meta) == 0
with pytest.raises(TypeError):
obslist.meta = [10]
with pytest.raises(TypeError):
obslist.set_meta([10])
new_meta = {'bla': 6}
new_meta.update(obslist.meta)
obslist.update_meta_data({'bla': 6})
assert obslist.meta == new_meta
with pytest.raises(TypeError):
obslist.update_meta_data([10])
rng = np.random.RandomState(seed=12)
new_obs = Observation(image=rng.normal(size=(13, 15)))
rng = np.random.RandomState(seed=11)
for obs in obslist:
assert np.all(obs.image == rng.normal(size=(13, 15)))
obslist[1] = new_obs
assert np.all(obslist[1].image == new_obs.image)
def test_observation_s2n(image_data):
obs = Observation(image=image_data['image'], weight=image_data['weight'])
s2n = obs.get_s2n()
s2n_true = (np.sum(image_data['image']) /
np.sqrt(np.sum(1.0 / image_data['weight'])))
assert np.allclose(s2n, s2n_true)
def test_observation_pixels_update_jacobian(image_data):
obs = Observation(image=image_data['image'],
weight=image_data['weight'],
jacobian=image_data['jacobian'])
new_jac = DiagonalJacobian(x=8, y=13, scale=1.2)
my_pixels = make_pixels(image_data['image'], image_data['weight'], new_jac)
assert np.all(obs.pixels != my_pixels)
obs.jacobian = new_jac
assert np.all(obs.pixels == my_pixels)
def test_observation_pixels_update_weight(image_data):
rng = np.random.RandomState(seed=11)
obs = Observation(image=image_data['image'],
weight=image_data['weight'],
jacobian=image_data['jacobian'])
new_arr = np.exp(rng.normal(size=obs.image.shape))
my_pixels = make_pixels(image_data['image'], new_arr,
image_data['jacobian'])
assert np.all(obs.pixels != my_pixels)
obs.weight = new_arr
assert np.all(obs.pixels == my_pixels)
def get_psf_obs(self, iobj, icutout):
"""Get an observation of the PSF for this object.
Parameters
----------
iobj : int
Index of the object.
icutout : int
Index of the cutout for this object.
Returns
-------
psf : ngmix.Observation
The PSF `Observation`.
"""
psf_im = self.get_psf(iobj, icutout)
# FIXME: fake the noise
noise = psf_im.max() / 1000.0
weight = psf_im * 0 + 1.0 / noise**2
jacobian = self.get_ngmix_jacobian(iobj, icutout)
row, col = self._get_psf_cen(iobj, icutout)
jacobian.set_cen(
row=row,
col=col,
)
return Observation(
psf_im,
weight=weight,
jacobian=jacobian,
)
def make_psf_observation(self):
"""
read the image and weight data
"""
ext=self.get_psf_ext()
self['psf_id'] = ext
self.data['psf_id'][self.dindex] = ext
image0 = self.psf_obj[ext][:,:]
noise=self['psf_pars']['addnoise']
image = image0 + numpy.random.normal(scale=noise,
size=image0.shape)
weight = image.copy()
weight *= 0
weight += 1.0/noise**2
jacob=self.get_jacobian(type='psf')
row,col=jacob.get_cen()
jacob.set_cen(row-1, col-1)
self.psf_obs = Observation(image, weight=weight, jacobian=jacob)
def _fit_with_em(self, obs_in, sigma_guess, ngauss):
"""
Fit the image using EM
"""
im_with_sky, sky = ngmix.em.prep_image(obs_in.image)
sky_obs = Observation(im_with_sky, jacobian=obs_in.jacobian)
ntry, maxiter, tol = self._get_em_pars()
for i in xrange(ntry):
guess = self._get_em_guess(sigma_guess, ngauss)
try:
fitter = ngmix.em.GMixEM(sky_obs)
fitter.go(guess, sky, maxiter=maxiter, tol=tol)
gm = fitter.get_gmix()
# this will raise an exception for crazy values
g1, g2, T = gm.get_g1g2T()
if T > 0.1 and numpy.isfinite(T):
tres = fitter.get_result()
print(" em numiter:", tres['numiter'])
break
except GMixMaxIterEM:
fitter = None
except GMixRangeError:
fitter = None
return fitter
def make_galaxy_observation(self):
"""
read the image and weight data
"""
image = self.meds.get_cutout(self.mindex, 0)
calc_weight = self.get('calc_weight', False)
if calc_weight:
import esutil as eu
border = 5
nrow, ncol = image.shape
row, col = numpy.mgrid[0:nrow, 0:ncol]
w = numpy.where((row < 5) | (row > (nrow - 5 - 1))
| (col < 5) | (col > (ncol - 5 - 1)))
mn, skysig_meas = eu.stat.sigma_clip(image[w].ravel())
print(" skysig_meas: %g" % skysig_meas)
weight = 0 * image + 1.0 / skysig_meas**2
else:
if self['noisefree']:
weight = image * 0 + 1.0 / self['skynoise']**2
else:
weight = self.meds.get_cutout(self.mindex, 0, type='weight')
jacob = self.get_jacobian()
self.obs = Observation(image,
weight=weight,
jacobian=jacob,
psf=self.psf_obs)
def _get_observation(self):
"""
Get the appropriate observation
"""
im, wt, jacob = self._get_image_data()
#print("weight image:",wt)
return Observation(im, weight=wt, jacobian=jacob)
def _fit_galaxy_em(self, obs):
sigma_guess = (self.res['psf_gmix'].get_T() * 0.5)
ngauss = 1
fitter = self._fit_em_ngauss(obs, sigma_guess, ngauss)
if fitter is None:
self.res['flags'] = EM_FIT_FAILURE
print("em utter failure")
else:
gmix = fitter.get_gmix()
#print(" em gmix:",gmix)
Tobs = Observation(obs.image,
jacobian=obs.get_jacobian(),
gmix=gmix)
Tfitter = ngmix.fitting.TemplateFluxFitter(Tobs)
Tfitter.go()
Tres = Tfitter.get_result()
self.res['em_gmix'] = gmix
self.res['em_flux_flags'] = Tres['flags']
self.res['em_flux'] = Tres['flux']
self.res['em_flux_err'] = Tres['flux_err']
self.res['em_flux_s2n'] = Tres['flux'] / Tres['flux_err']
print(" em flux: %g +/- %g" %
(Tres['flux'], Tres['flux_err']))
def test_multibandobslist_set():
rng = np.random.RandomState(seed=11)
meta = {'duh': 5}
mbobs = MultiBandObsList(meta=meta)
for _ in range(5):
obslist = ObsList()
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
mbobs.append(obslist)
assert mbobs.meta == meta
new_meta = {'blah': 6}
mbobs.meta = new_meta
assert mbobs.meta == new_meta
mbobs.meta = None
assert len(mbobs.meta) == 0
with pytest.raises(TypeError):
mbobs.meta = [10]
with pytest.raises(TypeError):
mbobs.set_meta([10])
new_meta = {'bla': 6}
new_meta.update(mbobs.meta)
mbobs.update_meta_data({'bla': 6})
assert mbobs.meta == new_meta
with pytest.raises(TypeError):
mbobs.update_meta_data([10])
rng = np.random.RandomState(seed=12)
new_obs = Observation(image=rng.normal(size=(13, 15)))
rng = np.random.RandomState(seed=11)
for obslist in mbobs:
for obs in obslist:
assert np.all(obs.image == rng.normal(size=(13, 15)))
mbobs[1][2] = new_obs
assert np.all(mbobs[1][2].image == new_obs.image)
rng = np.random.RandomState(seed=13)
obslist = ObsList()
for _ in range(4):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
mbobs[2] = obslist
rng = np.random.RandomState(seed=13)
for obs in mbobs[2]:
assert np.all(obs.image == rng.normal(size=(13, 15)))
def _fit_psf(self):
"""
Fit a psf to a single gauss and more complex model
"""
from ngmix import GMixMaxIterEM
if 'psf_gmix' in self.res:
del self.res['psf_gmix']
del self.res['psf_obs']
self.fitting_galaxy=False
jacobian=self._get_jacobian(cen=self.psf_cen_guess_pix)
psf_obs = Observation(self.psf_image, jacobian=jacobian)
if 'em' in self['psf_model']:
fitter=self._fit_em_ngauss(psf_obs,
self['psf_sigma_guess'],
self['psf_ngauss'])
else:
fitter=self._fit_psf_max(psf_obs)
if fitter is None:
self.res['flags'] = PSF_FIT_FAILURE
print("psf utter failure")
else:
pres=fitter.get_result()
if pres['flags'] != 0:
self.res['flags'] = PSF_FIT_FAILURE
print("psf convergence failure")
else:
psf_gmix = fitter.get_gmix()
#print("psf fit:")
#print(psf_gmix)
print(" psf fwhm:",2.35*sqrt( psf_gmix.get_T()/2. ))
self.res['psf_gmix']=psf_gmix
psf_obs.set_gmix(psf_gmix)
self.res['psf_obs'] = psf_obs
if self['make_plots']:
self._compare_psf(fitter)
def test_observation_nopixels(image_data):
obs = Observation(
image=image_data['image'],
weight=image_data['weight'],
jacobian=image_data['jacobian'],
store_pixels=False,
)
pixels = obs.pixels
assert pixels is None
def _fit_psf(self):
"""
Fit a psf to a single gauss and more complex model
"""
from ngmix import GMixMaxIterEM
if 'psf_gmix' in self.res:
del self.res['psf_gmix']
del self.res['psf_obs']
self.fitting_galaxy = False
jacobian = self._get_jacobian(cen=self.psf_cen_guess_pix)
psf_obs = Observation(self.psf_image, jacobian=jacobian)
if 'em' in self['psf_model']:
fitter = self._fit_em_ngauss(psf_obs, self['psf_sigma_guess'],
self['psf_ngauss'])
else:
fitter = self._fit_psf_max(psf_obs)
if fitter is None:
self.res['flags'] = PSF_FIT_FAILURE
print("psf utter failure")
else:
pres = fitter.get_result()
if pres['flags'] != 0:
self.res['flags'] = PSF_FIT_FAILURE
print("psf convergence failure")
else:
psf_gmix = fitter.get_gmix()
#print("psf fit:")
#print(psf_gmix)
print(" psf fwhm:", 2.35 * sqrt(psf_gmix.get_T() / 2.))
self.res['psf_gmix'] = psf_gmix
psf_obs.set_gmix(psf_gmix)
self.res['psf_obs'] = psf_obs
if self['make_plots']:
self._compare_psf(fitter)
def test_observation_pixels(image_data):
obs = Observation(image=image_data['image'],
weight=image_data['weight'],
jacobian=image_data['jacobian'])
pixels = obs.pixels
my_pixels = make_pixels(image_data['image'], image_data['weight'],
image_data['jacobian'])
assert np.all(pixels == my_pixels)
def test_observation_context(image_data):
obs = Observation(
image=image_data['image'].copy(),
weight=image_data['weight'].copy(),
bmask=image_data['bmask'].copy(),
ormask=image_data['ormask'].copy(),
noise=image_data['noise'].copy(),
jacobian=image_data['jacobian'],
)
_dotest_readonly_attrs(obs)
# should be a no-op outside of a context
obs.writeable()
_dotest_readonly_attrs(obs)
with obs.writeable():
_dotest_writeable_attrs(obs)
def test_obslist_smoke():
rng = np.random.RandomState(seed=11)
meta = {'duh': 5}
obslist = ObsList(meta=meta)
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
assert obslist.meta == meta
rng = np.random.RandomState(seed=11)
for obs in obslist:
assert np.all(obs.image == rng.normal(size=(13, 15)))
def image_data():
dims = (13, 15)
rng = np.random.RandomState(seed=10)
data = {}
for key in ['image', 'weight', 'bmask', 'ormask', 'noise']:
data[key] = rng.normal(size=dims)
if key == 'weight':
data[key] = np.exp(data[key])
elif key in ['bmask', 'ormask']:
data[key] = np.clip(data[key] * 100, 0, np.inf).astype(np.int32)
data['jacobian'] = DiagonalJacobian(x=7, y=6, scale=0.25)
data['meta'] = {'pi': 3.14}
data['psf'] = Observation(image=rng.normal(size=dims),
meta={'ispsf': True})
data['psf_gmix'] = Observation(image=rng.normal(size=dims),
meta={'ispsf': True},
gmix=GMix(pars=rng.uniform(size=6)))
data['gmix'] = GMix(pars=rng.uniform(size=6))
return data
def test_obslist_s2n_zeroweight():
rng = np.random.RandomState(seed=11)
obslist = ObsList()
for _ in range(3):
obs = Observation(
image=rng.normal(size=(13, 15)),
weight=np.zeros((13, 15)),
store_pixels=False,
)
obslist.append(obs)
assert np.allclose(obslist.get_s2n(), -9999)
def test_obslist_s2n():
rng = np.random.RandomState(seed=11)
obslist = ObsList()
numer = 0
denom = 0
for _ in range(3):
obs = Observation(image=rng.normal(size=(13, 15)))
numer += np.sum(obs.image)
denom += np.sum(1.0 / obs.weight)
obslist.append(obs)
s2n = obslist.get_s2n()
assert s2n == numer / np.sqrt(denom)
def get_exp_list(self, psf2=None):
if psf2 is None:
psf2 = self.psfs
obs_list = ObsList()
psf_list = ObsList()
w = []
for i in range(len(self.gals)):
im = self.gals[i].array
im_psf = self.psfs[i].array
im_psf2 = psf2[i].array
weight = 1 / self.skys[i].array
jacob = self.gals[i].wcs.jacobian()
dx = self.offsets[i].x
dy = self.offsets[i].y
gal_jacob = Jacobian(row=self.gals[i].true_center.y + dy,
col=self.gals[i].true_center.x + dx,
dvdrow=jacob.dvdy,
dvdcol=jacob.dvdx,
dudrow=jacob.dudy,
dudcol=jacob.dudx)
psf_jacob2 = gal_jacob
mask = np.where(weight != 0)
w.append(np.mean(weight[mask]))
noise = old_div(np.ones_like(weight), w[-1])
psf_obs = Observation(im_psf,
jacobian=gal_jacob,
meta={
'offset_pixels': None,
'file_id': None
})
psf_obs2 = Observation(im_psf2,
jacobian=psf_jacob2,
meta={
'offset_pixels': None,
'file_id': None
})
obs = Observation(im,
weight=weight,
jacobian=gal_jacob,
psf=psf_obs,
meta={
'offset_pixels': None,
'file_id': None
})
obs.set_noise(noise)
obs_list.append(obs)
psf_list.append(psf_obs2)
return obs_list, psf_list, np.array(w)
def test_get_mbobs():
rng = np.random.RandomState(seed=11)
obs = Observation(image=rng.normal(size=(11, 13)))
mbobs = get_mb_obs(obs)
rng = np.random.RandomState(seed=11)
assert np.all(mbobs[0][0].image == rng.normal(size=(11, 13)))
assert len(mbobs) == 1
assert len(mbobs[0]) == 1
rng = np.random.RandomState(seed=12)
obslist = ObsList()
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(11, 13))))
mbobs = get_mb_obs(obslist)
rng = np.random.RandomState(seed=12)
for obs in mbobs[0]:
assert np.all(obs.image == rng.normal(size=(11, 13)))
assert len(mbobs) == 1
assert len(mbobs[0]) == 3
rng = np.random.RandomState(seed=13)
mbobs = MultiBandObsList()
for _ in range(5):
obslist = ObsList()
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
mbobs.append(obslist)
new_mbobs = get_mb_obs(mbobs)
rng = np.random.RandomState(seed=13)
for obslist in new_mbobs:
for obs in obslist:
assert np.all(obs.image == rng.normal(size=(13, 15)))
with pytest.raises(ValueError):
get_mb_obs(None)
def make_psf_observation(self):
"""
read the image and weight data
"""
ext = self.get_psf_ext()
self['psf_id'] = ext
self.data['psf_id'][self.dindex] = ext
image0 = self.psf_obj[ext][:, :]
noise = self['psf_pars']['addnoise']
image = image0 + numpy.random.normal(scale=noise, size=image0.shape)
weight = image.copy()
weight *= 0
weight += 1.0 / noise**2
jacob = self.get_jacobian(type='psf')
row, col = jacob.get_cen()
jacob.set_cen(row - 1, col - 1)
self.psf_obs = Observation(image, weight=weight, jacobian=jacob)
def test_multibandobslist_smoke():
rng = np.random.RandomState(seed=11)
meta = {'duh': 5}
mbobs = MultiBandObsList(meta=meta)
for _ in range(5):
obslist = ObsList()
for _ in range(3):
obslist.append(Observation(image=rng.normal(size=(13, 15))))
mbobs.append(obslist)
assert mbobs.meta == meta
rng = np.random.RandomState(seed=11)
for obslist in mbobs:
for obs in obslist:
assert np.all(obs.image == rng.normal(size=(13, 15)))
def test_multibandobslist_s2n_zeroweight():
rng = np.random.RandomState(seed=11)
mbobs = MultiBandObsList()
for _ in range(5):
obslist = ObsList()
for _ in range(3):
img = rng.normal(size=(13, 15))
obslist.append(
Observation(
image=img,
weight=np.zeros((13, 15)),
store_pixels=False,
))
mbobs.append(obslist)
assert np.allclose(mbobs.get_s2n(), -9999)
def test_multibandobslist_s2n():
rng = np.random.RandomState(seed=11)
mbobs = MultiBandObsList()
numer = 0
denom = 0
for _ in range(5):
obslist = ObsList()
for _ in range(3):
img = rng.normal(size=(13, 15))
obslist.append(Observation(image=img))
numer += np.sum(img)
denom += np.sum(1.0 / obslist[-1].weight)
mbobs.append(obslist)
s2n = mbobs.get_s2n()
assert s2n == numer / np.sqrt(denom)
def test_observation_pixels_noignore_zero(image_data):
weight = image_data['weight'].copy()
weight[:, 5] = 0.0
obs = Observation(
image=image_data['image'],
weight=weight,
jacobian=image_data['jacobian'],
ignore_zero_weight=False,
)
pixels = obs.pixels
my_pixels = make_pixels(
image_data['image'],
weight,
image_data['jacobian'],
ignore_zero_weight=False,
)
assert np.all(pixels == my_pixels)
def test_observation_s2n(image_data):
obs = Observation(image=image_data['image'], weight=image_data['weight'])
s2n = obs.get_s2n()
s2n_true = (np.sum(image_data['image']) /
np.sqrt(np.sum(1.0 / image_data['weight'])))
assert np.allclose(s2n, s2n_true)
# if we are not storing pixels, when we can have an all zero weight map
# in this case s2n is -9999
obs = Observation(
image=image_data['image'],
weight=np.zeros_like(obs.weight),
store_pixels=False,
)
assert np.allclose(obs.get_s2n(), -9999)
def get_exp_list(gals_array, psfs_array, offsets, skys_array, gal_true, gal_jacobs, psf2=None):
#def get_exp_list(gal, psf, sky_stamp, psf2=None):
if psf2 is None:
psf2 = psfs_array
obs_list=ObsList()
psf_list=ObsList()
w = []
for i in range(len(gals_array)):
im = gals_array[i]
im_psf = psfs_array[i]
im_psf2 = psf2[i]
weight = 1/skys_array[i]
jacob = gal_jacobs[i]
dx = offsets[i].x
dy = offsets[i].y
gal_jacob = Jacobian(
row=gal_true[i].y+dy,
col=gal_true[i].x+dx,
dvdrow=jacob.dvdy,
dvdcol=jacob.dvdx,
dudrow=jacob.dudy,
dudcol=jacob.dudx)
psf_jacob2 = gal_jacob
print(gal_jacob)
mask = np.where(weight!=0)
w.append(np.mean(weight[mask]))
noise = old_div(np.ones_like(weight),w[-1])
psf_obs = Observation(im_psf, jacobian=gal_jacob, meta={'offset_pixels':None,'file_id':None})
psf_obs2 = Observation(im_psf2, jacobian=psf_jacob2, meta={'offset_pixels':None,'file_id':None})
obs = Observation(im, weight=weight, jacobian=gal_jacob, psf=psf_obs, meta={'offset_pixels':None,'file_id':None})
obs.set_noise(noise)
obs_list.append(obs)
psf_list.append(psf_obs2)
return obs_list,psf_list,np.array(w)
def multiband_coadd():
local_Hmeds = './fiducial_H158_2285117.fits'
local_Jmeds = './fiducial_J129_2285117.fits'
local_Fmeds = './fiducial_F184_2285117.fits'
truth = fio.FITS(
'/hpc/group/cosmology/phy-lsst/my137/roman_H158/g1002/truth/fiducial_lensing_galaxia_g1002_truth_gal.fits'
)[-1]
m_H158 = meds.MEDS(local_Hmeds)
m_J129 = meds.MEDS(local_Jmeds)
m_F184 = meds.MEDS(local_Fmeds)
indices_H = np.arange(len(m_H158['number'][:]))
indices_J = np.arange(len(m_J129['number'][:]))
indices_F = np.arange(len(m_F184['number'][:]))
roman_H158_psfs = get_psf_SCA('H158')
roman_J129_psfs = get_psf_SCA('J129')
roman_F184_psfs = get_psf_SCA('F184')
oversample = 1
metacal_keys = ['noshear', '1p', '1m', '2p', '2m']
res_noshear = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float),
('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_1p = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_1m = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_2p = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_2m = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_tot = [res_noshear, res_1p, res_1m, res_2p, res_2m]
for i, ii in enumerate(
indices_H): # looping through all the objects in meds file.
if i % 100 == 0:
print('object number ', i)
ind = m_H158['number'][ii]
t = truth[ind]
sca_Hlist = m_H158[ii][
'sca'] # List of SCAs for the same object in multiple observations.
sca_Jlist = m_J129[ii]['sca']
sca_Flist = m_F184[ii]['sca']
m2_H158_coadd = [
roman_H158_psfs[j - 1] for j in sca_Hlist[:m_H158['ncutout'][i]]
]
m2_J129_coadd = [
roman_J129_psfs[j - 1] for j in sca_Jlist[:m_J129['ncutout'][i]]
]
m2_F184_coadd = [
roman_F184_psfs[j - 1] for j in sca_Flist[:m_F184['ncutout'][i]]
]
obs_Hlist, psf_Hlist, included_H, w_H = get_exp_list_coadd(
m_H158, ii, m2=m2_H158_coadd)
coadd_H = psc.Coadder(obs_Hlist, flat_wcs=True).coadd_obs
coadd_H.psf.image[
coadd_H.psf.image < 0] = 0 # set negative pixels to zero.
coadd_H.set_meta({'offset_pixels': None, 'file_id': None})
obs_Jlist, psf_Jlist, included_J, w_J = get_exp_list_coadd(
m_J129, ii, m2=m2_J129_coadd)
coadd_J = psc.Coadder(obs_Jlist, flat_wcs=True).coadd_obs
coadd_J.psf.image[
coadd_J.psf.image < 0] = 0 # set negative pixels to zero.
coadd_J.set_meta({'offset_pixels': None, 'file_id': None})
obs_Flist, psf_Flist, included_F, w_F = get_exp_list_coadd(
m_F184, ii, m2=m2_F184_coadd)
coadd_F = psc.Coadder(obs_Flist, flat_wcs=True).coadd_obs
coadd_F.psf.image[
coadd_F.psf.image < 0] = 0 # set negative pixels to zero.
coadd_F.set_meta({'offset_pixels': None, 'file_id': None})
coadd = [coadd_H, coadd_J, coadd_F]
mb_obs_list = MultiBandObsList()
#coadd = [coadd_H]
for band in range(3):
obs_list = ObsList()
new_coadd_psf_block = block_reduce(coadd[band].psf.image,
block_size=(4, 4),
func=np.sum)
new_coadd_psf_jacob = Jacobian(
row=15.5,
col=15.5,
dvdrow=(coadd[band].psf.jacobian.dvdrow * oversample),
dvdcol=(coadd[band].psf.jacobian.dvdcol * oversample),
dudrow=(coadd[band].psf.jacobian.dudrow * oversample),
dudcol=(coadd[band].psf.jacobian.dudcol * oversample))
coadd_psf_obs = Observation(new_coadd_psf_block,
jacobian=new_coadd_psf_jacob,
meta={
'offset_pixels': None,
'file_id': None
})
coadd[band].psf = coadd_psf_obs
obs_list.append(coadd[band])
mb_obs_list.append(obs_list)
iteration = 0
for key in metacal_keys:
res_tot[iteration]['ind'][i] = ind
res_tot[iteration]['ra'][i] = t['ra']
res_tot[iteration]['dec'][i] = t['dec']
iteration += 1
#print(i, t['size'], mb_obs_list[0][0].image.sum(), mb_obs_list[1][0].image.sum(), mb_obs_list[2][0].image.sum())
res_ = measure_shape_metacal_multiband(
mb_obs_list,
t['size'],
method='bootstrap',
fracdev=t['bflux'],
use_e=[t['int_e1'], t['int_e2']])
iteration = 0
for key in metacal_keys:
if res_ == 0:
res_tot[iteration]['ind'][i] = 0
elif res_[key]['flags'] == 0:
res_tot[iteration]['coadd_px'][i] = res_[key]['pars'][0]
res_tot[iteration]['coadd_py'][i] = res_[key]['pars'][1]
res_tot[iteration]['coadd_snr'][i] = res_[key]['s2n']
res_tot[iteration]['coadd_e1'][i] = res_[key]['pars'][2]
res_tot[iteration]['coadd_e2'][i] = res_[key]['pars'][3]
res_tot[iteration]['coadd_hlr'][i] = res_[key]['pars'][4]
iteration += 1
mask = res_tot[0]['ind'] != 0
print(len(res_tot[0]), len(res_tot[0][mask]))
#print(res_['noshear'].dtype.names)
print('done')
def get_exp_list_coadd(m, i, m2=None):
def make_jacobian(dudx, dudy, dvdx, dvdy, x, y):
j = galsim.JacobianWCS(dudx, dudy, dvdx, dvdy)
return j.withOrigin(galsim.PositionD(x, y))
oversample = 1
#def psf_offset(i,j,star_):
m3 = [0]
#relative_offset=[0]
for jj, psf_ in enumerate(m2): # m2 has psfs for each observation.
if jj == 0:
continue
gal_stamp_center_row = m['orig_start_row'][i][jj] + m['box_size'][
i] / 2 - 0.5 # m['box_size'] is the galaxy stamp size.
gal_stamp_center_col = m['orig_start_col'][i][jj] + m['box_size'][
i] / 2 - 0.5 # m['orig_start_row/col'] is in SCA coordinates.
psf_stamp_size = 32
# Make the bounds for the psf stamp.
b = galsim.BoundsI(
xmin=(m['orig_start_col'][i][jj] +
(m['box_size'][i] - 32) / 2. - 1) * oversample + 1,
xmax=(m['orig_start_col'][i][jj] +
(m['box_size'][i] - 32) / 2. + psf_stamp_size - 1) *
oversample,
ymin=(m['orig_start_row'][i][jj] +
(m['box_size'][i] - 32) / 2. - 1) * oversample + 1,
ymax=(m['orig_start_row'][i][jj] +
(m['box_size'][i] - 32) / 2. + psf_stamp_size - 1) *
oversample)
# Make wcs for oversampled psf.
wcs_ = make_jacobian(
m.get_jacobian(i, jj)['dudcol'] / oversample,
m.get_jacobian(i, jj)['dudrow'] / oversample,
m.get_jacobian(i, jj)['dvdcol'] / oversample,
m.get_jacobian(i, jj)['dvdrow'] / oversample,
m['orig_col'][i][jj] * oversample,
m['orig_row'][i][jj] * oversample)
# Taken from galsim/roman_psfs.py line 266. Update each psf to an object-specific psf using the wcs.
scale = galsim.PixelScale(wfirst.pixel_scale / oversample)
psf_ = wcs_.toWorld(scale.toImage(psf_),
image_pos=galsim.PositionD(wfirst.n_pix / 2,
wfirst.n_pix / 2))
# Convolve with the star model and get the psf stamp.
#st_model = galsim.DeltaFunction(flux=1.)
#st_model = st_model.evaluateAtWavelength(wfirst.getBandpasses(AB_zeropoint=True)['H158'].effective_wavelength)
#st_model = st_model.withFlux(1.)
#st_model = galsim.Convolve(st_model, psf_)
psf_ = galsim.Convolve(psf_, galsim.Pixel(wfirst.pixel_scale))
psf_stamp = galsim.Image(b, wcs=wcs_)
# Galaxy is being drawn with some subpixel offsets, so we apply the offsets when drawing the psf too.
offset_x = m['orig_col'][i][jj] - gal_stamp_center_col
offset_y = m['orig_row'][i][jj] - gal_stamp_center_row
offset = galsim.PositionD(offset_x, offset_y)
if (offset_x <= -1.0 or offset_y <= -1.0):
print(offset)
elif (offset_x >= 1.0 or offset_y >= 1.0):
print(offset)
psf_.drawImage(image=psf_stamp, offset=offset, method='no_pixel')
m3.append(psf_stamp.array)
obs_list = ObsList()
psf_list = ObsList()
included = []
w = []
# For each of these objects create an observation
for j in range(m['ncutout'][i]):
if j == 0:
continue
# if j>1:
# continue
im = m.get_cutout(i, j, type='image')
weight = m.get_cutout(i, j, type='weight')
im_psf = m3[j]
im_psf2 = im_psf
if np.sum(im) == 0.:
#print(local_meds, i, j, np.sum(im))
print('no flux in image ', i, j)
continue
jacob = m.get_jacobian(i, j)
# Get a galaxy jacobian.
gal_jacob = Jacobian(
row=(m['orig_row'][i][j] - m['orig_start_row'][i][j]),
col=(m['orig_col'][i][j] - m['orig_start_col'][i][j]),
dvdrow=jacob['dvdrow'],
dvdcol=jacob['dvdcol'],
dudrow=jacob['dudrow'],
dudcol=jacob['dudcol'])
psf_center = (32 / 2.) + 0.5
# Get a oversampled psf jacobian.
if oversample == 1:
psf_jacob2 = Jacobian(
row=15.5 +
(m['orig_row'][i][j] - m['orig_start_row'][i][j] + 1 -
(m['box_size'][i] / 2. + 0.5)) * oversample,
col=15.5 +
(m['orig_col'][i][j] - m['orig_start_col'][i][j] + 1 -
(m['box_size'][i] / 2. + 0.5)) * oversample,
dvdrow=jacob['dvdrow'] / oversample,
dvdcol=jacob['dvdcol'] / oversample,
dudrow=jacob['dudrow'] / oversample,
dudcol=jacob['dudcol'] / oversample)
elif oversample == 4:
psf_jacob2 = Jacobian(
row=63.5 +
(m['orig_row'][i][j] - m['orig_start_row'][i][j] + 1 -
(m['box_size'][i] / 2. + 0.5)) * oversample,
col=63.5 +
(m['orig_col'][i][j] - m['orig_start_col'][i][j] + 1 -
(m['box_size'][i] / 2. + 0.5)) * oversample,
dvdrow=jacob['dvdrow'] / oversample,
dvdcol=jacob['dvdcol'] / oversample,
dudrow=jacob['dudrow'] / oversample,
dudcol=jacob['dudcol'] / oversample)
# Create an obs for each cutout
mask = np.where(weight != 0)
if 1. * len(weight[mask]) / np.product(np.shape(weight)) < 0.8:
continue
w.append(np.mean(weight[mask]))
noise = np.ones_like(weight) / w[-1]
psf_obs = Observation(im_psf,
jacobian=gal_jacob,
meta={
'offset_pixels': None,
'file_id': None
})
psf_obs2 = Observation(im_psf2,
jacobian=psf_jacob2,
meta={
'offset_pixels': None,
'file_id': None
})
#obs = Observation(im, weight=weight, jacobian=gal_jacob, psf=psf_obs, meta={'offset_pixels':None,'file_id':None})
# oversampled PSF
obs = Observation(im,
weight=weight,
jacobian=gal_jacob,
psf=psf_obs2,
meta={
'offset_pixels': None,
'file_id': None
})
obs.set_noise(noise)
obs_list.append(obs)
psf_list.append(psf_obs2)
included.append(j)
return obs_list, psf_list, np.array(included) - 1, np.array(w)
class MedsFitBase(dict):
def __init__(self, meds_file, truth_file, psf_file, **keys):
self['meds_file']=meds_file
self['truth_file']=truth_file
self['psf_file']=psf_file
self.update(keys)
self.set_defaults()
self.load_data()
self.set_indices()
self.make_struct()
def set_defaults(self):
"""
deal with default parameters and conversions
"""
# in arcsec
sigma_guess=self['psf_pars']['fwhm_guess']/2.35
self['psf_Tguess'] = 2*sigma_guess**2
self['make_plots']=self.get('make_plots',False)
self['randomize_psf'] = self.get('randomize_psf',False)
def get_data(self):
"""
get a reference to the data structure
"""
return self.data
def do_fits(self):
"""
loop and fit all objects
"""
t0=time.time()
last=self.indices.size-1
for dindex,mindex in enumerate(self.indices):
self.dindex=dindex
self.mindex=mindex
print("%d:%d %d:%d" % (dindex, last, mindex, self['end']))
self.data['number'][dindex] = self.meds['number'][mindex]
self.make_psf_observation()
self.make_galaxy_observation()
self.run_fitters()
if self['make_plots']:
self.compare_psf()
self.do_gal_plots()
tm=time.time()-t0
num=len(self.indices)
print("time:",tm)
print("time per:",tm/num)
def run_fitters(self):
from great3.generic import PSFFailure,GalFailure
dindex=self.dindex
boot=self.get_bootstrapper()
# find the center and reset the jacobian
boot.find_cen()
self.boot=boot
try:
self.fit_psf()
self.fit_psf_flux()
try:
self.fit_galaxy()
self.copy_galaxy_result()
self.print_galaxy_result()
except GalFailure:
print(" galaxy fitting failed")
self.data['flags'][dindex] = GAL_FIT_FAILURE
except PSFFailure:
print(" psf fitting failed")
self.data['flags'][dindex] = PSF_FIT_FAILURE
def fit_psf(self):
dindex=self.dindex
boot=self.boot
psf_pars=self['psf_pars']
boot.fit_psf(psf_pars['model'],
Tguess=self['psf_Tguess'],
ntry=psf_pars['ntry'])
self.psf_fitter=self.boot.get_psf_fitter()
self.copy_psf_result()
def fit_psf_flux(self):
"""
fit psf model to galaxy with one free parameter for flux
"""
boot=self.boot
dindex=self.dindex
boot.fit_gal_psf_flux()
data=self.data
data['psf_flux'][dindex] = boot.psf_flux
data['psf_flux_err'][dindex] = boot.psf_flux_err
def fit_galaxy(self):
"""
over-ride for different fitters
"""
raise RuntimeError("over-ride me")
def fit_max(self):
"""
do a maximum likelihood fit
"""
boot=self.boot
model=self['model_pars']['model']
max_pars=self['max_pars']
boot.fit_max(model,
max_pars,
prior=self['prior'],
ntry=max_pars['ntry'])
def get_bootstrapper(self):
"""
get the bootstrapper for fitting psf through galaxy
"""
from great3.sfit import get_bootstrapper
boot = get_bootstrapper(self.psf_obs, self.obs)
return boot
def make_psf_observation(self):
"""
read the image and weight data
"""
ext=self.get_psf_ext()
self['psf_id'] = ext
self.data['psf_id'][self.dindex] = ext
image0 = self.psf_obj[ext][:,:]
noise=self['psf_pars']['addnoise']
image = image0 + numpy.random.normal(scale=noise,
size=image0.shape)
weight = image.copy()
weight *= 0
weight += 1.0/noise**2
jacob=self.get_jacobian()
self.psf_obs = Observation(image, weight=weight, jacobian=jacob)
def get_psf_ext(self):
if self['randomize_psf']:
ext=numpy.random.randint(0,self.npsf)
print("getting random psf:",ext)
else:
ext=self.truth['id_psf'][self.mindex]
return ext
def make_galaxy_observation(self):
"""
read the image and weight data
"""
image = self.meds.get_cutout(self.mindex, 0)
calc_weight=self.get('calc_weight',False)
if calc_weight:
import esutil as eu
border=5
nrow,ncol=image.shape
row,col=numpy.mgrid[0:nrow, 0:ncol]
w=numpy.where((row < 5) | (row > (nrow-5-1))
| (col < 5) | (col > (ncol-5-1)) )
mn, skysig_meas = eu.stat.sigma_clip(image[w].ravel())
print(" skysig_meas: %g" % skysig_meas)
weight = 0*image + 1.0/skysig_meas**2
else:
if self['noisefree']:
weight = image*0 + 1.0/self['skynoise']**2
else:
weight = self.meds.get_cutout(self.mindex, 0, type='weight')
jacob=self.get_jacobian()
self.obs = Observation(image, weight=weight, jacobian=jacob)
def get_psf_guesser(self):
"""
get guesser based of size of psf and psf flux
"""
from gmix_meds.util import FromPSFGuesser
data=self.data
T=data['psf_T'][self.dindex]
flux=data['psf_flux'][self.dindex]
if self['use_logpars']:
scaling='log'
else:
scaling='linear'
guesser=FromPSFGuesser(T, flux, scaling=scaling)
return guesser
def get_flux_and_prior_guesser(self):
"""
from the psf flux and the prior
"""
from gmix_meds.util import FluxAndPriorGuesser
psf_flux=self.data['psf_flux'][self.dindex]
psf_flux=psf_flux.clip(min=0.1, max=1.0e9)
if self['use_logpars']:
scaling='log'
else:
scaling='linear'
guesser=FluxAndPriorGuesser(psf_flux, self['prior'],scaling=scaling)
return guesser
def get_T_flux_and_prior_guesser(self):
"""
from the psf flux and the prior
"""
from gmix_meds.util import TFluxAndPriorGuesser
psf_gmix = self.psf_obs.get_gmix()
psf_T = psf_gmix.get_T()
psf_flux=self.data['psf_flux'][self.dindex]
psf_flux=psf_flux.clip(min=0.1, max=1.0e9)
if self['use_logpars']:
scaling='log'
else:
scaling='linear'
guesser=TFluxAndPriorGuesser(psf_T,
psf_flux,
self['prior'],
scaling=scaling)
return guesser
def try_replace_cov(self, fitter):
"""
the lm cov sucks, try to replace it
"""
# reference to res
res=fitter.get_result()
print(" replacing cov")
max_pars=self['max_pars']
fitter.calc_cov(max_pars['cov_h'], max_pars['cov_m'])
if res['flags'] != 0:
print(" replacement failed")
res['flags']=0
def compare_psf(self):
"""
compare psf image to best fit model
"""
import images
fitter=self.psf_fitter
model=self['psf_pars']['model']
obs=self.psf_obs
if 'em' in model:
model_image = fitter.make_image(counts=obs.image.sum())
else:
gm=fitter.get_gmix()
j=obs.get_jacobian()
model_image = gm.make_image(obs.image.shape,
jacobian=j)
plt=images.compare_images(obs.image,
model_image,
label1='psf',
label2=model,
show=False)
pname='psf-resid-%s-%06d.png' % (model, self.mindex)
print(" ",pname)
plt.write_img(1400,800,pname)
def do_gal_plots(self):
"""
Make residual plot and trials plot
"""
res=self.gal_fitter.get_result()
if res['flags'] != 0:
return
self.compare_gal()
#self.make_trials_plot()
#self.plot_autocorr()
def compare_gal(self):
"""
compare psf image to best fit model
"""
import images
fitter=self.gal_fitter
model=self['model_pars']['model']
title = '%d %s' % (self.mindex, model)
gmix = fitter.get_gmix()
obs = self.obs
res=fitter.get_result()
psf_gmix = self.psf_obs.get_gmix()
gmix_conv = gmix.convolve(psf_gmix)
image=obs.image
model_image = gmix_conv.make_image(image.shape,
jacobian=obs.get_jacobian())
plt=images.compare_images(image,
model_image,
label1='galaxy',
label2=model,
show=False)
plt.title=title
pname='gal-resid-%06d-%s.png' % (self.mindex,model)
resid_std = (image-model_image).std()
print(" residual std:",resid_std)
print(" ",pname)
plt.write_img(1400,800,pname)
def get_jacobian(self):
"""
get the jacobian and return a Jacobian object
"""
jdict = self.meds.get_jacobian(self.mindex,0)
jacob = Jacobian(jdict['row0']-1,
jdict['col0']-1,
jdict['dudrow'],
jdict['dudcol'],
jdict['dvdrow'],
jdict['dvdcol'])
return jacob
def load_data(self):
"""
read or load all data
"""
print("loading:",self['meds_file'])
self.meds=meds.MEDS(self['meds_file'])
print("reading:",self['truth_file'])
self.truth=fitsio.read(self['truth_file'])
print("loading:",self['psf_file'])
self.psf_obj = fitsio.FITS(self['psf_file'])
self.npsf = len(self.psf_obj)
def set_indices(self):
"""
this version we don't support work dir
"""
obj_range = self.get('obj_range',None)
if obj_range is not None:
self['start'] = obj_range[0]
self['end'] = obj_range[1]
self.indices = arange(obj_range[0], obj_range[1]+1)
else:
self['start']=0
self['end']=self.meds.size-1
self.indices = arange(self.meds.size)
def copy_psf_result(self):
"""
copy some subset of the psf parameters
"""
ppars=self['psf_pars']
data=self.data
fitter=self.psf_fitter
res=fitter.get_result()
data['psf_flags'][self.dindex] = res['flags']
if 'nfev' in res:
data['psf_nfev'][self.dindex] = res['nfev']
elif 'numiter' in res:
data['psf_nfev'][self.dindex] = res['numiter']
if res['flags'] != 0:
return
psf_gmix=fitter.get_gmix()
g1,g2,T=psf_gmix.get_g1g2T()
print(" psf_id: %d psf_fwhm: %.3f g: %.3g %.3g" % (self['psf_id'],sqrt(T/2)*2.35,g1,g2) )
if 'em' in ppars['model']:
print(" niter: %d fdiff: %g" % (res['numiter'],res['fdiff']))
else:
print_pars(res['pars'], front=' psf_pars: ')
print_pars(res['pars_err'],front=' psf_perr: ')
data['psf_g'][self.dindex, 0] = g1
data['psf_g'][self.dindex, 1] = g2
data['psf_T'][self.dindex] = T
def get_namer(self):
from great3.generic import Namer
if self['use_logpars']:
n=Namer('log')
else:
n=Namer()
return n
def copy_galaxy_result(self):
"""
copy some subset of the psf parameters
"""
from pprint import pprint
n=self.get_namer()
data=self.data
dindex=self.dindex
fitter=self.gal_fitter
res=fitter.get_result()
#pprint(res)
if res['flags'] != 0:
print(" galaxy fit failure")
data['flags'][dindex] = GAL_FIT_FAILURE
return
else:
data['flags'][dindex]=0
jacob=self.boot.gal_obs.jacobian
jrow, jcol = jacob.get_cen()
scale = jacob.get_scale()
row = jrow + res['pars'][0]/scale
col = jcol + res['pars'][1]/scale
data['pars'][dindex] = res['pars']
data['pars_cov'][dindex] = res['pars_cov']
data['cen_pix'][dindex] = array([row,col])
data['g'][dindex] = res['g']
data['g_cov'][dindex] = res['g_cov']
data[n('flux')][dindex] = res['pars'][5]
data[n('flux_err')][dindex] = sqrt(res['pars_cov'][5,5])
data[n('T')][dindex] = res['pars'][4]
data[n('T_err')][dindex] = sqrt(res['pars_cov'][4,4])
if self['use_logpars']:
Ts2n = 1.0/data[n('T_err')][dindex]
else:
Ts2n = data[n('T')][dindex]/data[n('T_err')][dindex]
data['T_s2n'][dindex] = Ts2n
data['s2n_w'][dindex] = res['s2n_w']
data['chi2per'][dindex] = res['chi2per']
data['dof'][dindex] = res['dof']
def print_galaxy_result(self):
res=self.gal_fitter.get_result()
if 'pars' in res:
print_pars(res['pars'], front=' gal_pars: ')
if 'pars_err' in res:
print_pars(res['pars_err'],front=' gal_perr: ')
def make_dtype(self):
"""
make the output data type
"""
n=self.get_namer()
np=ngmix.gmix.get_model_npars(self['model_pars']['model'])
dt=[
('number','i4'),
('flags','i4'),
('psf_id','i4'),
('psf_flags','i4'),
('psf_nfev','i4'),
('psf_g','f8',2),
('psf_T','f8'),
('psf_flux','f8'),
('psf_flux_err','f8'),
('pars','f8',np),
('pars_cov','f8',(np,np)),
('cen_pix','f8',2),
(n('flux'),'f8'),
(n('flux_err'),'f8'),
(n('T'),'f8'),
(n('T_err'),'f8'),
('T_s2n','f8'),
('g','f8',2),
('g_cov','f8',(2,2)),
('s2n_w','f8'),
('chi2per','f8'),
('dof','f8'),
]
self.dtype=dt
def make_struct(self):
self.make_dtype()
n=self.get_namer()
num=self.indices.size
data=zeros(num, dtype=self.dtype)
data['flags'] = NO_ATTEMPT
data['psf_g'] = DEFVAL
data['psf_T'] = DEFVAL
data['psf_flux'] = DEFVAL
data['psf_flux_err'] = PDEFVAL
data['pars'] = DEFVAL
data['pars_cov'] = PDEFVAL
data[n('flux')] = DEFVAL
data[n('flux_err')] = PDEFVAL
data[n('T')] = DEFVAL
data[n('T_err')] = PDEFVAL
data['g'] = DEFVAL
data['g_cov'] = PDEFVAL
data['s2n_w'] = DEFVAL
data['chi2per'] = PDEFVAL
self.data=data
def set_fracdev_stuff(self):
self['fracdev_grid']=self.get('fracdev_grid',None)
self['fracdev_prior'] = self.get('fracdev_prior',None)