def get_chunk_ranges(**keys):
import meds
nper=int(keys['nper'])
keys['ftype']='meds'
meds_file=get_input_file(**keys)
m=meds.MEDS(meds_file)
ntotal=m.size
nchunks = ntotal/nper
nleft = ntotal % nper
if nleft != 0:
nchunks += 1
low=[]
high=[]
for i in xrange(nchunks):
low_i = i*nper
# minus one becuase it is inclusive
if i == (nchunks-1) and nleft != 0:
high_i = low_i + nleft -1
else:
high_i = low_i + nper - 1
low.append( low_i )
high.append( high_i )
return low,high
def _load_meds_files(self):
"""
Load all listed meds files
"""
self.meds_list = []
self.meds_meta_list = []
for i, funexp in enumerate(self.meds_files):
f = os.path.expandvars(funexp)
print('band %d meds: %s' % (i, f))
medsi = meds.MEDS(f)
medsi_meta = medsi.get_meta()
if i == 0:
nobj_tot = medsi.size
else:
nobj = medsi.size
if nobj != nobj_tot:
raise ValueError("mismatch in meds "
"sizes: %d/%d" % (nobj_tot, nobj))
self.meds_list.append(medsi)
self.meds_meta_list.append(medsi_meta)
verify_meds(self.meds_list)
self.nobj_tot = self.meds_list[0].size
def load_data(self):
"""
read or load all data
"""
print("loading:", self['meds_file'])
self.meds = meds.MEDS(self['meds_file'])
print("reading:", self['truth_file'])
self.truth = fitsio.read(self['truth_file'])
print("loading:", self['psf_file'])
self.psf_obj = fitsio.FITS(self['psf_file'])
self.npsf = len(self.psf_obj)
def _load_meds_files(self):
"""
Load all listed meds files
We check the flags indicated by image_flags2check. the saved
flags are 0 or IMAGE_FLAGS_SET
"""
self.meds_list = []
self.meds_meta_list = []
self.all_image_flags = []
for i, funexp in enumerate(self.meds_files):
f = os.path.expandvars(funexp)
print('band %d meds: %s' % (i, f))
medsi = meds.MEDS(f)
medsi_meta = medsi.get_meta()
image_info = medsi.get_image_info()
if i == 0:
nobj_tot = medsi.size
else:
nobj = medsi.size
if nobj != nobj_tot:
raise ValueError("mismatch in meds "
"sizes: %d/%d" % (nobj_tot, nobj))
self.meds_list.append(medsi)
self.meds_meta_list.append(medsi_meta)
image_flags = image_info['image_flags'].astype('i8')
if 'replacement_flags' in self.conf and self.conf[
'replacement_flags'] is not None and image_flags.size > 1:
print(" replacing image flags")
image_flags[1:] = \
self._get_replacement_flags(image_info['image_path'][1:])
# now we reduce the flags to zero or IMAGE_FLAGS_SET
# copy out and check image flags just for cutouts
cimage_flags = image_flags[1:].copy()
w, = numpy.where(
(cimage_flags & self.conf['image_flags2check']) != 0)
print(" flags set for: %d/%d" % (w.size, cimage_flags.size))
cimage_flags[:] = 0
if w.size > 0:
cimage_flags[w] = IMAGE_FLAGS_SET
# copy back in reduced flags
image_flags[1:] = cimage_flags
self.all_image_flags.append(image_flags)
self.nobj_tot = self.meds_list[0].size
def _load_meds(self):
"""
Load all listed meds files
"""
print(self['meds_file'])
self.meds = meds.MEDS(self['meds_file'])
self.meds_meta = self.meds.get_meta()
self.nobj_tot = self.meds.size
self.mindex = 0
jtmp = self._get_jacobian([0.0, 0.0])
self['pixel_scale'] = jtmp.get_scale()
print("pixel scale:", self['pixel_scale'])
def __init__(self,
ngmix_config,
meds_files,
psf_map_files=None,
walltime='48:00',
walltime_e='4:00',
ncores=1,
missing=False,
blind=False,
skip_errors=False,
clobber=False,
seed=None):
self.ngmix_config = os.path.abspath(
os.path.expanduser(os.path.expandvars(ngmix_config)))
if psf_map_files is not None:
self.psf_map_files = [os.path.abspath(f) for f in psf_map_files]
else:
self.psf_map_files = None
self.missing = missing
self.walltime = walltime
self.walltime_e = walltime_e
self.ncores = ncores
self.blind = blind
self.skip_errors = skip_errors
self.clobber = clobber
self._load_config()
self._make_rng(seed=seed)
self.meds_files = [os.path.abspath(mf) for mf in meds_files]
meds_list = [meds.MEDS(f) for f in self.meds_files]
try:
ngmixer.imageio.medsio.verify_meds(meds_list)
except AttributeError:
print("ngmixer.imageio.medsio.verify_meds not found - "
"probably this version is too old")
pass
self.meds_string = ' '.join(self.meds_files)
self.info = files.get_meds_info(meds_files[0])
self._set_files()
def __init__(self, meds_info=None):
# The meds file groups data together by source.
# To run, we need to make an ObservationList object for each source.
# Each ObservationList has a single Observation frome each epoch.
# Each Observation object needs an image, a psf, a weightmap, and a wcs, expressed as a Jacobian object.
# The MEDS object has methods to get all of these.
# Once that's done, we create a Bootstrapper, populate it with initial guesses
# and priors, and run it.
# The results get compiled into a catalog (numpy structured array with named fields),
# which is either written to a .fits table or returned in memory.
#
# self.metcal will hold result of metacalibration
# self.gal_results is the result of a simple fit to the galaxy shape within metcal bootstrapper
self.medsObj = meds.MEDS(meds_info['meds_file'])
self.catalog = self.medsObj.get_cat()
self.metacal = None
self.gal_fit = None
def _setup_work_files(self):
"""
Set up local, possibly sub-range meds files
"""
self.meds_files_full = self.meds_files
self.fof_file_full = self.fof_file
self.extracted = None
if 'correct_meds' in self.conf and self.fof_range is None:
with meds.MEDS(self.meds_files_full[0]) as m:
self.fof_range = [0, m.size - 1]
if self.fof_range is not None:
extracted = self._get_sub()
meds_files = [ex.sub_file for ex in extracted if ex is not None]
if extracted[-1] is not None:
self.fof_file = extracted[-1].sub_file
meds_files = meds_files[:-1]
self.meds_files = meds_files
self.extracted = extracted
def get_fof_ranges(self, files):
if self['model_nbrs']:
fofs = fitsio.read(files['fof_file'])
num_fofs = len(np.unique(fofs['fofid']))
else:
m = meds.MEDS(files['meds_files'][0])
num_fofs = m.size
m.close()
nchunks = num_fofs / self['num_fofs_per_chunk']
if nchunks * self['num_fofs_per_chunk'] < num_fofs:
nchunks += 1
fof_ranges = []
for chunk in xrange(nchunks):
sr = chunk * self['num_fofs_per_chunk']
sp = sr + self['num_fofs_per_chunk'] - 1
if sp >= num_fofs:
sp = num_fofs - 1
fof_ranges.append([sr, sp])
return fof_ranges
def __enter__(self):
# build the meds list on entry
tname = self.coadd_run.split('_')[-1]
self.meds_list = []
for band in self.bands:
meds_path = os.path.join(
'${DESDATA}', 'meds', self.meds_version, self.coadd_run,
'%s-%s-meds-%s.fits.fz' % (tname, band, self.meds_version))
meds_path = os.path.expandvars(meds_path)
if not os.path.exists(meds_path):
raise ValueError("MEDS file '%s' not found!" % meds_path)
try:
m = meds.MEDS(meds_path)
except:
raise ValueError("MEDS file '%s' could not be read!" %
meds_path)
self.meds_list.append(m)
return self
def search_se_snr():
local_Hmeds = './fiducial_H158_2285117.fits'
truth = fio.FITS('/hpc/group/cosmology/phy-lsst/my137/roman_H158_final/g1002/truth/fiducial_lensing_galaxia_g1002_truth_gal.fits')[-1]
m_H158 = meds.MEDS(local_Hmeds)
indices_H = np.arange(len(m_H158['number'][:]))
roman_H158_psfs = get_psf_SCA('H158')
oversample = 1
metacal_keys=['noshear', '1p', '1m', '2p', '2m']
snr=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('snr',float)])
for i,ii in enumerate(indices_H): # looping through all the objects in meds file.
if i%100==0:
print('object number ',i)
ind = m_H158['number'][ii]
t = truth[ind]
sca_Hlist = m_H158[ii]['sca'] # List of SCAs for the same object in multiple observations.
m2_H158_coadd = [roman_H158_psfs[j-1] for j in sca_Hlist[:m_H158['ncutout'][i]]]
obs_Hlist,psf_Hlist,included_H,w_H = get_exp_list_coadd(m_H158,ii,oversample,m2=m2_H158_coadd)
res_ = measure_shape_metacal(obs_Hlist, t['size'], method='bootstrap', fracdev=t['bflux'],use_e=[t['int_e1'],t['int_e2']])
snr['ind'][i] = i # different from ind defined.
snr['snr'][i] = res_['noshear']['s2n']
np.savetxt('snr_example.txt', snr)
print(min(snr['snr']), snr['ind'][snr['snr'].index(min(snr['snr']))])
print(max(snr['snr']), snr['ind'][snr['snr'].index(max(snr['snr']))])
def test_meds():
"""
Create two objects, each with three exposures. Save them to a MEDS file.
Load the MEDS file. Compare the created objects with the one read by MEDS.
"""
# initialise empty MultiExposureObject list
objlist = []
# we will be using 2 objects for testing, each with 3 cutouts
n_obj_test = 2
n_cut_test = 3
# set the image size
box_size = 32
# first obj
img11 = galsim.Image(box_size, box_size, init_value=111)
img12 = galsim.Image(box_size, box_size, init_value=112)
img13 = galsim.Image(box_size, box_size, init_value=113)
seg11 = galsim.Image(box_size, box_size, init_value=121)
seg12 = galsim.Image(box_size, box_size, init_value=122)
seg13 = galsim.Image(box_size, box_size, init_value=123)
wth11 = galsim.Image(box_size, box_size, init_value=131)
wth12 = galsim.Image(box_size, box_size, init_value=132)
wth13 = galsim.Image(box_size, box_size, init_value=133)
psf11 = galsim.Image(box_size, box_size, init_value=141)
psf12 = galsim.Image(box_size, box_size, init_value=142)
psf13 = galsim.Image(box_size, box_size, init_value=143)
dudx = 11.1
dudy = 11.2
dvdx = 11.3
dvdy = 11.4
x0 = 11.5
y0 = 11.6
wcs11 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
dudx = 12.1
dudy = 12.2
dvdx = 12.3
dvdy = 12.4
x0 = 12.5
y0 = 12.6
wcs12 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
dudx = 13.1
dudy = 13.2
dvdx = 13.3
dvdy = 13.4
x0 = 13.5
y0 = 13.6
wcs13 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
# create lists
images = [img11, img12, img13]
weight = [wth11, wth12, wth13]
seg = [seg11, seg12, seg13]
psf = [psf11, psf12, psf13]
wcs = [wcs11, wcs12, wcs13]
# create object
obj1 = galsim.des.MultiExposureObject(images=images,
weight=weight,
seg=seg,
psf=psf,
wcs=wcs,
id=1)
# second obj
img21 = galsim.Image(box_size, box_size, init_value=211)
img22 = galsim.Image(box_size, box_size, init_value=212)
img23 = galsim.Image(box_size, box_size, init_value=213)
seg21 = galsim.Image(box_size, box_size, init_value=221)
seg22 = galsim.Image(box_size, box_size, init_value=222)
seg23 = galsim.Image(box_size, box_size, init_value=223)
wth21 = galsim.Image(box_size, box_size, init_value=231)
wth22 = galsim.Image(box_size, box_size, init_value=332)
wth23 = galsim.Image(box_size, box_size, init_value=333)
psf21 = galsim.Image(box_size, box_size, init_value=241)
psf22 = galsim.Image(box_size, box_size, init_value=342)
psf23 = galsim.Image(box_size, box_size, init_value=343)
dudx = 21.1
dudy = 21.2
dvdx = 21.3
dvdy = 21.4
x0 = 21.5
y0 = 21.6
wcs21 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
dudx = 22.1
dudy = 22.2
dvdx = 22.3
dvdy = 22.4
x0 = 22.5
y0 = 22.6
wcs22 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
dudx = 23.1
dudy = 23.2
dvdx = 23.3
dvdy = 23.4
x0 = 23.5
y0 = 23.6
wcs23 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy,
galsim.PositionD(x0, y0))
# create lists
images = [img21, img22, img23]
weight = [wth21, wth22, wth23]
seg = [seg21, seg22, seg23]
psf = [psf21, psf22, psf23]
wcs = [wcs21, wcs22, wcs23]
# create object
# This time put the wcs in the image and get it there.
img21.wcs = wcs21
img22.wcs = wcs22
img23.wcs = wcs23
obj2 = galsim.des.MultiExposureObject(images=images,
weight=weight,
seg=seg,
psf=psf,
id=2)
# create an object list
objlist = [obj1, obj2]
# save objects to MEDS file
filename_meds = 'output/test_meds.fits'
galsim.des.WriteMEDS(objlist, filename_meds, clobber=True)
# Note that while there are no tests prior to this, the above still checks for
# syntax errors in the meds creation software, so it's still worth running as part
# of the normal unit tests.
# But for the rest of the tests, we'll use the meds module to make sure our code
# stays in sync with any changes there.
try:
import meds
except ImportError:
print('Failed to import meds. Unable to do tests of meds file.')
return
try:
# Meds will import this, so check for this too.
import fitsio
except ImportError:
print('Failed to import fitsio. Unable to do tests of meds file.')
return
# Run meds module's validate function
try:
meds.util.validate_meds(filename_meds)
except AttributeError:
print(
'Seems to be the wrong meds package. Unable to do tests of meds file.'
)
return
m = meds.MEDS(filename_meds)
# Check the image_info extension:
ref_info = meds.util.get_image_info_dtype(1)
info = m.get_image_info()
for name, dt in ref_info:
dt = numpy.dtype(dt)
print(name, dt, info.dtype[name], dt.char, info.dtype[name].char)
assert name in info.dtype.names, "column %s not present in image_info extension" % name
assert dt.char == info.dtype[
name].char, "column %s is the wrong type" % name
# Check the basic structure of the object_data extension
cat = m.get_cat()
ref_data = meds.util.get_meds_output_dtype(1)
for tup in ref_data:
# Some of these tuples have 3 items, not 2. The last two are the full dtype tuple.
name = tup[0]
if len(tup) == 2:
dt = tup[1]
else:
dt = tup[1:]
dt = numpy.dtype(dt)
print(name, dt, cat.dtype[name], dt.char, cat.dtype[name].char)
assert name in cat.dtype.names, "column %s not present in object_data extension" % name
assert dt.char == cat.dtype[
name].char, "column %s is the wrong type" % name
# Check that we have the right number of objects.
n_obj = len(cat)
print('number of objects is %d' % n_obj)
numpy.testing.assert_equal(n_obj,
n_obj_test,
err_msg="MEDS file has wrong number of objects")
# loop over objects and exposures - test get_cutout
for iobj in range(n_obj):
# check ID is correct
numpy.testing.assert_equal(
cat['id'][iobj],
iobj + 1,
err_msg="MEDS file has wrong id for object %d" % iobj)
# get number of cutouts and check if it's right
n_cut = cat['ncutout'][iobj]
numpy.testing.assert_equal(
n_cut,
n_cut_test,
err_msg="MEDS file has wrong ncutout for object %d" % iobj)
# loop over cutouts
for icut in range(n_cut):
# get the images etc to compare with originals
img = m.get_cutout(iobj, icut, type='image')
wth = m.get_cutout(iobj, icut, type='weight')
seg = m.get_cutout(iobj, icut, type='seg')
psf = m.get_psf(iobj, icut)
wcs_meds = m.get_jacobian(iobj, icut)
# Note: col == x, row == y.
wcs_array_meds = numpy.array([
wcs_meds['dudcol'], wcs_meds['dudrow'], wcs_meds['dvdcol'],
wcs_meds['dvdrow'], wcs_meds['col0'], wcs_meds['row0']
])
# compare
numpy.testing.assert_array_equal(
img,
objlist[iobj].images[icut].array,
err_msg="MEDS cutout has wrong img for object %d" % iobj)
numpy.testing.assert_array_equal(
wth,
objlist[iobj].weight[icut].array,
err_msg="MEDS cutout has wrong wth for object %d" % iobj)
numpy.testing.assert_array_equal(
seg,
objlist[iobj].seg[icut].array,
err_msg="MEDS cutout has wrong seg for object %d" % iobj)
numpy.testing.assert_array_equal(
psf,
objlist[iobj].psf[icut].array,
err_msg="MEDS cutout has wrong psf for object %d" % iobj)
wcs_orig = objlist[iobj].wcs[icut]
wcs_array_orig = numpy.array([
wcs_orig.dudx, wcs_orig.dudy, wcs_orig.dvdx, wcs_orig.dvdy,
wcs_orig.origin.x, wcs_orig.origin.y
])
numpy.testing.assert_array_equal(
wcs_array_meds,
wcs_array_orig,
err_msg="MEDS cutout has wrong wcs for object %d" % iobj)
# get the mosaic to compare with originals
img = m.get_mosaic(iobj, type='image')
wth = m.get_mosaic(iobj, type='weight')
seg = m.get_mosaic(iobj, type='seg')
# There is currently no get_mosaic option for the psfs.
#psf = m.get_mosaic( iobj, type='psf')
psf = numpy.concatenate(
[m.get_psf(iobj, icut) for icut in range(n_cut)], axis=0)
# get the concatenated images - create the true mosaic
true_mosaic_img = numpy.concatenate(
[x.array for x in objlist[iobj].images], axis=0)
true_mosaic_wth = numpy.concatenate(
[x.array for x in objlist[iobj].weight], axis=0)
true_mosaic_seg = numpy.concatenate(
[x.array for x in objlist[iobj].seg], axis=0)
true_mosaic_psf = numpy.concatenate(
[x.array for x in objlist[iobj].psf], axis=0)
# compare
numpy.testing.assert_array_equal(
true_mosaic_img,
img,
err_msg="MEDS mosaic has wrong img for object %d" % iobj)
numpy.testing.assert_array_equal(
true_mosaic_wth,
wth,
err_msg="MEDS mosaic has wrong wth for object %d" % iobj)
numpy.testing.assert_array_equal(
true_mosaic_seg,
seg,
err_msg="MEDS mosaic has wrong seg for object %d" % iobj)
numpy.testing.assert_array_equal(
true_mosaic_psf,
psf,
err_msg="MEDS mosaic has wrong psf for object %d" % iobj)
annular -c"X_IMAGE Y_IMAGE g1_Rinv g2_Rinv" -s 100 -e 2200 -n 11 mcal-ngmix2-zreal_debug.fiat 3505 2340 > mcal-ngmix1-shallow.Rinv.annular
annular -c"X_IMAGE Y_IMAGE g1 g2" -s 120 -e 2200 -n 9 fitvd-optics-jitter-exp.fiat 3505 2340 > fitvd_optics_jitter.annular
python annular_jmac.py fitvd-optics-jitter-exp.csv X_IMAGE Y_IMAGE g1 g2
########################################################################
##
## Debugging T/PSF/etc issues in metacal
## (this is basically ngmix_fit_superbit, copied over for ipython sesh)
##
########################################################################
medsObj=meds.MEDS('/Users/jemcclea/Research/SuperBIT/superbit-ngmix/scripts/output-mock/mock_superbit.meds')
index = 1260
psf = medsObj.get_cutout(index,0,type='psf')
im = medsObj.get_cutout(index,0,type='image')
weight = medsObj.get_cutout(index,0,type='weight')
jj = medsObj.get_jacobian(index,0)
jac = ngmix.Jacobian(row=jj['row0'],col=jj['col0'],dvdrow = jj['dvdrow'],dvdcol=jj['dvdcol'],dudrow=jj['dudrow'],dudcol=jj['dudcol'])
psf_noise = 1e-6
this_psf = psf + 1e-6*np.random.randn(psf.shape[0],psf.shape[1])
this_psf_weight = np.zeros_like(this_psf) + 1./psf_noise**2
psfObs = ngmix.observation.Observation(this_psf,weight = this_psf_weight, jacobian = jac) # can I add a kw for no_pix here???
imageObs = ngmix.observation.Observation(image=im,weight = weight, jacobian = jac, psf = psfObs)
def test_meds_config():
"""
Create a meds file from a config and compare with a manual creation.
"""
# Some parameters:
if __name__ == '__main__':
nobj = 5
n_per_obj = 8
else:
nobj = 5
n_per_obj = 3
file_name = 'output/test_meds.fits'
stamp_size = 64
pixel_scale = 0.26
seed = 5757231
g1 = -0.17
g2 = 0.23
# The config dict to write some images to a MEDS file
config = {
'gal': {
'type': 'Sersic',
'n': 1.3,
'half_light_radius': {
'type': 'Sequence',
'first': 0.7,
'step': 0.1,
'repeat': n_per_obj
},
'shear': {
'type': 'G1G2',
'g1': g1,
'g2': g2
},
},
'psf': {
'type': 'Moffat',
'beta': 2.9,
'fwhm': 0.7
},
'image': {
'pixel_scale': pixel_scale,
'offset': {
'type': 'XY',
'x': -0.17,
'y': 0.23
},
'size': stamp_size
},
'output': {
'type': 'MEDS',
'nobjects': nobj,
'nstamps_per_object': n_per_obj,
'file_name': file_name
}
}
import logging
logging.basicConfig(format="%(message)s",
level=logging.WARN,
stream=sys.stdout)
logger = logging.getLogger('test_meds_config')
galsim.config.Process(config, logger=logger)
# Now repeat, making a separate file for each
config['gal']['half_light_radius'] = {
'type': 'Sequence',
'first': 0.7,
'step': 0.1,
'index_key': 'file_num'
}
config['output'] = {
'type': 'Fits',
'nfiles': nobj,
'weight': {
'hdu': 1
},
'badpix': {
'hdu': 2
},
'psf': {
'hdu': 3
},
'dir': 'output',
'file_name': {
'type': 'NumberedFile',
'root': 'test_meds'
}
}
config['image'] = {
'type': 'Tiled',
'nx_tiles': 1,
'ny_tiles': n_per_obj,
'pixel_scale': pixel_scale,
'offset': config['image']['offset'],
'stamp_size': stamp_size
}
galsim.config.Process(config, logger=logger)
try:
import meds
import fitsio
except ImportError:
print(
'Failed to import either meds or fitsio. Unable to do tests of meds file.'
)
return
try:
m = meds.MEDS(file_name)
except AttributeError:
print(
'Seems to be the wrong meds package. Unable to do tests of meds file.'
)
return
assert m.size == nobj
# Test that the images made as meds mosaics match the ones written to the separate fits files.
cat = m.get_cat()
for iobj in range(nobj):
ref_file = os.path.join('output', 'test_meds%d.fits' % iobj)
ref_im = galsim.fits.read(ref_file)
meds_im_array = m.get_mosaic(iobj)
# Just for reference. If you get an error, you can open this file with ds9.
alt_meds_file = os.path.join('output', 'test_alt_meds%d.fits' % iobj)
alt_meds_im = galsim.Image(meds_im_array)
alt_meds_im.write(alt_meds_file)
numpy.testing.assert_array_equal(
ref_im.array,
meds_im_array,
err_msg="config MEDS has wrong im for object %d" % iobj)
meds_wt_array = m.get_mosaic(iobj, type='weight')
ref_wt_im = galsim.fits.read(ref_file, hdu=1)
numpy.testing.assert_array_equal(
ref_wt_im.array,
meds_wt_array,
err_msg="config MEDS has wrong wt for object %d" % iobj)
meds_seg_array = m.get_mosaic(iobj, type='seg')
ref_seg_im = galsim.fits.read(ref_file, hdu=2)
ref_seg_im = 1 - ref_seg_im # The seg mag is 1 where badpix == 0
numpy.testing.assert_array_equal(
ref_seg_im.array,
meds_seg_array,
err_msg="config MEDS has wrong seg for object %d" % iobj)
meds_psf_array = numpy.concatenate(
[m.get_psf(iobj, icut) for icut in range(n_per_obj)], axis=0)
ref_psf_im = galsim.fits.read(ref_file, hdu=3)
numpy.testing.assert_array_equal(
ref_psf_im.array,
meds_psf_array,
err_msg="config MEDS has wrong psf for object %d" % iobj)
# Check that the various positions and sizes are set correctly.
info = m.get_image_info()
for iobj in range(nobj):
n_cut = cat['ncutout'][iobj]
for icut in range(n_cut):
# This should be stamp_size
box_size = cat['box_size'][iobj]
numpy.testing.assert_almost_equal(box_size, stamp_size)
# These should be (box_size-1)/2
center = galsim.PositionD((box_size - 1.) / 2.,
(box_size - 1.) / 2.)
cutout_row = cat['cutout_row'][iobj][icut]
cutout_col = cat['cutout_col'][iobj][icut]
print('nominal position = ', cutout_col, cutout_row)
numpy.testing.assert_almost_equal(cutout_col, center.x)
numpy.testing.assert_almost_equal(cutout_row, center.y)
# The col0 and row0 here should be the same.
wcs_meds = m.get_jacobian(iobj, icut)
numpy.testing.assert_almost_equal(wcs_meds['col0'],
(box_size - 1) / 2.)
numpy.testing.assert_almost_equal(wcs_meds['row0'],
(box_size - 1) / 2.)
# The centroid should be (roughly) at the nominal center + offset
img = m.get_cutout(iobj, icut, type='image')
x, y = numpy.meshgrid(range(img.shape[1]), range(img.shape[0]))
itot = numpy.sum(img)
ix = numpy.sum(x * img)
iy = numpy.sum(y * img)
print('centroid = ', ix / itot, iy / itot)
offset = galsim.PositionD(-0.17, 0.23)
print('center + offset = ', center + offset)
numpy.testing.assert_almost_equal(ix / itot, (center + offset).x,
decimal=2)
numpy.testing.assert_almost_equal(iy / itot, (center + offset).y,
decimal=2)
# The orig positions are irrelevant and should be 0.
orig_row = cat['orig_row'][iobj][icut]
orig_col = cat['orig_col'][iobj][icut]
orig_start_row = cat['orig_start_row'][iobj][icut]
orig_start_col = cat['orig_start_col'][iobj][icut]
numpy.testing.assert_almost_equal(orig_col, 0.)
numpy.testing.assert_almost_equal(orig_row, 0.)
numpy.testing.assert_almost_equal(orig_start_col, 0.)
numpy.testing.assert_almost_equal(orig_start_row, 0.)
# This should be also be 0.
numpy.testing.assert_almost_equal(info['position_offset'], 0.)
def test_meds_config():
"""
Create a meds file from a config and compare with a manual creation.
"""
# Some parameters:
nobj = 5
n_per_obj = 8
file_name = 'test_meds.fits'
stamp_size = 32
pixel_scale = 0.26
seed = 5757231
g1 = -0.17
g2 = 0.23
# The config dict to write some images to a MEDS file
config = {
'gal' : { 'type' : 'Sersic',
'n' : 3,
'half_light_radius' : { 'type' : 'Sequence', 'first' : 1.7, 'step' : 0.2,
'repeat' : n_per_obj },
'shear' : { 'type' : 'G1G2', 'g1' : g1, 'g2' : g2 },
'shift' : { 'type' : 'XY' , 'x' : 1, 'y' : 2}
},
'psf' : { 'type' : 'Moffat', 'beta' : 2.9, 'fwhm' : 0.7 },
'image' : { 'pixel_scale' : pixel_scale,
'random_seed' : seed,
'size' : stamp_size },
'output' : { 'type' : 'des_meds',
'nobjects' : nobj,
'nstamps_per_object' : n_per_obj,
'file_name' : file_name
}
}
import logging
logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger('test_meds_config')
galsim.config.Process(config, logger=logger)
# Now repeat, making a separate file for each
config['gal']['half_light_radius'] = { 'type' : 'Sequence', 'first' : 1.7, 'step' : 0.2,
'index' : 'file_num' }
config['output'] = { 'type' : 'Fits',
'nfiles' : nobj,
'file_name' : { 'type' : 'NumberedFile', 'root' : 'test_meds' }
}
config['image'] = { 'type' : 'Tiled',
'nx_tiles' : 1,
'ny_tiles' : n_per_obj,
'pixel_scale' : pixel_scale,
'random_seed' : seed,
'stamp_size' : stamp_size
}
galsim.config.Process(config, logger=logger)
# test functions in des_meds.py
print 'reading %s' % file_name
import meds
m = meds.MEDS(file_name)
print 'number of objects is %d' % m.size
assert m.size == nobj
# get the catalog
cat = m.get_cat()
# loop over objects and exposures - test get_cutout
for iobj in range(nobj):
ref_file = 'test_meds%d.fits' % iobj
ref_im = galsim.fits.read(ref_file)
meds_im_array = m.get_mosaic(iobj)
alt_meds_file = 'test_alt_meds%d.fits' % iobj
alt_meds_im = galsim.Image(meds_im_array)
alt_meds_im.write(alt_meds_file)
numpy.testing.assert_array_equal(ref_im.array, meds_im_array)
meds_wt_array = m.get_mosaic(iobj, type='weight')
meds_seg_array = m.get_mosaic(iobj, type='seg')
print 'all asserts succeeded'
def test_meds():
"""
Create two objects, each with two exposures. Save them to a MEDS file.
Load the MEDS file. Compare the created objects with the one read by MEDS.
"""
# initialise empty MultiExposureObject list
objlist = []
# we will be using 2 objects for testing, each with 2 cutouts
n_obj_test = 2
n_cut_test = 2
# set the image size
box_size = 32
# first obj
img11 = galsim.Image(box_size, box_size, init_value=111)
img12 = galsim.Image(box_size, box_size, init_value=112)
seg11 = galsim.Image(box_size, box_size, init_value=121)
seg12 = galsim.Image(box_size, box_size, init_value=122)
wth11 = galsim.Image(box_size, box_size, init_value=131)
wth12 = galsim.Image(box_size, box_size, init_value=132)
dudx = 11.1; dudy = 11.2; dvdx = 11.3; dvdy = 11.4; x0 = 11.5; y0 = 11.6;
wcs11 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, galsim.PositionD(x0, y0))
dudx = 12.1; dudy = 12.2; dvdx = 12.3; dvdy = 12.4; x0 = 12.5; y0 = 12.6;
wcs12 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, galsim.PositionD(x0, y0))
# create lists
images = [img11, img12]
weights = [wth11, wth12]
segs = [seg11, seg12]
wcs = [wcs11, wcs12]
# create object
obj1 = galsim.des.MultiExposureObject(images=images, weights=weights, segs=segs, wcs=wcs, id=1)
# second obj
img21 = galsim.Image(box_size, box_size, init_value=211)
img22 = galsim.Image(box_size, box_size, init_value=212)
seg21 = galsim.Image(box_size, box_size, init_value=221)
seg22 = galsim.Image(box_size, box_size, init_value=222)
wth21 = galsim.Image(box_size, box_size, init_value=231)
wth22 = galsim.Image(box_size, box_size, init_value=332)
dudx = 21.1; dudy = 21.2; dvdx = 21.3; dvdy = 21.4; x0 = 21.5; y0 = 21.6;
wcs21 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, galsim.PositionD(x0, y0))
dudx = 22.1; dudy = 22.2; dvdx = 22.3; dvdy = 22.4; x0 = 22.5; y0 = 22.6;
wcs22 = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, galsim.PositionD(x0, y0))
# create lists
images = [img21, img22]
weights = [wth21, wth22]
segs = [seg21, seg22]
wcs = [wcs22, wcs22]
# create object
obj2 = galsim.des.MultiExposureObject(images=images, weights=weights, segs=segs, wcs=wcs, id=2)
# create an object list
objlist = [obj1, obj2]
# save objects to MEDS file
filename_meds = 'test_meds.fits'
galsim.des.write_meds(filename_meds, objlist, clobber=True)
print 'wrote MEDS file %s ' % filename_meds
# test functions in des_meds.py
print 'reading %s' % filename_meds
import meds
m = meds.MEDS(filename_meds)
# get the catalog
cat = m.get_cat()
# get number of objects
n_obj = len(cat)
# check if the number of objects is correct
numpy.testing.assert_equal(n_obj,n_obj_test)
print 'number of objects is %d' % n_obj
print 'testing if loaded images are the same as original images'
# loop over objects and exposures - test get_cutout
for iobj in range(n_obj):
# check ID is correct
numpy.testing.assert_equal(cat['id'][iobj], iobj+1)
# get number of cutouts and check if it's right
n_cut = cat['ncutout'][iobj]
numpy.testing.assert_equal(n_cut,n_cut_test)
# loop over cutouts
for icut in range(n_cut):
# get the images etc to compare with originals
img = m.get_cutout( iobj, icut, type='image')
wth = m.get_cutout( iobj, icut, type='weight')
seg = m.get_cutout( iobj, icut, type='seg')
wcs_meds = m.get_jacobian(iobj, icut)
wcs_array_meds= numpy.array( [ wcs_meds['dudrow'], wcs_meds['dudcol'],
wcs_meds['dvdrow'], wcs_meds['dvdcol'], wcs_meds['row0'],
wcs_meds['col0'] ] )
# compare
numpy.testing.assert_array_equal(img, objlist[iobj].images[icut].array)
numpy.testing.assert_array_equal(wth, objlist[iobj].weights[icut].array)
numpy.testing.assert_array_equal(seg, objlist[iobj].segs[icut].array)
wcs_orig = objlist[iobj].wcs[icut]
wcs_array_orig = numpy.array(
[ wcs_orig.dudx, wcs_orig.dudy, wcs_orig.dvdx, wcs_orig.dvdy,
wcs_orig.origin.x, wcs_orig.origin.y ])
numpy.testing.assert_array_equal(wcs_array_meds, wcs_array_orig)
print 'test passed get_cutout obj=%d icut=%d' % (iobj, icut)
# loop over objects - test get_mosaic
for iobj in range(n_obj):
# get the mosaic to compare with originals
img = m.get_mosaic( iobj, type='image')
wth = m.get_mosaic( iobj, type='weight')
seg = m.get_mosaic( iobj, type='seg')
# get the concatenated images - create the true mosaic
true_mosaic_img = numpy.concatenate([x.array for x in objlist[iobj].images], axis=0)
true_mosaic_wth = numpy.concatenate([x.array for x in objlist[iobj].weights], axis=0)
true_mosaic_seg = numpy.concatenate([x.array for x in objlist[iobj].segs], axis=0)
# compare
numpy.testing.assert_array_equal(true_mosaic_img, img)
numpy.testing.assert_array_equal(true_mosaic_wth, wth)
numpy.testing.assert_array_equal(true_mosaic_seg, seg)
print 'test passed get_mosaic for obj=%d' % (iobj)
print 'all asserts succeeded'
def multiband_coadd():
local_Hmeds = './fiducial_H158_2285117.fits'
local_Jmeds = './fiducial_J129_2285117.fits'
local_Fmeds = './fiducial_F184_2285117.fits'
truth = fio.FITS(
'/hpc/group/cosmology/phy-lsst/my137/roman_H158/g1002/truth/fiducial_lensing_galaxia_g1002_truth_gal.fits'
)[-1]
m_H158 = meds.MEDS(local_Hmeds)
m_J129 = meds.MEDS(local_Jmeds)
m_F184 = meds.MEDS(local_Fmeds)
indices_H = np.arange(len(m_H158['number'][:]))
indices_J = np.arange(len(m_J129['number'][:]))
indices_F = np.arange(len(m_F184['number'][:]))
roman_H158_psfs = get_psf_SCA('H158')
roman_J129_psfs = get_psf_SCA('J129')
roman_F184_psfs = get_psf_SCA('F184')
oversample = 1
metacal_keys = ['noshear', '1p', '1m', '2p', '2m']
res_noshear = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float),
('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_1p = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_1m = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_2p = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_2m = np.zeros(len(m_H158['number'][:]),
dtype=[('ind', int), ('ra', float), ('dec', float),
('flags', int), ('coadd_px', float),
('coadd_py', float), ('coadd_flux', float),
('coadd_snr', float), ('coadd_e1', float),
('coadd_e2', float), ('coadd_hlr', float),
('coadd_psf_e1', float), ('coadd_psf_e2', float),
('coadd_psf_T', float)])
res_tot = [res_noshear, res_1p, res_1m, res_2p, res_2m]
for i, ii in enumerate(
indices_H): # looping through all the objects in meds file.
if i % 100 == 0:
print('object number ', i)
ind = m_H158['number'][ii]
t = truth[ind]
sca_Hlist = m_H158[ii][
'sca'] # List of SCAs for the same object in multiple observations.
sca_Jlist = m_J129[ii]['sca']
sca_Flist = m_F184[ii]['sca']
m2_H158_coadd = [
roman_H158_psfs[j - 1] for j in sca_Hlist[:m_H158['ncutout'][i]]
]
m2_J129_coadd = [
roman_J129_psfs[j - 1] for j in sca_Jlist[:m_J129['ncutout'][i]]
]
m2_F184_coadd = [
roman_F184_psfs[j - 1] for j in sca_Flist[:m_F184['ncutout'][i]]
]
obs_Hlist, psf_Hlist, included_H, w_H = get_exp_list_coadd(
m_H158, ii, m2=m2_H158_coadd)
coadd_H = psc.Coadder(obs_Hlist, flat_wcs=True).coadd_obs
coadd_H.psf.image[
coadd_H.psf.image < 0] = 0 # set negative pixels to zero.
coadd_H.set_meta({'offset_pixels': None, 'file_id': None})
obs_Jlist, psf_Jlist, included_J, w_J = get_exp_list_coadd(
m_J129, ii, m2=m2_J129_coadd)
coadd_J = psc.Coadder(obs_Jlist, flat_wcs=True).coadd_obs
coadd_J.psf.image[
coadd_J.psf.image < 0] = 0 # set negative pixels to zero.
coadd_J.set_meta({'offset_pixels': None, 'file_id': None})
obs_Flist, psf_Flist, included_F, w_F = get_exp_list_coadd(
m_F184, ii, m2=m2_F184_coadd)
coadd_F = psc.Coadder(obs_Flist, flat_wcs=True).coadd_obs
coadd_F.psf.image[
coadd_F.psf.image < 0] = 0 # set negative pixels to zero.
coadd_F.set_meta({'offset_pixels': None, 'file_id': None})
coadd = [coadd_H, coadd_J, coadd_F]
mb_obs_list = MultiBandObsList()
#coadd = [coadd_H]
for band in range(3):
obs_list = ObsList()
new_coadd_psf_block = block_reduce(coadd[band].psf.image,
block_size=(4, 4),
func=np.sum)
new_coadd_psf_jacob = Jacobian(
row=15.5,
col=15.5,
dvdrow=(coadd[band].psf.jacobian.dvdrow * oversample),
dvdcol=(coadd[band].psf.jacobian.dvdcol * oversample),
dudrow=(coadd[band].psf.jacobian.dudrow * oversample),
dudcol=(coadd[band].psf.jacobian.dudcol * oversample))
coadd_psf_obs = Observation(new_coadd_psf_block,
jacobian=new_coadd_psf_jacob,
meta={
'offset_pixels': None,
'file_id': None
})
coadd[band].psf = coadd_psf_obs
obs_list.append(coadd[band])
mb_obs_list.append(obs_list)
iteration = 0
for key in metacal_keys:
res_tot[iteration]['ind'][i] = ind
res_tot[iteration]['ra'][i] = t['ra']
res_tot[iteration]['dec'][i] = t['dec']
iteration += 1
#print(i, t['size'], mb_obs_list[0][0].image.sum(), mb_obs_list[1][0].image.sum(), mb_obs_list[2][0].image.sum())
res_ = measure_shape_metacal_multiband(
mb_obs_list,
t['size'],
method='bootstrap',
fracdev=t['bflux'],
use_e=[t['int_e1'], t['int_e2']])
iteration = 0
for key in metacal_keys:
if res_ == 0:
res_tot[iteration]['ind'][i] = 0
elif res_[key]['flags'] == 0:
res_tot[iteration]['coadd_px'][i] = res_[key]['pars'][0]
res_tot[iteration]['coadd_py'][i] = res_[key]['pars'][1]
res_tot[iteration]['coadd_snr'][i] = res_[key]['s2n']
res_tot[iteration]['coadd_e1'][i] = res_[key]['pars'][2]
res_tot[iteration]['coadd_e2'][i] = res_[key]['pars'][3]
res_tot[iteration]['coadd_hlr'][i] = res_[key]['pars'][4]
iteration += 1
mask = res_tot[0]['ind'] != 0
print(len(res_tot[0]), len(res_tot[0][mask]))
#print(res_['noshear'].dtype.names)
print('done')
cen_prior,
g_prior,
T_prior,
F_prior)
return priors
########################################################################
##
## Debugging T/PSF/etc issues in metacal
## (this is basically ngmix_fit_superbit, copied over for ipython sesh)
##
########################################################################
medsObj=meds.MEDS('/Users/jemcclea/Research/SuperBIT/superbit-ngmix/scripts/cluster3-debug/opticsSigmaJitter_noDilate/cluster3_debug_2hr.meds')
index=12010
psf = medsObj.get_cutout(index,20,type='psf')
im = medsObj.get_cutout(index,20,type='image')
weight = medsObj.get_cutout(index,20,type='weight')
plt.figure()
plt.imshow(im)
plt.figure()
plt.imshow(psf)
jj = medsObj.get_jacobian(index,0)
jac = ngmix.Jacobian(row=jj['row0'],col=jj['col0'],dvdrow = jj['dvdrow'],dvdcol=jj['dvdcol'],dudrow=jj['dudrow'],dudcol=jj['dudcol'])
psf_noise = 1e-6
this_psf = psf + 1e-6*np.random.randn(psf.shape[0],psf.shape[1])
this_psf_weight = np.zeros_like(this_psf) + 1./psf_noise**2
def make_multiband_coadd_stamp():
oversample = 4
H = './fiducial_H158_2285117.fits'
J = './fiducial_J129_2285117.fits'
F = './fiducial_F184_2285117.fits'
truth = fio.FITS('/hpc/group/cosmology/phy-lsst/my137/roman_H158/g1002/truth/fiducial_lensing_galaxia_g1002_truth_gal.fits')[-1]
m_H158 = meds.MEDS(H)
m_J129 = meds.MEDS(J)
m_F184 = meds.MEDS(F)
indices_H = np.arange(len(m_H158['number'][:]))
indices_J = np.arange(len(m_J129['number'][:]))
indices_F = np.arange(len(m_F184['number'][:]))
roman_H158_psfs = get_psf_SCA('H158')
roman_J129_psfs = get_psf_SCA('J129')
roman_F184_psfs = get_psf_SCA('F184')
for i,ii in enumerate(indices_H):
if i%100==0:
print('made it to object',i)
try_save = False
ind = m_H158['number'][ii]
t = truth[ind]
if (ind not in m_J129['number']) or (ind not in m_F184['number']):
continue
print(i)
# if i==13:
sca_Hlist = m_H158[ii]['sca'] # List of SCAs for the same object in multiple observations.
ii_J = m_J129[m_J129['number']==ind]['id'][0]
sca_Jlist = m_J129[ii_J]['sca']
m2_H158_coadd = [roman_H158_psfs[j-1] for j in sca_Hlist[:m_H158['ncutout'][i]]]
m2_J129_coadd = [roman_J129_psfs[j-1] for j in sca_Jlist[:m_J129['ncutout'][ii_J]]]
ii_F = m_F184[m_F184['number']==ind]['id'][0]
sca_Flist = m_F184[ii_F]['sca']
m2_F184_coadd = [roman_F184_psfs[j-1] for j in sca_Flist[:m_F184['ncutout'][ii_F]]]
obs_Hlist,psf_Hlist,included_H,w_H = get_exp_list_coadd(m_H158,ii,oversample,m2=m2_H158_coadd)
obs_Jlist,psf_Jlist,included_J,w_J = get_exp_list_coadd(m_J129,ii_J,oversample,m2=m2_J129_coadd)
obs_Flist,psf_Flist,included_F,w_F = get_exp_list_coadd(m_F184,ii_F,oversample,m2=m2_F184_coadd)
# check if masking is less than 20%
if len(obs_Hlist)==0 or len(obs_Jlist)==0 or len(obs_Flist)==0:
continue
coadd_H = psc.Coadder(obs_Hlist,flat_wcs=True).coadd_obs
coadd_H.psf.image[coadd_H.psf.image<0] = 0 # set negative pixels to zero.
coadd_H.psf.set_meta({'offset_pixels':None,'file_id':None})
coadd_H.set_meta({'offset_pixels':None,'file_id':None})
coadd_J = psc.Coadder(obs_Jlist,flat_wcs=True).coadd_obs
coadd_J.psf.image[coadd_J.psf.image<0] = 0 # set negative pixels to zero.
coadd_J.psf.set_meta({'offset_pixels':None,'file_id':None})
coadd_J.set_meta({'offset_pixels':None,'file_id':None})
coadd_F = psc.Coadder(obs_Flist,flat_wcs=True).coadd_obs
coadd_F.psf.image[coadd_F.psf.image<0] = 0 # set negative pixels to zero.
coadd_F.psf.set_meta({'offset_pixels':None,'file_id':None})
coadd_F.set_meta({'offset_pixels':None,'file_id':None})
obs_list = ObsList()
multiband = [coadd_H, coadd_J, coadd_F]
for f in range(3):
obs_list.append(multiband[f])
multiband_coadd = psc.Coadder(obs_list,flat_wcs=True).coadd_obs
print('single snr', get_snr(obs_Jlist), get_snr(obs_Hlist), get_snr(obs_Flist))
print('coadd snr', get_snr([coadd_J]), get_snr([coadd_H]), get_snr([coadd_F]))
print('final', get_snr(obs_list))
def single_vs_coadd_images():
local_Hmeds = './fiducial_F184_2285117.fits'
truth = fio.FITS('/hpc/group/cosmology/phy-lsst/my137/roman_F184/g1002/truth/fiducial_lensing_galaxia_g1002_truth_gal.fits')[-1]
m_H158 = meds.MEDS(local_Hmeds)
indices_H = np.arange(len(m_H158['number'][:]))
roman_H158_psfs = get_psf_SCA('F184')
oversample = 4
metacal_keys=['noshear', '1p', '1m', '2p', '2m']
res_noshear=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('ra',float), ('dec',float), ('flags',int),('int_e1',float), ('int_e2',float),('coadd_px',float), ('coadd_py',float), ('coadd_flux',float), ('coadd_snr',float), ('coadd_e1',float), ('coadd_e2',float), ('coadd_hlr',float),('coadd_psf_e1',float), ('coadd_psf_e2',float), ('coadd_psf_T',float)])
res_1p=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('ra',float), ('dec',float), ('flags',int),('int_e1',float), ('int_e2',float),('coadd_px',float), ('coadd_py',float), ('coadd_flux',float), ('coadd_snr',float), ('coadd_e1',float), ('coadd_e2',float), ('coadd_hlr',float),('coadd_psf_e1',float), ('coadd_psf_e2',float), ('coadd_psf_T',float)])
res_1m=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('ra',float), ('dec',float), ('flags',int),('int_e1',float), ('int_e2',float),('coadd_px',float), ('coadd_py',float), ('coadd_flux',float), ('coadd_snr',float), ('coadd_e1',float), ('coadd_e2',float), ('coadd_hlr',float),('coadd_psf_e1',float), ('coadd_psf_e2',float), ('coadd_psf_T',float)])
res_2p=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('ra',float), ('dec',float), ('flags',int),('int_e1',float), ('int_e2',float),('coadd_px',float), ('coadd_py',float), ('coadd_flux',float), ('coadd_snr',float), ('coadd_e1',float), ('coadd_e2',float), ('coadd_hlr',float),('coadd_psf_e1',float), ('coadd_psf_e2',float), ('coadd_psf_T',float)])
res_2m=np.zeros(len(m_H158['number'][:]),dtype=[('ind',int), ('ra',float), ('dec',float), ('flags',int),('int_e1',float), ('int_e2',float),('coadd_px',float), ('coadd_py',float), ('coadd_flux',float), ('coadd_snr',float), ('coadd_e1',float), ('coadd_e2',float), ('coadd_hlr',float),('coadd_psf_e1',float), ('coadd_psf_e2',float), ('coadd_psf_T',float)])
res_tot=[res_noshear, res_1p, res_1m, res_2p, res_2m]
for i,ii in enumerate(indices_H): # looping through all the objects in meds file.
if i%100==0:
print('object number ',i)
# if i not in [1,600]:
# continue
ind = m_H158['number'][ii]
t = truth[ind]
sca_Hlist = m_H158[ii]['sca'] # List of SCAs for the same object in multiple observations.
m2_H158_coadd = [roman_H158_psfs[j-1] for j in sca_Hlist[:m_H158['ncutout'][i]]]
obs_Hlist,psf_Hlist,included_H,w_H = get_exp_list_coadd(m_H158,ii,oversample,m2=m2_H158_coadd)
s2n_test = get_snr(obs_Hlist)
# if i in [1,600]: #in [ 309, 444, 622, 644, 854, 1070, 1282, 1529]:
# for l in range(len(obs_Hlist)):
# #print(i, obs_Hlist[l].jacobian, obs_Hlist[l].psf.jacobian)
# print(i, obs_Hlist[l].weight)
# np.savetxt('/hpc/group/cosmology/masaya/wfirst_simulation/paper/single_image_oversample4_08scaling_'+str(i)+'_'+str(l)+'.txt', obs_Hlist[l].image)
# np.savetxt('/hpc/group/cosmology/masaya/wfirst_simulation/paper/single_psf_oversample4_08scaling_'+str(i)+'.txt', obs_Hlist[0].psf.image)
coadd_H = psc.Coadder(obs_Hlist,flat_wcs=True).coadd_obs
coadd_H.psf.image[coadd_H.psf.image<0] = 0 # set negative pixels to zero.
coadd_H.set_meta({'offset_pixels':None,'file_id':None})
obs_list = ObsList()
if oversample == 4:
new_coadd_psf_block = block_reduce(coadd_H.psf.image, block_size=(4,4), func=np.sum)
new_coadd_psf_jacob = Jacobian( row=15.5,
col=15.5,
dvdrow=(coadd_H.psf.jacobian.dvdrow*oversample),
dvdcol=(coadd_H.psf.jacobian.dvdcol*oversample),
dudrow=(coadd_H.psf.jacobian.dudrow*oversample),
dudcol=(coadd_H.psf.jacobian.dudcol*oversample))
coadd_psf_obs = Observation(new_coadd_psf_block, jacobian=new_coadd_psf_jacob, meta={'offset_pixels':None,'file_id':None})
coadd_H.psf = coadd_psf_obs
obs_list.append(coadd_H)
s2n_coadd = get_snr(obs_list)
# if i in [1,600]:
# print(i, coadd_H.weight)
#np.savetxt('/hpc/group/cosmology/masaya/wfirst_simulation/paper/coadd_image_oversample4_08scaling_'+str(i)+'.txt', coadd_H.image)
# np.savetxt('/hpc/group/cosmology/masaya/wfirst_simulation/paper/coadd_weight_oversample4_08scaling_'+str(i)+'.txt', coadd_H.weight)
#np.savetxt('/hpc/group/cosmology/masaya/wfirst_simulation/paper/coadd_psf_over-downsample4_08scaling_'+str(i)+'.txt', coadd_H.psf.image)
iteration=0
for key in metacal_keys:
res_tot[iteration]['ind'][i] = ind
res_tot[iteration]['ra'][i] = t['ra']
res_tot[iteration]['dec'][i] = t['dec']
res_tot[iteration]['int_e1'][i] = t['int_e1']
res_tot[iteration]['int_e2'][i] = t['int_e2']
iteration+=1
res_ = measure_shape_metacal(obs_list, t['size'], method='bootstrap', fracdev=t['bflux'],use_e=[t['int_e1'],t['int_e2']])
print('signal to noise test', i, s2n_test, s2n_coadd, res_['noshear']['s2n_r'])
# if res_[key]['s2n'] > 1e7:
# print('coadd snr', res_[key]['s2n'])
# np.savetxt('large_coadd_snr_image.txt', coadd_H.image)
iteration=0
for key in metacal_keys:
if res_==0:
res_tot[iteration]['ind'][i] = 0
elif res_[key]['flags']==0:
res_tot[iteration]['coadd_px'][i] = res_[key]['pars'][0]
res_tot[iteration]['coadd_py'][i] = res_[key]['pars'][1]
res_tot[iteration]['coadd_snr'][i] = res_[key]['s2n']
res_tot[iteration]['coadd_e1'][i] = res_[key]['pars'][2]
res_tot[iteration]['coadd_e2'][i] = res_[key]['pars'][3]
res_tot[iteration]['coadd_hlr'][i] = res_[key]['pars'][4]
iteration+=1
mask=res_tot[0]['ind']!=0
print(len(res_tot[0]), len(res_tot[0][mask]))
#print(res_['noshear'].dtype.names)
print('done')
import meds
import sys
import matplotlib.pyplot as plt
import numpy as np
if __name__ == "__main__":
filename = sys.argv[1]
# create a MEDS object for the given MEDS file
m = meds.MEDS(filename)
# open log file
log = open('testDESMEDS.log', 'w')
log.write('MEDS file name: {}\n'.format(filename))
# print the number of objects
nobj = m.size
log.write('number of objects: {}\n'.format(nobj))
# read a cutout for object 35, cutout index 2 (0-indexed)
index = 35
cutout_index = 2
image = m.get_cutout(index, cutout_index)
# get other image types
seg = m.get_cutout(index, cutout_index, type='seg')
wt = m.get_cutout(index, cutout_index, type='weight')
mask = m.get_cutout(index, cutout_index, type='bmask')
# get a python list of all cutouts for this object
