def transform(boxsize, opt=None):
"""Dummy im3shape transform, setting the MCMC chain started in the centre of the stamp."""
if opt is None:
options = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
transform = structs.Transform_struct()
transform.ra0 = 0.0 # these are not used in this case
transform.dec0 = 0.0
transform.cosdec0 = 0.0
transform.x0 = boxsize / 2
transform.y0 = boxsize / 2
print "starting chain with centroid position (%d %d)" % (transform.x0,
transform.y0)
dudcol = 0.27
dudrow = -1e-7
dvdcol = -1e-7
dvdrow = 0.27
B = np.array([[dudcol, dudrow], [dvdcol, dvdrow]])
A = np.linalg.inv(B)
transform.A[0][0] = A[0, 0]
transform.A[0][1] = A[0, 1]
transform.A[1][0] = A[1, 0]
transform.A[1][1] = A[1, 1]
#print 'transform det = ', np.linalg.det(B)
#print 'A = ', A
return transform
def run(galaxy_image,
psf_image,
save=None,
show=False,
weights=[[None]],
opt=None,
ncols=1,
col=1,
return_cat=False,
flat_background=0,
noise=0,
check=False):
if opt is None:
opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
boxsize = galaxy_image.array.shape[0]
opt.stamp_size = boxsize
if weights[0][0] is None:
wt = np.ones((boxsize, boxsize))
else:
wt = weights
trans = [transform(boxsize, opt)]
if noise == 0:
noise = np.zeros_like(wt)
else:
noise = noise * (
np.random.rand(wt.size).reshape(wt.shape[0], wt.shape[1]) - 0.5)
result, best_img, images, weights = p3s.analyze_multiexposure(
[galaxy_image.array + flat_background + noise], [psf_image], [wt],
trans,
opt,
ID=3000000,
bands="?")
if show:
cmap = plt.get_cmap("jet")
mosaic = np.vstack((galaxy_image.array, best_img))
plt.subplot(2, ncols, col)
plt.imshow(galaxy_image.array, interpolation="none", cmap=cmap)
plt.subplot(2, ncols, ncols + col)
plt.imshow(best_img, interpolation="none", cmap=cmap)
plt.subplots_adjust(hspace=0)
if save is not None:
np.savetxt(save + "/model_tm.txt", best_img)
np.savetxt(save + "/image_tm.txt", galaxy_image.array)
np.savetxt(save + "/weights_tm.txt", wt)
if check:
sel = check_cuts(result)
print "Object passes info_flag cuts:", sel
return result.get_params(), result, check
if return_cat:
return result.get_params(), result
else:
return result.get_params()
def setup(self):
self.object_mask = np.loadtxt("/home/samuroff/mask_template.txt")
# WCS
wcs_path = "/share/des/disc2/y1/OPS/coadd/20141118000051_DES0014-4414/coadd/DES0014-4414_r.fits.fz"
orig_col = 1000
orig_row = 1000
image_pos = galsim.PositionD(orig_col, orig_row)
self.wcs = galsim.FitsWCS(wcs_path)
# im3shape config
self.opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
def analyse1(self,
central_ellipticity,
dneigh=20,
central_flux=1912.0,
psf_size=3.7,
neighbour_flux=1912.0,
neighbour_size=3.2):
# Level 1 - loop over SNR
#-----------------------------------------------------------------------
self.m = {}
self.m[1] = []
self.m[2] = []
self.snr = []
self.dneigh = 20
self.central_flux = 1912.0
self.psf_size = 3.7
self.neighbour_flux = 1912.0
self.neighbour_size = 3.2
print "Toy model has a central galaxy with shape ", central_ellipticity
self.e1 = central_ellipticity[0]
self.e2 = central_ellipticity[1]
# Setup a dummy wcs
# We load it here to avoid too much unncessary io
# In fact it should provide a skeleton only, since we overwrite the Jacobian with an identity matrix
wcs_path = "/share/des/disc2/y1/OPS/coadd/20141118000051_DES0014-4414/coadd/DES0014-4414_r.fits.fz"
orig_col = 1000
orig_row = 1000
image_pos = galsim.PositionD(orig_col, orig_row)
self.wcs = galsim.FitsWCS(wcs_path)
self.opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
self.binning = np.logspace(1, 3.5, 12)
# Cycle over the central flux, as a proxy for SNR
for i, proxy in enumerate(self.binning):
print "Level 1 iteration: %d %f" % (i, proxy)
self.do_position_loop(proxy)
print "Done all loops"
def analyse1(
self,
central_data,
neighbour_data,
nreal=1000
): #, central_ellipticity, dneigh=20, central_flux=1912.0, psf_size=3.7, neighbour_flux=1912.0, neighbour_size=3.2, nrealisations=1000):
# Level 1 - loop over SNR
#-----------------------------------------------------------------------
# Initialise the random number generator
np.random.seed(self.random_seed)
self.object_mask = np.loadtxt("mask_template.txt")
m = {}
m[1] = []
m[2] = []
c = {}
c[1] = []
c[2] = []
snr = []
# Setup a dummy wcs
# We load it here to avoid too much unncessary io
# In fact it should provide a skeleton only, since we overwrite the Jacobian with an identity matrix
wcs_path = "/share/des/disc2/y1/OPS/coadd/20141118000051_DES0014-4414/coadd/DES0014-4414_r.fits.fz"
orig_col = 1000
orig_row = 1000
image_pos = galsim.PositionD(orig_col, orig_row)
self.wcs = galsim.FitsWCS(wcs_path)
self.opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
self.binning = np.logspace(1, 2.8, 12)
upper = self.binning[1:]
lower = self.binning[:-1]
# Cycle over the central flux, as a proxy for SNR
for i, limits in enumerate(zip(lower, upper)):
snr_min = limits[0]
snr_max = limits[1]
print "Level 1 iteration: %d SNR = %3.3f - %3.3f" % (
i + 1, snr_min, snr_max)
# Selection function for this bin
sel = (central_data.res["snr"] >
snr_min) & (central_data.res["snr"] < snr_max)
print "Will do %d realisations for this bin:" % nreal
De1 = []
De2 = []
sn = []
g1 = []
g2 = []
for j in xrange(nreal):
print "%d/%d" % (j + 1, nreal)
self.generate_central_realisation(central_data, sel)
self.generate_neighbour_realisation(neighbour_data,
central_data, sel)
# Draw this neighbour realisation repeatedly on a ring of angular positions
snr, e1, e2 = self.do_position_loop()
sn.append(snr)
De1.append(e1)
De2.append(e2)
g1.append(self.g[0])
g2.append(self.g[1])
data = np.zeros(len(g1),
dtype=[("e1", float), ("e2", float),
("true_g1", float), ("true_g2", float)])
data["e1"], data["e2"] = np.array(De1), np.array(De2)
data["true_g1"], data["true_g2"] = np.array(g1), np.array(g2)
bias = di.get_bias(data,
nbins=5,
names=["m11", "m22", "c11", "c22"],
binning="equal_number",
silent=True)
print bias
m[1].append(bias["m11"][0])
m[2].append(bias["m22"][0])
c[1].append(bias["c11"][0])
c[2].append(bias["c22"][0])
import pdb
pdb.set_trace()
print "Done all loops"
def setup_simple(boxsize=32,
model="disc",
upsample=1,
size=2,
flux=None,
shear=(0, 0),
neighbour=[np.inf, np.inf],
neighbour_size=2.0,
neighbour_flux=0.13,
neighbour_ellipticity=(0, 0),
psf_ellipticity=(0, 0),
psf_size=0.1,
wcs=None,
opt=None):
"""Basic function to construct a test galaxy for exploring shape biases."""
if opt is None:
opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
opt.stamp_size = boxsize
psfsize = (boxsize +
opt._struct.contents.padding) * opt._struct.contents.upsampling
upsampling = opt._struct.contents.upsampling
large_boxsize = boxsize * 7
gal1 = get_model(size=size,
type=model,
g1=shear[0],
g2=shear[1],
flux=flux)
pos1 = galsim.PositionD(0, 0)
# Add a tiny nominal PSF
psf, psf_image = get_fixed_gaussian_psf(opt, psf_size, psf_ellipticity[0],
psf_ellipticity[1], wcs)
#import pdb ; pdb.set_trace()
#galsim.Gaussian(fwhm=psf_size*0.27)
if wcs is None:
wcs_path = "/share/des/disc2/y1/OPS/coadd/20141118000051_DES0014-4414/coadd/DES0014-4414_r.fits.fz"
wcs = galsim.FitsWCS(wcs_path)
orig_col = 1000
orig_row = 1000
image_pos = galsim.PositionD(orig_col, orig_row)
local_wcs = wcs.local(image_pos)
local_wcs._dudx = 1.0
local_wcs._dudy = 0.0
local_wcs._dvdx = 0.0
local_wcs._dvdy = 1.0
A = np.array([[local_wcs._dudx, local_wcs._dudy],
[local_wcs._dvdx, local_wcs._dvdy]])
local_wcs._det = np.linalg.det(A)
pix = wcs.toWorld(galsim.Pixel(0.27), image_pos=image_pos)
obj1 = galsim.Convolve([gal1, psf, pix])
im1 = obj1.drawImage(wcs=local_wcs,
nx=large_boxsize,
ny=large_boxsize,
method='no_pixel')
im3 = obj1.drawImage(wcs=local_wcs,
nx=large_boxsize,
ny=large_boxsize,
method='no_pixel')
# And a neighbouring object if specified
if np.isfinite(np.array(neighbour)).all():
print "drawing neighbour at (%d, %d)" % (int(
neighbour[0]), int(neighbour[1]))
gal2 = galsim.Sersic(n=1, half_light_radius=neighbour_size)
gal2 = gal2.withFlux(neighbour_flux)
nshear = galsim.Shear(g1=neighbour_ellipticity[0],
g2=neighbour_ellipticity[1])
gal2 = gal2.shear(nshear)
obj2 = galsim.Convolve([gal2, psf, pix])
pos2 = galsim.PositionD(x=neighbour[0], y=neighbour[1])
im2 = obj2.drawImage(wcs=local_wcs,
nx=large_boxsize,
ny=large_boxsize,
method='no_pixel',
offset=pos2)
im3 += im2.array
trim = (large_boxsize - boxsize) / 2.
lim = galsim.BoundsI(xmin=trim,
xmax=large_boxsize - trim - 1,
ymin=trim,
ymax=large_boxsize - trim - 1)
del (wcs)
gc.collect()
return im3[lim].array, psf_image
def generate(self,
dneigh=20,
central_flux=1912.0,
psf_size=3.7,
neighbour_flux=1912.0,
neighbour_size=3.2,
vary="psf_e"):
# PSF leakage toy model
#-----------------------------------------------------------------------
g_theta = {}
theta = np.linspace(0, 2, 50) * np.pi
self.g = {}
self.g["e1"] = []
self.g["e2"] = []
ang = []
# Setup a dummy wcs
# We load it here to avoid too much unncessary io
# In fact it should provide a skeleton only, since we overwrite the Jacobian with an identity matrix
wcs_path = "/share/des/disc2/y1/OPS/coadd/20141118000051_DES0014-4414/coadd/DES0014-4414_r.fits.fz"
orig_col = 1000
orig_row = 1000
image_pos = galsim.PositionD(orig_col, orig_row)
wcs = galsim.FitsWCS(wcs_path)
opt = p3s.Options(
"/home/samuroff/shear_pipeline/end-to-end/end-to-end_code/config_files/im3shape/params_disc.ini"
)
binning = {
"psf_e1": np.linspace(-0.2, 0.2, 12),
"cent_e1": [-0.65, 0.65]
}
self.binning = binning[vary]
# Cycle over psf ellipticities
for e in self.binning:
if vary == "psf_e1":
pe = e
ce = 0
elif vary == "cent_e1":
pe = 0
ce = e
g_theta["e1"] = []
g_theta["e2"] = []
ang = []
# For each psf ellipticity average over a ring of neighbour positions
# For the moment we hardcode the other relevant parameters to their mean values from the simulation
for i, t in enumerate(theta):
x = dneigh * np.cos(t)
y = dneigh * np.sin(t)
gal, psf = setup_simple(boxsize=48,
shear=(ce, 0.0),
flux=central_flux,
psf_ellipticity=(pe, 0.0),
psf_size=psf_size,
neighbour_ellipticity=(0.0, 0.0),
neighbour=[x, y],
neighbour_flux=neighbour_flux,
neighbour_size=neighbour_size,
wcs=wcs,
opt=opt)
res = run(gal, psf, opt=opt)
g_theta["e1"].append(res.e1)
gal, psf = setup_simple(boxsize=48,
shear=(ce, 0.0),
flux=central_flux,
psf_ellipticity=(pe, 0.0),
psf_size=psf_size,
neighbour_ellipticity=(0.0, 0.0),
neighbour=[np.inf, np.inf],
neighbour_flux=neighbour_flux,
neighbour_size=neighbour_size,
wcs=wcs,
opt=opt)
res = run(gal, psf, opt=opt)
g_theta["e2"].append(res.e1)
ang.append(t)
self.g["e1"].append(np.mean(g_theta["e1"]))
self.g["e2"].append(np.mean(g_theta["e2"]))
for k in self.g.keys():
self.g[k] = np.array(self.g[k])
def main(args):
# load the options file
options = py3shape.Options(args.ini_filename)
options.validate()
#Get extra command line ini file options
if args.extra_options is not None:
for opt in args.extra_options:
key, val = opt.split('=')
setattr(options, key, val)
# Read in the FITS data
img_filename, img_ext = get_fits_extension(args.img_filename)
if args.weight_filename:
weight_filename, weight_ext = get_fits_extension(args.weight_filename)
weight_gs = galsim.fits.read(weight_filename, hdu=weight_ext)
if args.seg_filenames:
seg_imgs = []
for seg_file in args.seg_filenames:
seg_filename, seg_ext = get_fits_extension(seg_file)
seg_imgs.append(galsim.fits.read(seg_filename, hdu=seg_ext))
#Read in image
image_gs = galsim.fits.read(img_filename, hdu=img_ext)
img_data = image_gs.array
#Get wcs info:
wcs = image_gs.wcs
#Read psfex file
try:
psfexer = galsim.des.DES_PSFEx(args.psf_filename, args.img_filename)
except Exception:
logging.error('failed to read psf file %s', (args.psf_filename))
raise
#Read in catalog data
gal_cat = get_gal_cat(args)
# Create i3_image of certain stamp size
stamp_size = options.stamp_size
#overwrite the output filename
options.output_filename = args.out_filename
options.save_output = False
extra_cols = ['e1_sky', 'e2_sky']
if args.psf_props:
extra_cols += ['psf_fwhm', 'psf_e1', 'psf_e2']
if args.masking_type == 'all':
extra_cols += ['e1_nomask', 'e2_nomask', 'e1_uber', 'e2_uber']
#exclude following columns...this is messy, probably better to define new single epoch output object in im3shape...
excluded_cols = [
'exposure_x', 'exposure_y', 'exposure_e1', 'exposure_e2',
'exposure_chi2', 'mean_flux', 'exposure_residual_stdev'
]
output = py3shape.output.Output(args.out_filename,
options,
excluded_cols=excluded_cols)
extra_lines = [
'driver: Balrog/run_im3shape.py',
'ini file: %s' % (args.ini_filename, ),
'catalog file: %s' % (args.cat_filename, ),
'image file: %s' % (args.img_filename, ),
'psfex file: %s' % (args.psf_filename, ),
'first object: %s' % (args.first, ),
'last object: %s' % (args.last, )
]
output.write_header(include_radec=True,
extra_cols=extra_cols,
extra_lines=extra_lines)
#Write extra lines to info() log
for extra_line in extra_lines:
logging.info(extra_line)
#main galaxy loop
ID = gal_cat['ID']
X_IMAGE = gal_cat['X_IMAGE']
Y_IMAGE = gal_cat['Y_IMAGE']
RA, DEC = gal_cat['RA'], gal_cat['DEC']
if args.last == None or args.last > len(ID) - 1:
args.last = len(ID) - 1
# Loop over objects in catalog
logging.info('Analyzing %d/%d galaxies' % (min(
args.last - args.first + 1, len(gal_cat['ID'])), len(gal_cat['ID'])))
start_time = time.clock()
for i in range(args.first, args.last + 1):
# Read galaxy catalog entry
identifier = ID[i]
print identifier
xpos = X_IMAGE[i]
ypos = Y_IMAGE[i]
#Get image stamp
stamp_stuff = get_stamp(image_gs, X_IMAGE[i], Y_IMAGE[i],
options.stamp_size)
try:
stamp, x0, y0 = stamp_stuff
except TypeError:
#if stamp_stuff==1:
# logging.warning('galaxy %d stamp overlaps edge of image, skipping',identifier)
#else:
# logging.warning('failed to get stamp for galaxy %d, skipping',identifier)
continue
#Get weight stamp and mask stamp:
if args.weight_filename:
weight_stamp, _, _ = get_stamp(weight_gs, xpos, ypos,
options.stamp_size)
#Set negative values to zero
weight_stamp[weight_stamp < 0] = 0
else:
weight_stamp = None
if not args.masking_type == 'none':
#Get seg stamp data...decide what to do depending on how many seg files provided.
#If only one provided, this should be the seg map from the original image. Set all non-zero pixels in this image
#to -1, then create mask, that way they won't match ID of any detected simulated objects.
if len(seg_imgs) == 1:
seg_stamp, _, _ = get_stamp(seg_imgs[0], xpos, ypos,
options.stamp_size)
#np.set_printoptions(threshold=np.nan)
#print 'seg_stamp',seg_stamp
seg_stamp[(seg_stamp != 0)] = -1
mask = seg_to_mask_basic(identifier, seg_stamp)
mask_stamp = py3shape.Image(mask)
#If two provided, combine them in the following way:
if len(seg_imgs) == 2:
#np.set_printoptions(threshold=np.nan)
noassoc_seg_stamp, _, _ = get_stamp(seg_imgs[0], xpos, ypos,
options.stamp_size)
assoc_seg_stamp, _, _ = get_stamp(seg_imgs[1], xpos, ypos,
options.stamp_size)
#Find target object pixels in assoc stamp, and find which pixel value they overlap
#most with in noassoc stamp. What if this is zero? Can't see why this would happen if same SExtractor settings
#used for both seg maps, so for no just log a warning and skip object if this happens...
assoc_obj_inds = np.where(assoc_seg_stamp == identifier)
noassoc_seg_vals = noassoc_seg_stamp[assoc_obj_inds]
try:
noassoc_identifier = mode(noassoc_seg_vals)[0]
except UnboundLocalError:
logging.warning('No object found in seg map....skipping')
continue
#Set pixels in noassoc seg stamp with pixel value noassoc_identifier to identifier, and others to -1
noassoc_obj_inds = np.where(
noassoc_seg_stamp == noassoc_identifier)
noassoc_seg_stamp[(noassoc_seg_stamp != 0)] = -1
noassoc_seg_stamp[noassoc_obj_inds] = identifier
if args.masking_type == 'seg':
mask = seg_to_mask_basic(identifier, noassoc_seg_stamp)
mask_stamp = py3shape.Image(mask)
if args.masking_type == 'uberseg':
mask = seg_to_mask_uber(identifier, noassoc_seg_stamp)
mask_stamp = py3shape.Image(mask)
if args.masking_type == 'all':
mask = seg_to_mask_basic(identifier, noassoc_seg_stamp)
uberseg_mask = seg_to_mask_uber(identifier,
noassoc_seg_stamp)
mask_stamp = py3shape.Image(mask)
extra_mask_stamps = [py3shape.Image(uberseg_mask), None]
else:
mask_stamp = None
#np.set_printoptions(threshold=np.nan)
#print 'mask_stamp',mask
options.sersics_x0_start = x0
options.sersics_y0_start = y0
#print 'starting coords:',x0,y0
#Get position in stamp for starting position:
galaxy = py3shape.Image(stamp)
#Get psf image
print xpos, ypos
pos = galsim.PositionD(
float(xpos), float(ypos)
) #Not quite sure why I need float() here...but get galism error if not
psf_size = (options.stamp_size + options.padding) * options.upsampling
try:
psf = getPSFExarray(psfexer,
pos,
psf_size,
psf_size,
upsampling=options.upsampling)
psf_Image = py3shape.Image(psf)
except Exception:
logging.warning('failed to get psf for galaxy %d, skipping',
identifier)
continue
try:
result, model = py3shape.analyze(galaxy,
py3shape.Image(psf),
options,
weight=weight_stamp,
mask=mask_stamp,
ID=identifier)
except Exception, emsg:
print emsg
logging.error(
'im3shape failed for galaxy %d, with following error message:',
identifier)
logging.error(emsg)
continue
if args.masking_type == 'all':
result_nomask, _ = py3shape.analyze(galaxy,
psf_Image,
options,
weight=weight_stamp,
mask=extra_mask_stamps[1],
ID=identifier)
e1_nomask, e2_nomask = result_nomask.get_params(
).e1, result_nomask.get_params().e2
result_uber, _ = py3shape.analyze(galaxy,
psf_Image,
options,
weight=weight_stamp,
mask=extra_mask_stamps[0],
ID=identifier)
e1_uber, e2_uber = result_uber.get_params(
).e1, result_uber.get_params().e2
#Convert e's to sky coordinates...not totally sure about this function...
local_wcs = wcs.local(image_pos=pos)
e1_sky, e2_sky = convert_g_image2sky(local_wcs,
result.get_params().e1,
result.get_params().e2)
#Measure psf properties
psf_fwhm, psf_e1, psf_e2 = (get_psf_params([psf_Image],
options,
radialProfile=True,
hsm=False))[0]
if args.plot:
pylab.subplot(231)
pylab.imshow(stamp, origin='lower', interpolation='nearest')
pylab.subplot(232)
pylab.imshow(model, origin='lower', interpolation='nearest')
pylab.subplot(233)
pylab.imshow(stamp - (model) * stamp.sum(),
origin='lower',
interpolation='nearest')
pylab.subplot(234)
pylab.imshow(mask, origin='lower', interpolation='nearest')
if args.masking_type == 'all':
pylab.subplot(235)
pylab.imshow(uberseg_mask,
origin='lower',
interpolation='nearest')
pylab.show()
extra_output = [e1_sky, e2_sky]
if args.psf_props:
extra_output += [psf_fwhm, psf_e1, psf_e2]
if args.masking_type == 'all':
extra_output += [e1_nomask, e2_nomask, e1_uber, e2_uber]
output.write_row(result,
options,
psf,
extra_output=extra_output,
ra=RA[i],
dec=DEC[i])