def test_cosmos_deep():
"""Test the deep option of makeGalaxy"""
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
# Pick a random galaxy
# Turn off noise padding to make the comparisons more reliable.
gal_shallow = cat.makeGalaxy(index=17, gal_type='real', noise_pad_size=0)
print('gal_shallow = ', gal_shallow)
shallow_flux = gal_shallow.flux
shallow_hlr = gal_shallow.calculateHLR()
print('flux = ', shallow_flux)
print('hlr = ', shallow_hlr)
gal_deep = cat.makeGalaxy(index=17,
gal_type='real',
deep=True,
noise_pad_size=0)
print('gal_deep = ', gal_deep)
deep_flux = gal_deep.flux
deep_hlr = gal_deep.calculateHLR()
print('flux = ', deep_flux)
print('hlr = ', deep_hlr)
# Deep galaxy is fainter and smaller.
np.testing.assert_almost_equal(deep_flux / shallow_flux, 10.**(-0.6))
np.testing.assert_almost_equal(deep_hlr / shallow_hlr, 0.6)
# With samples other than 23.5, it raises a warning and doesn't do any scaling.
cat.use_sample = '25.2'
with assert_warns(galsim.GalSimWarning):
gal_not_deep = cat.makeGalaxy(index=17,
gal_type='real',
deep=True,
noise_pad_size=0)
assert gal_not_deep.flux == shallow_flux
assert gal_not_deep.calculateHLR() == shallow_hlr
cat.use_sample = 'user_defined'
with assert_warns(galsim.GalSimWarning):
gal_not_deep = cat.makeGalaxy(index=17,
gal_type='real',
deep=True,
noise_pad_size=0)
assert gal_not_deep.flux == shallow_flux
assert gal_not_deep.calculateHLR() == shallow_hlr
def test_cosmos_fluxnorm():
"""Check for flux normalization properties of COSMOSCatalog class."""
# Check that if we make a RealGalaxy catalog, and a COSMOSCatalog, and draw the real object, the
# fluxes should match very well. These correspond to 1s exposures.
test_ind = 54
rand_seed = 12345
cat = galsim.COSMOSCatalog(file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath, exclusion_level='none')
rgc = galsim.RealGalaxyCatalog(file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath)
final_psf = galsim.Airy(diam=1.2, lam=800.) # PSF twice as big as HST in F814W.
gal1 = cat.makeGalaxy(test_ind, gal_type='real', rng=galsim.BaseDeviate(rand_seed))
gal2 = galsim.RealGalaxy(rgc, index=test_ind, rng=galsim.BaseDeviate(rand_seed))
gal1 = galsim.Convolve(gal1, final_psf)
gal2 = galsim.Convolve(gal2, final_psf)
im1 = gal1.drawImage(scale=0.05)
im2 = gal2.drawImage(scale=0.05)
# Then check that if we draw a parametric representation that is achromatic, that the flux
# matches reasonably well (won't be exact because model isn't perfect).
gal1_param = cat.makeGalaxy(test_ind, gal_type='parametric', chromatic=False)
gal1_param_final = galsim.Convolve(gal1_param, final_psf)
im1_param = gal1_param_final.drawImage(scale=0.05)
# Then check the same for a chromatic parametric representation that is drawn into the same
# band.
bp_file = os.path.join(galsim.meta_data.share_dir, 'bandpasses', 'ACS_wfc_F814W.dat')
bandpass = galsim.Bandpass(bp_file, wave_type='nm').withZeropoint(25.94)#34.19)
gal1_chrom = cat.makeGalaxy(test_ind, gal_type='parametric', chromatic=True)
gal1_chrom = galsim.Convolve(gal1_chrom, final_psf)
im1_chrom = gal1_chrom.drawImage(bandpass, scale=0.05)
ref_val = [im1.array.sum(), im1.array.sum(), im1.array.sum()]
test_val = [im2.array.sum(), im1_param.array.sum(), im1_chrom.array.sum()]
np.testing.assert_allclose(ref_val, test_val, rtol=0.1,
err_msg='Flux normalization problem in COSMOS galaxies')
# Finally, check that the original COSMOS info is stored properly after transformations, for
# both Sersic galaxies (like galaxy 0 in the catalog) and the one that is gal1_param above.
gal0_param = cat.makeGalaxy(0, gal_type='parametric', chromatic=False)
assert hasattr(gal0_param.shear(g1=0.05).original, 'index'), \
'Sersic galaxy does not retain index information after transformation'
assert hasattr(gal1_param.shear(g1=0.05).original, 'index'), \
'Bulge+disk galaxy does not retain index information after transformation'
assert_raises(ValueError, galsim.COSMOSCatalog, 'real_galaxy_catalog_23.5_example.fits',
dir=datapath, exclusion_level='invalid')
def encode_examples(hparams, task_id=0, sample="25.2", cosmos_dir=None, exclusion_level="marginal", min_flux=0.):
"""
Generates and yields postage stamps obtained with GalSim.
"""
catalog = galsim.COSMOSCatalog(sample=sample, dir=cosmos_dir, exclusion_level=exclusion_level, min_flux=min_flux)
index = range(task_id * hparams.example_per_shard,
min((task_id+1) * hparams.example_per_shard, catalog.getNObjects()))
# Extracts additional information about the galaxies
cat_param = catalog.param_cat[catalog.orig_index]
# bparams = cat_param['bulgefit']
sparams = cat_param['sersicfit']
cat_param = append_fields(cat_param, 'sersic_q', sparams[:, 3])
cat_param = append_fields(cat_param, 'sersic_n', sparams[:, 2])
for ind in index:
# Draw a galaxy using GalSim, any kind of operation can be done here
gal = catalog.makeGalaxy(ind, noise_pad_size=hparams.img_len * hparams.pixel_scale)
psf = gal.original_psf
# Apply random rotation if requested
if hasattr(hparams, "rotation") and hparams.rotation:
rotation_angle = galsim.Angle(-np.random.rand() * 2 * np.pi,
galsim.radians)
gal = gal.rotate(rotation_angle)
psf = psf.rotate(rotation_angle)
# We save the corresponding attributes for this galaxy
if hasattr(hparams, 'attributes'):
params = cat_param[ind]
attributes = {k: params[k] for k in hparams.attributes}
else:
attributes = None
# Utility function encodes the postage stamp for serialized features
yield draw_and_encode_stamp(gal, psf,
stamp_size=hparams.img_len,
pixel_scale=hparams.pixel_scale,
attributes=attributes)
def from_file(cls, catalog_files, exclusion_level="marginal", verbose=False):
"""Constructs the catalog object from a file. It also places exclusion level cuts.
For more details: (https://galsim-developers.github.io/GalSim/_build/html/real_gal.html)
Args:
catalog_files(list): list containing the two paths to the COSMOS data. Please see
the tutorial page for more details
(https://lsstdesc.org/BlendingToolKit/tutorials.html#using-cosmos-galaxies).
exclusion_level(str): Level of additional cuts to make on the galaxies based on the
quality of postage stamp definition and/or parametric fit quality [beyond the
minimal cuts imposed when making the catalog - see Mandelbaum et
al. (2012, MNRAS, 420, 1518) for details].
Options:
- "none": No cuts.
- "bad_stamp": Apply cuts to eliminate galaxies that have failures in
postage stamp definition. These cuts may also eliminate a small
subset of the good postage stamps as well.
- "bad_fits": Apply cuts to eliminate galaxies that have failures in the
parametric fits. These cuts may also eliminate a small
subset of the good parametric fits as well.
- "marginal": Apply the above cuts, plus ones that eliminate some more
marginal cases.
Note that the _selection.fits file must be present in the same repo as the real
images catalog, Otherwise the "bad_stamp" and "marginal" cuts will fail
[default: "marginal"]
verbose: whether to print verbose info.
"""
galsim_catalog = galsim.COSMOSCatalog(catalog_files[0], exclusion_level=exclusion_level)
catalog_coord = astropy.table.Table.read(catalog_files[0])
catalog_fit = astropy.table.Table.read(catalog_files[1])
catalog_coord = catalog_coord[galsim_catalog.orig_index]
catalog_fit = catalog_fit[galsim_catalog.orig_index]
catalog = astropy.table.hstack([catalog_coord, catalog_fit])
return cls(catalog, galsim_catalog, verbose=verbose)
def main(argv):
# Where to find and output data.
path, filename = os.path.split(__file__)
outpath = os.path.abspath(os.path.join(path, "output/"))
# Just use a few galaxies, to save time. Note that we are going to put 4000 galaxy images into
# our big image, so if we have n_use=10, each galaxy will appear 400 times. Users who want a
# more interesting image with greater variation in the galaxy population can change `n_use` to
# something larger (but it should be <=100, the number of galaxies in this small example
# catalog). With 4000 galaxies in a 4k x 4k image with the WFIRST pixel scale, the effective
# galaxy number density is 74/arcmin^2. This is not the number density that is expected for a
# sample that is so bright (I<23.5) but it makes the image more visually interesting. One could
# think of it as what you'd get if you added up several images at once, making the images for a
# sample that is much deeper have the same S/N as that for an I<23.5 sample in a single image.
n_use = 10
n_tot = 4000
# Default is to use all filters. Specify e.g. 'YJH' to only do Y106, J129, and H158.
use_filters = None
# quick and dirty command line parsing.
for var in argv:
if var.startswith('data='): datapath = var[5:]
if var.startswith('out='): outpath = var[4:]
if var.startswith('nuse='): n_use = int(var[5:])
if var.startswith('ntot='): n_tot = int(var[5:])
if var.startswith('filters='): use_filters = var[8:].upper()
# Make output directory if not already present.
if not os.path.isdir(outpath):
os.mkdir(outpath)
# In non-script code, use getLogger(__name__) at module scope instead.
logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger("demo13")
# Initialize (pseudo-)random number generator.
random_seed = 123456
rng = galsim.BaseDeviate(random_seed)
# Generate a Poisson noise model.
poisson_noise = galsim.PoissonNoise(rng)
logger.info('Poisson noise model created.')
# Read in the WFIRST filters, setting an AB zeropoint appropriate for this telescope given its
# diameter and (since we didn't use any keyword arguments to modify this) using the typical
# exposure time for WFIRST images. By default, this routine truncates the parts of the
# bandpasses that are near 0 at the edges, and thins them by the default amount.
filters = wfirst.getBandpasses(AB_zeropoint=True)
logger.debug('Read in WFIRST imaging filters.')
logger.info('Reading from a parametric COSMOS catalog.')
# Read in a galaxy catalog - just a random subsample of 100 galaxies for F814W<23.5 from COSMOS.
cat_file_name = 'real_galaxy_catalog_23.5_example_fits.fits'
dir = 'data'
# Use the routine that can take COSMOS real or parametric galaxy information, and tell it we
# want parametric galaxies that represent an I<23.5 sample.
cat = galsim.COSMOSCatalog(cat_file_name, dir=dir, use_real=False)
logger.info('Read in %d galaxies from catalog'%cat.nobjects)
# Here we carry out the initial steps that are necessary to get a fully chromatic PSF. We use
# the getPSF() routine in the WFIRST module, which knows all about the telescope parameters
# (diameter, bandpasses, obscuration, etc.). Note that we arbitrarily choose a single SCA
# (Sensor Chip Assembly) rather than all of them, for faster calculations, and use a simple
# representation of the struts for faster calculations. To do a more exact calculation of the
# chromaticity and pupil plane configuration, remove the `approximate_struts` and the `n_waves`
# keyword from the call to getPSF():
use_SCA = 7 # This could be any number from 1...18
logger.info('Doing expensive pre-computation of PSF.')
t1 = time.time()
logger.setLevel(logging.DEBUG)
# Need to make a separate PSF for each filter. We are, however, ignoring the
# position-dependence of the PSF within each SCA, just using the PSF at the center of the SCA
# (default kwargs).
PSFs = {}
for filter_name, filter_ in filters.items():
logger.info('PSF pre-computation for SCA %d, filter %s.'%(use_SCA, filter_name))
PSFs[filter_name] = wfirst.getPSF(use_SCA, filter_name,
approximate_struts=True, n_waves=10, logger=logger)
logger.setLevel(logging.INFO)
t2 = time.time()
logger.info('Done PSF precomputation in %.1f seconds!'%(t2-t1))
# Define the size of the postage stamp that we use for each individual galaxy within the larger
# image, and for the PSF images.
stamp_size = 256
# We choose a particular (RA, dec) location on the sky for our observation.
ra_targ = 90.*galsim.degrees
dec_targ = -10.*galsim.degrees
targ_pos = galsim.CelestialCoord(ra=ra_targ, dec=dec_targ)
# Get the WCS for an observation at this position. We are not supplying a date, so the routine
# will assume it's the vernal equinox. We are also not supplying a position angle for the
# observatory, which means that it will just find the optimal one (the one that has the solar
# panels pointed most directly towards the Sun given this targ_pos and date). The output of
# this routine is a dict of WCS objects, one for each SCA. We then take the WCS for the SCA
# that we are using.
wcs_list = wfirst.getWCS(world_pos=targ_pos, SCAs=use_SCA)
wcs = wcs_list[use_SCA]
# We need to find the center position for this SCA. We'll tell it to give us a CelestialCoord
# corresponding to (X, Y) = (wfirst.n_pix/2, wfirst.n_pix/2).
SCA_cent_pos = wcs.toWorld(galsim.PositionD(wfirst.n_pix/2, wfirst.n_pix/2))
# We randomly distribute points in (X, Y) on the CCD.
# If we had a real galaxy catalog with positions in terms of RA, dec we could use wcs.toImage()
# to find where those objects should be in terms of (X, Y).
pos_rng = galsim.UniformDeviate(random_seed)
# Make a list of (X, Y, F814W magnitude, n_rot, flip) values.
# (X, Y) give the position of the galaxy centroid (or the center of the postage stamp into which
# we draw the galaxy) in the big image.
# F814W magnitudes are randomly drawn from the catalog, and are used to create a more realistic
# flux distribution for the galaxies instead of just having the 10 flux values for the galaxies
# we have chosen to draw.
# n_rot says how many 90 degree rotations to include for a given realization of each galaxy, so
# it doesn't appear completely identical each time we put it in the image.
# flip is a random number that will determine whether we include an x-y flip for this appearance
# of the galaxy or not.
x_stamp = []
y_stamp = []
mag_stamp = []
n_rot_stamp = []
flip_stamp = []
for i_gal in range(n_tot):
x_stamp.append(pos_rng()*wfirst.n_pix)
y_stamp.append(pos_rng()*wfirst.n_pix)
# Note that we could use wcs.toWorld() to get the (RA, dec) for these (x, y) positions. Or,
# if we had started with (RA, dec) positions, we could have used wcs.toImage() to get the
# CCD coordinates for those positions.
mag_stamp.append(cat.param_cat['mag_auto'][int(pos_rng()*cat.nobjects)])
n_rot_stamp.append(int(4*pos_rng()))
flip_stamp.append(pos_rng())
# Make the 2-component parametric GSObjects for each object, including chromaticity (roughly
# appropriate SEDs per galaxy component, at the appropriate galaxy redshift). Note that since
# the PSF is position-independent within the SCA, we can simply do the convolution with that PSF
# now instead of using a different one for each position. We also have to include the correct
# flux scaling: The catalog returns objects that would be observed by HST in 1 second
# exposures. So for our telescope we scale up by the relative area and exposure time. Note that
# what is important is the *effective* area after taking into account obscuration.
logger.info('Processing the objects in the catalog to get GSObject representations')
# Choose a random set of unique indices in the catalog (will be the same each time script is
# run, due to use of the same random seed):
rand_indices = []
while len(rand_indices)<n_use:
tmp_ind = int(pos_rng()*cat.nobjects)
if tmp_ind not in rand_indices:
rand_indices.append(tmp_ind)
obj_list = cat.makeGalaxy(rand_indices, chromatic=True, gal_type='parametric')
hst_eff_area = 2.4**2 * (1.-0.33**2)
wfirst_eff_area = galsim.wfirst.diameter**2 * (1.-galsim.wfirst.obscuration**2)
flux_scaling = (wfirst_eff_area/hst_eff_area) * wfirst.exptime
mag_list = []
for ind in range(len(obj_list)):
# First, let's check what magnitude this object has in F814W. We want to do this because
# (to inject some variety into our images) we are going to rescale the fluxes in all bands
# for different instances of this galaxy in the final image in order to get a reasonable S/N
# distribution. So we need to save the original magnitude in F814W, to compare with a
# randomly drawn one from the catalog. This is not something that most users would need to
# do.
mag_list.append(cat.param_cat['mag_auto'][cat.orig_index[rand_indices[ind]]])
# Calculate the sky level for each filter, and draw the PSF and the galaxies through the
# filters.
for filter_name, filter_ in filters.items():
if use_filters is not None and filter_name[0] not in use_filters:
logger.info('Skipping filter {0}.'.format(filter_name))
continue
logger.info('Beginning work for {0}.'.format(filter_name))
# Drawing PSF. Note that the PSF object intrinsically has a flat SED, so if we convolve it
# with a galaxy, it will properly take on the SED of the galaxy. For the sake of this demo,
# we will simply convolve with a 'star' that has a flat SED and unit flux in this band, so
# that the PSF image will be normalized to unit flux. This does mean that the PSF image
# being drawn here is not quite the right PSF for the galaxy. Indeed, the PSF for the
# galaxy effectively varies within it, since it differs for the bulge and the disk. To make
# a real image, one would have to choose SEDs for stars and convolve with a star that has a
# reasonable SED, but we just draw with a flat SED for this demo.
out_filename = os.path.join(outpath, 'demo13_PSF_{0}.fits'.format(filter_name))
# Generate a point source.
point = galsim.DeltaFunction(flux=1.)
# Use a flat SED here, but could use something else. A stellar SED for instance.
# Or a typical galaxy SED. Depending on your purpose for drawing the PSF.
star_sed = galsim.SED(lambda x:1, 'nm', 'flambda').withFlux(1.,filter_) # Give it unit flux in this filter.
star = galsim.Convolve(point*star_sed, PSFs[filter_name])
img_psf = galsim.ImageF(64,64)
star.drawImage(bandpass=filter_, image=img_psf, scale=wfirst.pixel_scale)
img_psf.write(out_filename)
logger.debug('Created PSF with flat SED for {0}-band'.format(filter_name))
# Set up the full image that will contain all the individual galaxy images, with information
# about WCS:
final_image = galsim.ImageF(wfirst.n_pix,wfirst.n_pix, wcs=wcs)
# Draw the galaxies into the image.
for i_gal in range(n_use):
logger.info('Drawing image for the object at row %d in the input catalog'%i_gal)
# We want to only draw the galaxy once (for speed), not over and over with different
# sub-pixel offsets. For this reason we ignore the sub-pixel offset entirely. Note
# that we are setting the postage stamp to have the average WFIRST pixel scale. This is
# simply an approximation for the purpose of speed; really, one should draw directly
# into final_image, which has the appropriate WCS for WFIRST. In that case, the image
# of the galaxy might look different in different parts of the detector due to the WCS
# (including distortion), and we would have to re-draw each time. To keep the demo
# relatively quick, we are just using the approximate average pixel scale and drawing
# once.
stamp = galsim.Image(stamp_size, stamp_size, scale=wfirst.pixel_scale)
# Convolve the chromatic galaxy and the chromatic PSF for this bandpass, and rescale flux.
final = galsim.Convolve(flux_scaling*obj_list[ind], PSFs[filter_name])
final.drawImage(filter_, image=stamp)
# Have to find where to place it:
for i_gal_use in range(i_gal*n_tot//n_use, (i_gal+1)*n_tot//n_use):
# Account for the fractional part of the position:
ix = int(math.floor(x_stamp[i_gal_use]+0.5))
iy = int(math.floor(y_stamp[i_gal_use]+0.5))
# We don't actually use this offset.
offset = galsim.PositionD(x_stamp[i_gal]-ix, y_stamp[i_gal]-iy)
# Create a nominal bound for the postage stamp given the integer part of the
# position.
stamp_bounds = galsim.BoundsI(ix-0.5*stamp_size, ix+0.5*stamp_size-1,
iy-0.5*stamp_size, iy+0.5*stamp_size-1)
# Find the overlapping bounds between the large image and the individual postage
# stamp.
bounds = stamp_bounds & final_image.bounds
# Just to inject a bit of variety into the image, so it isn't *quite* as obvious
# that we've repeated the same 10 objects over and over, we randomly rotate the
# postage stamp by some factor of 90 degrees and possibly include a random flip.
if flip_stamp[i_gal_use] > 0.5:
new_arr = numpy.ascontiguousarray(
numpy.rot90(stamp.array, n_rot_stamp[i_gal_use]))
else:
new_arr = numpy.ascontiguousarray(
numpy.fliplr(numpy.rot90(stamp.array, n_rot_stamp[i_gal_use])))
stamp_rot = galsim.Image(new_arr, scale=stamp.scale)
stamp_rot.setOrigin(galsim.PositionI(stamp_bounds.xmin, stamp_bounds.ymin))
# Rescale the flux to match that of a randomly chosen galaxy in the galaxy, but
# keeping the same SED as for this particular galaxy. This gives a bit more
# variety in the flux values and SNR of the galaxies in the image without having
# to render images of many more objects.
flux_scaling = 10**(-0.4*(mag_stamp[i_gal_use]-mag_list[i_gal]))
# Copy the image into the right place in the big image.
final_image[bounds] += flux_scaling * stamp_rot[bounds]
# Now we're done with the per-galaxy drawing for this image. The rest will be done for the
# entire image at once.
logger.info('Postage stamps of all galaxies drawn on a single big image for this filter.')
logger.info('Adding the sky level, noise and detector non-idealities.')
# First we get the amount of zodaical light for a position corresponding to the center of
# this SCA. The results are provided in units of e-/arcsec^2, using the default WFIRST
# exposure time since we did not explicitly specify one. Then we multiply this by a factor
# >1 to account for the amount of stray light that is expected. If we do not provide a date
# for the observation, then it will assume that it's the vernal equinox (sun at (0,0) in
# ecliptic coordinates) in 2025.
sky_level = wfirst.getSkyLevel(filters[filter_name], world_pos=SCA_cent_pos)
sky_level *= (1.0 + wfirst.stray_light_fraction)
# Make a image of the sky that takes into account the spatially variable pixel scale. Note
# that makeSkyImage() takes a bit of time. If you do not care about the variable pixel
# scale, you could simply compute an approximate sky level in e-/pix by multiplying
# sky_level by wfirst.pixel_scale**2, and add that to final_image.
sky_image = final_image.copy()
wcs.makeSkyImage(sky_image, sky_level)
# This image is in units of e-/pix. Finally we add the expected thermal backgrounds in this
# band. These are provided in e-/pix/s, so we have to multiply by the exposure time.
sky_image += wfirst.thermal_backgrounds[filter_name]*wfirst.exptime
# Adding sky level to the image.
final_image += sky_image
# Now that all sources of signal (from astronomical objects and background) have been added
# to the image, we can start adding noise and detector effects. There is a utility,
# galsim.wfirst.allDetectorEffects(), that can apply ALL implemented noise and detector
# effects in the proper order. Here we step through the process and explain these in a bit
# more detail without using that utility.
# First, we include the expected Poisson noise:
final_image.addNoise(poisson_noise)
# The subsequent steps account for the non-ideality of the detectors.
# 1) Reciprocity failure:
# Reciprocity, in the context of photography, is the inverse relationship between the
# incident flux (I) of a source object and the exposure time (t) required to produce a given
# response(p) in the detector, i.e., p = I*t. However, in NIR detectors, this relation does
# not hold always. The pixel response to a high flux is larger than its response to a low
# flux. This flux-dependent non-linearity is known as 'reciprocity failure', and the
# approximate amount of reciprocity failure for the WFIRST detectors is known, so we can
# include this detector effect in our images.
if diff_mode:
# Save the image before applying the transformation to see the difference
save_image = final_image.copy()
# If we had wanted to, we could have specified a different exposure time than the default
# one for WFIRST, but otherwise the following routine does not take any arguments.
wfirst.addReciprocityFailure(final_image)
logger.debug('Included reciprocity failure in {0}-band image'.format(filter_name))
if diff_mode:
# Isolate the changes due to reciprocity failure.
diff = final_image-save_image
out_filename = os.path.join(outpath,'demo13_RecipFail_{0}.fits'.format(filter_name))
final_image.write(out_filename)
out_filename = os.path.join(outpath,
'demo13_diff_RecipFail_{0}.fits'.format(filter_name))
diff.write(out_filename)
# At this point in the image generation process, an integer number of photons gets
# detected, hence we have to round the pixel values to integers:
final_image.quantize()
# 2) Adding dark current to the image:
# Even when the detector is unexposed to any radiation, the electron-hole pairs that
# are generated within the depletion region due to finite temperature are swept by the
# high electric field at the junction of the photodiode. This small reverse bias
# leakage current is referred to as 'dark current'. It is specified by the average
# number of electrons reaching the detectors per unit time and has an associated
# Poisson noise since it is a random event.
dark_current = wfirst.dark_current*wfirst.exptime
dark_noise = galsim.DeviateNoise(galsim.PoissonDeviate(rng, dark_current))
final_image.addNoise(dark_noise)
# NOTE: Sky level and dark current might appear like a constant background that can be
# simply subtracted. However, these contribute to the shot noise and matter for the
# non-linear effects that follow. Hence, these must be included at this stage of the
# image generation process. We subtract these backgrounds in the end.
# 3) Applying a quadratic non-linearity:
# In order to convert the units from electrons to ADU, we must use the gain factor. The gain
# has a weak dependency on the charge present in each pixel. This dependency is accounted
# for by changing the pixel values (in electrons) and applying a constant nominal gain
# later, which is unity in our demo.
# Save the image before applying the transformation to see the difference:
if diff_mode:
save_image = final_image.copy()
# Apply the WFIRST nonlinearity routine, which knows all about the nonlinearity expected in
# the WFIRST detectors.
wfirst.applyNonlinearity(final_image)
# Note that users who wish to apply some other nonlinearity function (perhaps for other NIR
# detectors, or for CCDs) can use the more general nonlinearity routine, which uses the
# following syntax:
# final_image.applyNonlinearity(NLfunc=NLfunc)
# with NLfunc being a callable function that specifies how the output image pixel values
# should relate to the input ones.
logger.debug('Applied nonlinearity to {0}-band image'.format(filter_name))
if diff_mode:
diff = final_image-save_image
out_filename = os.path.join(outpath,'demo13_NL_{0}.fits'.format(filter_name))
final_image.write(out_filename)
out_filename = os.path.join(outpath,'demo13_diff_NL_{0}.fits'.format(filter_name))
diff.write(out_filename)
# Save this image to do the diff after applying IPC.
save_image = final_image.copy()
# 4) Including Interpixel capacitance:
# The voltage read at a given pixel location is influenced by the charges present in the
# neighboring pixel locations due to capacitive coupling of sense nodes. This interpixel
# capacitance effect is modeled as a linear effect that is described as a convolution of a
# 3x3 kernel with the image. The WFIRST IPC routine knows about the kernel already, so the
# user does not have to supply it.
wfirst.applyIPC(final_image)
logger.debug('Applied interpixel capacitance to {0}-band image'.format(filter_name))
if diff_mode:
# Isolate the changes due to the interpixel capacitance effect.
diff = final_image-save_image
out_filename = os.path.join(outpath,'demo13_IPC_{0}.fits'.format(filter_name))
final_image.write(out_filename)
out_filename = os.path.join(outpath,'demo13_diff_IPC_{0}.fits'.format(filter_name))
diff.write(out_filename)
# 5) Adding read noise:
# Read noise is the noise due to the on-chip amplifier that converts the charge into an
# analog voltage. We already applied the Poisson noise due to the sky level, so read noise
# should just be added as Gaussian noise:
read_noise = galsim.GaussianNoise(rng, sigma=wfirst.read_noise)
final_image.addNoise(read_noise)
logger.debug('Added readnoise to {0}-band image'.format(filter_name))
# We divide by the gain to convert from e- to ADU. Currently, the gain value in the WFIRST
# module is just set to 1, since we don't know what the exact gain will be, although it is
# expected to be approximately 1. Eventually, this may change when the camera is assembled,
# and there may be a different value for each SCA. For now, there is just a single number,
# which is equal to 1.
final_image /= wfirst.gain
# Finally, the analog-to-digital converter reads in an integer value.
final_image.quantize()
# Note that the image type after this step is still a float. If we want to actually
# get integer values, we can do new_img = galsim.Image(final_image, dtype=int)
# Since many people are used to viewing background-subtracted images, we provide a
# version with the background subtracted (also rounding that to an int).
sky_image.quantize()
tot_sky_image = (sky_image + round(dark_current))/wfirst.gain
tot_sky_image.quantize()
final_image -= tot_sky_image
logger.debug('Subtracted background for {0}-band image'.format(filter_name))
# Write the final image to a file.
out_filename = os.path.join(outpath,'demo13_{0}.fits'.format(filter_name))
final_image.write(out_filename)
logger.info('Completed {0}-band image.'.format(filter_name))
logger.info('You can display the output in ds9 with a command line that looks something like:')
logger.info('ds9 -zoom 0.5 -scale limits -500 1000 -rgb '+
'-red output/demo13_H158.fits '+
'-green output/demo13_J129.fits '+
'-blue output/demo13_Y106.fits')
n = len(files)
for ind in range(n):
file = files[ind]
data_dir = data_dirs[ind]
print data_dir, file
# First copy the original file just in case something gets messed up.
shutil.copy(os.path.join(data_dir, file), os.path.join(data_dir, tmp_file))
print 'Copied to ', os.path.join(data_dir, tmp_file)
# Load the appropriate data from the parametric fit file.
dat = pyfits.getdata(os.path.join(data_dir, file))
# And set up the COSMOSCatalog
ccat = galsim.COSMOSCatalog(file, dir=data_dir, exclusion_level='none')
print 'Read in ', ccat.nobjects, ' objects'
# Compute the flux in the postage stamp
stamp_flux = []
for ind_n in range(ccat.nobjects):
if ind_n % 1000 == 0:
print ' Working on galaxy %d.' % ind_n
gal = ccat.makeGalaxy(index=ind_n, gal_type='real')
stamp_flux.append(gal.gal_image.array.sum())
stamp_flux = np.array(stamp_flux)
print len(stamp_flux[stamp_flux < 0]), ' negative flux stamps'
# Make a tbhdu for this data.
tbhdu = pyfits.new_table(
pyfits.ColDefs(
def test_cosmos_random():
"""Check the random object functionality of the COSMOS catalog."""
# For most of this test, we'll use the selectRandomIndex() routine, which does not try to
# construct the GSObjects. This makes the test go fast. However, we will at the end have a
# test to ensure that calling makeGalaxy() while requesting a random object has the same
# behavior as using selectRandomIndex() in limited cases.
# Initialize the catalog. The first will have weights, while the second will not (since they
# are in the RealGalaxyCatalog).
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
cat_param = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
use_real=False)
assert cat_param.real_cat is None
assert cat.real_cat is not None
# Check for exception handling if bad inputs given for the random functionality.
assert_raises(ValueError, cat.selectRandomIndex, 0)
assert_raises(ValueError, cat.selectRandomIndex, 10.7)
assert_raises(TypeError, cat.selectRandomIndex, 10, rng=3)
# Check that random objects give the right <weight> without/with weighting.
wt = cat.real_cat.weight[cat.orig_index]
wt /= np.max(wt)
avg_weight_val = np.sum(wt) / len(wt)
wavg_weight_val = np.sum(wt**2) / np.sum(wt)
with assert_raises(AssertionError):
np.testing.assert_almost_equal(avg_weight_val, wavg_weight_val, 3)
# Make sure we use enough objects that the mean weights converge properly.
randind_wt = cat.selectRandomIndex(30000, rng=galsim.BaseDeviate(1234))
wtrand = cat.real_cat.weight[cat.orig_index[randind_wt]] / \
np.max(cat.real_cat.weight[cat.orig_index])
# The average value of wtrand should be wavgw_weight_val in this case, since we used the weights
# to probabilistically select galaxies.
np.testing.assert_almost_equal(
np.mean(wtrand),
wavg_weight_val,
3,
err_msg='Average weight for random sample is wrong')
# The example catalog doesn't have weights, so it does unweighted selection, which emits a
# warning. We know about this and want to ignore it here.
with assert_warns(galsim.GalSimWarning):
randind = cat_param.selectRandomIndex(30000,
rng=galsim.BaseDeviate(1234))
wtrand = cat.real_cat.weight[cat.orig_index[randind]] / \
np.max(cat.real_cat.weight[cat.orig_index])
# The average value of wtrand should be avg_weight_val, since we did not do a weighted
# selection.
np.testing.assert_almost_equal(
np.mean(wtrand),
avg_weight_val,
3,
err_msg='Average weight for random sample is wrong')
# Check for consistency of randoms with same random seed. Do this both for the weighted and the
# unweighted calculation.
# Check for inconsistency of randoms with different random seed, or same seed but without/with
# weighting.
rng1 = galsim.BaseDeviate(1234)
rng2 = galsim.BaseDeviate(1234)
ind1 = cat.selectRandomIndex(10, rng=rng1)
ind2 = cat.selectRandomIndex(10, rng=rng2)
np.testing.assert_array_equal(
ind1,
ind2,
err_msg='Different random indices selected with same seed!')
with assert_warns(galsim.GalSimWarning):
ind1p = cat_param.selectRandomIndex(10, rng=rng1)
with assert_warns(galsim.GalSimWarning):
ind2p = cat_param.selectRandomIndex(10, rng=rng2)
np.testing.assert_array_equal(
ind1p,
ind2p,
err_msg='Different random indices selected with same seed!')
rng3 = galsim.BaseDeviate(5678)
ind3 = cat.selectRandomIndex(10, rng=rng3)
with assert_warns(galsim.GalSimWarning):
ind3p = cat_param.selectRandomIndex(10) # initialize RNG based on time
assert_raises(AssertionError, np.testing.assert_array_equal, ind1, ind1p)
assert_raises(AssertionError, np.testing.assert_array_equal, ind1, ind3)
assert_raises(AssertionError, np.testing.assert_array_equal, ind1p, ind3p)
# Finally, make sure that directly calling selectRandomIndex() gives the same random ones as
# makeGalaxy(). We'll do one real object because they are slower, and multiple parametric (just
# to make sure that the multi-object selection works consistently).
use_seed = 567
obj = cat.makeGalaxy(rng=galsim.BaseDeviate(use_seed))
ind = cat.selectRandomIndex(1, rng=galsim.BaseDeviate(use_seed))
obj_2 = cat.makeGalaxy(ind)
# Note: for real galaxies we cannot require that obj==obj_2, just that obj.index==obj_2.index.
# That's because we want to make sure the same galaxy is being randomly selected, but we cannot
# require that noise padding be the same, given the inconsistency in how the BaseDeviates are
# used in the above cases.
assert obj.index == obj_2.index, 'makeGalaxy selects random objects inconsistently'
n_random = 3
with assert_warns(galsim.GalSimWarning):
# Warns because we aren't using weights
objs = cat_param.makeGalaxy(rng=galsim.BaseDeviate(use_seed),
n_random=n_random)
with assert_warns(galsim.GalSimWarning):
inds = cat_param.selectRandomIndex(n_random,
rng=galsim.BaseDeviate(use_seed))
objs_2 = cat_param.makeGalaxy(inds)
for i in range(n_random):
# With parametric objects there is no noise padding, so we can require completely identical
# GSObjects (not just equal indices).
assert objs[i] == objs_2[
i], 'makeGalaxy selects random objects inconsistently'
# Finally, check for consistency with random object selected from RealGalaxyCatalog. For this
# case, we need to make another COSMOSCatalog that does not flag the bad objects.
use_seed = 31415
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='none')
rgc = galsim.RealGalaxyCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
ind_cc = cat.selectRandomIndex(1, rng=galsim.BaseDeviate(use_seed))
foo = galsim.RealGalaxy(rgc, random=True, rng=galsim.BaseDeviate(use_seed))
ind_rgc = foo.index
assert ind_cc==ind_rgc,\
'Different weighted random index selected from COSMOSCatalog and RealGalaxyCatalog'
# Also check for the unweighted case. Just remove that info from the catalogs and redo the
# test.
cat.real_cat = None
del rgc.weight
with assert_warns(galsim.GalSimWarning):
ind_cc = cat.selectRandomIndex(1, rng=galsim.BaseDeviate(use_seed))
foo = galsim.RealGalaxy(rgc, random=True, rng=galsim.BaseDeviate(use_seed))
ind_rgc = foo.index
assert ind_cc==ind_rgc,\
'Different unweighted random index selected from COSMOSCatalog and RealGalaxyCatalog'
# Check that setting _n_rng_calls properly tracks the RNG calls for n_random=1 and >1.
test_seed = 123456
ud = galsim.UniformDeviate(test_seed)
with assert_warns(galsim.GalSimWarning):
obj, n_rng_calls = cat.selectRandomIndex(1, rng=ud, _n_rng_calls=True)
ud2 = galsim.UniformDeviate(test_seed)
ud2.discard(n_rng_calls)
assert ud() == ud2(
), '_n_rng_calls kwarg did not give proper tracking of RNG calls'
ud = galsim.UniformDeviate(test_seed)
with assert_warns(galsim.GalSimWarning):
obj, n_rng_calls = cat.selectRandomIndex(17, rng=ud, _n_rng_calls=True)
ud2 = galsim.UniformDeviate(test_seed)
ud2.discard(n_rng_calls)
assert ud() == ud2(
), '_n_rng_calls kwarg did not give proper tracking of RNG calls'
# Invalid to both privide index and ask for random selection
with assert_raises(galsim.GalSimIncompatibleValuesError):
cat_param.makeGalaxy(index=(11, 13, 17), n_random=3)
def main(argv):
"""
Make images using constant PSF and variable shear:
- The main image is 2048 x 2048 pixels.
- Pixel scale is 0.2 arcsec/pixel, hence the image is about 0.11 degrees on a side.
- Applied shear is from a cosmological power spectrum read in from file.
- The PSF is a real one from SDSS, and corresponds to a convolution of atmospheric PSF,
optical PSF, and pixel response, which has been sampled at pixel centers. We used a PSF
from SDSS in order to have a PSF profile that could correspond to what you see with a real
telescope. However, in order that the galaxy resolution not be too poor, we tell GalSim that
the pixel scale for that PSF image is 0.2" rather than 0.396". We are simultaneously lying
about the intrinsic size of the PSF and about the pixel scale when we do this.
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles. We choose 30% of the galaxies
to use the images, and the other 60% to use the parametric fits
- The real galaxy images include some initial correlated noise from the original HST
observation. However, we whiten the noise of the final image so the final image has
stationary Gaussian noise, rather than correlated noise.
"""
logging.basicConfig(format="%(message)s",
level=logging.INFO,
stream=sys.stdout)
logger = logging.getLogger("demo11")
# Define some parameters we'll use below.
# Normally these would be read in from some parameter file.
pixel_scale = 0.2 # arcsec/pixel
image_size = 2048 # size of image in pixels
image_size_arcsec = image_size * pixel_scale # size of big image in each dimension (arcsec)
noise_variance = 5.e4 # ADU^2 (Just use simple Gaussian noise here.)
nobj = 288 # number of galaxies in entire field
# (This corresponds to 8 galaxies / arcmin^2)
grid_spacing = 90.0 # The spacing between the samples for the power spectrum
# realization (arcsec)
tel_diam = 4 # Let's figure out the flux for a 4 m class telescope
exp_time = 300 # exposing for 300 seconds.
center_ra = 19.3 * galsim.hours # The RA, Dec of the center of the image on the sky
center_dec = -33.1 * galsim.degrees
# The catalog returns objects that are appropriate for HST in 1 second exposures. So for our
# telescope we scale up by the relative area and exposure time. Note that what is important is
# the *effective* area after taking into account obscuration. For HST, the telescope diameter
# is 2.4 but there is obscuration (a linear factor of 0.33). Here, we assume that the telescope
# we're simulating effectively has no obscuration factor. We're also ignoring the pi/4 factor
# since it appears in the numerator and denominator, so we use area = diam^2.
hst_eff_area = 2.4**2 * (1. - 0.33**2)
flux_scaling = (tel_diam**2 / hst_eff_area) * exp_time
# random_seed is used for both the power spectrum realization and the random properties
# of the galaxies.
random_seed = 24783923
file_name = os.path.join('output', 'tabulated_power_spectrum.fits.fz')
logger.info('Starting demo script 11')
# Read in galaxy catalog
# The COSMOSCatalog uses the same input file as we have been using for RealGalaxyCatalogs
# along with a second file called real_galaxy_catalog_23.5_examples_fits.fits, which stores
# the information about the parameteric fits. There is no need to specify the second file
# name, since the name is derivable from the name of the main catalog.
if True:
# The catalog we distribute with the GalSim code only has 100 galaxies.
# The galaxies will typically be reused several times here.
cat_file_name = 'real_galaxy_catalog_23.5_example.fits'
dir = 'data'
cosmos_cat = galsim.COSMOSCatalog(cat_file_name, dir=dir)
else:
# If you've run galsim_download_cosmos, you can leave out the cat_file_name and dir
# to use the full COSMOS catalog with 56,000 galaxies in it.
cosmos_cat = galsim.COSMOSCatalog()
logger.info('Read in %d galaxies from catalog', cosmos_cat.nobjects)
# Setup the PowerSpectrum object we'll be using:
# To do this, we first have to read in the tabulated shear power spectrum, often denoted
# C_ell(ell), where ell has units of inverse angle and C_ell has units of angle^2. However,
# GalSim works in the flat-sky approximation, so we use this notation interchangeably with
# P(k). GalSim does not calculate shear power spectra for users, who must be able to provide
# their own (or use the examples in the repository).
#
# Here we use a tabulated power spectrum from iCosmo (http://icosmo.org), with the following
# cosmological parameters and survey design:
# H_0 = 70 km/s/Mpc
# Omega_m = 0.25
# Omega_Lambda = 0.75
# w_0 = -1.0
# w_a = 0.0
# n_s = 0.96
# sigma_8 = 0.8
# Smith et al. prescription for the non-linear power spectrum.
# Eisenstein & Hu transfer function with wiggles.
# Default dN/dz with z_med = 1.0
# The file has, as required, just two columns which are k and P(k). However, iCosmo works in
# terms of ell and C_ell; ell is inverse radians and C_ell in radians^2. Since GalSim tends to
# work in terms of arcsec, we have to tell it that the inputs are radians^-1 so it can convert
# to store in terms of arcsec^-1.
pk_file = os.path.join('data', 'cosmo-fid.zmed1.00.out')
ps = galsim.PowerSpectrum(pk_file, units=galsim.radians)
# The argument here is "e_power_function" which defines the E-mode power to use.
logger.info('Set up power spectrum from tabulated P(k)')
# Now let's read in the PSF. It's a real SDSS PSF, which means pixel scale of 0.396". However,
# the typical seeing is 1.2" and we want to simulate better seeing, so we will just tell GalSim
# that the pixel scale is 0.2". We have to be careful with SDSS PSF images, as they have an
# added 'soft bias' of 1000 which has been removed before creation of this file, so that the sky
# level is properly zero. Also, the file is bzipped, to demonstrate the ability of GalSim
# handle this kind of compressed file (among others). We read the image directly into an
# InterpolatedImage GSObject, so we can manipulate it as needed (here, the only manipulation
# needed is convolution). The flux is 1 as needed for a PSF.
psf_file = os.path.join('data', 'example_sdss_psf_sky0.fits.bz2')
psf = galsim.InterpolatedImage(psf_file, scale=pixel_scale, flux=1.)
logger.info('Read in PSF image from bzipped FITS file')
# Setup the image:
full_image = galsim.ImageF(image_size, image_size)
# The default convention for indexing an image is to follow the FITS standard where the
# lower-left pixel is called (1,1). However, this can be counter-intuitive to people more
# used to C or python indexing, where indices start at 0. It is possible to change the
# coordinates of the lower-left pixel with the methods `setOrigin`. For this demo, we
# switch to 0-based indexing, so the lower-left pixel will be called (0,0).
full_image.setOrigin(0, 0)
# As for demo10, we use random_seed for the random numbers required for the
# whole image. In this case, both the power spectrum realization and the noise on the
# full image we apply later.
rng = galsim.BaseDeviate(random_seed)
# We want to make random positions within our image. However, currently for shears from a power
# spectrum we first have to get shears on a grid of positions, and then we can choose random
# positions within that. So, let's make the grid. We're going to make it as large as the
# image, with grid points spaced by 90 arcsec (hence interpolation only happens below 90"
# scales, below the interesting scales on which we want the shear power spectrum to be
# represented exactly). The lensing engine wants positions in arcsec, so calculate that:
ps.buildGrid(grid_spacing=grid_spacing,
ngrid=int(math.ceil(image_size_arcsec / grid_spacing)),
rng=rng)
logger.info('Made gridded shears')
# We keep track of how much noise is already in the image from the RealGalaxies.
# The default initial value is all pixels = 0.
noise_image = galsim.ImageF(image_size, image_size)
noise_image.setOrigin(0, 0)
# Make a slightly non-trivial WCS. We'll use a slightly rotated coordinate system
# and center it at the image center.
theta = 0.17 * galsim.degrees
# ( dudx dudy ) = ( cos(theta) -sin(theta) ) * pixel_scale
# ( dvdx dvdy ) ( sin(theta) cos(theta) )
# Aside: You can call numpy trig functions on Angle objects directly, rather than getting
# their values in radians first. Or, if you prefer, you can write things like
# theta.sin() or theta.cos(), which are equivalent.
dudx = numpy.cos(theta) * pixel_scale
dudy = -numpy.sin(theta) * pixel_scale
dvdx = numpy.sin(theta) * pixel_scale
dvdy = numpy.cos(theta) * pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx,
dudy,
dvdx,
dvdy,
origin=full_image.true_center)
# We can also put it on the celestial sphere to give it a bit more realism.
# The TAN projection takes a (u,v) coordinate system on a tangent plane and projects
# that plane onto the sky using a given point as the tangent point. The tangent
# point should be given as a CelestialCoord.
sky_center = galsim.CelestialCoord(ra=center_ra, dec=center_dec)
# The third parameter, units, defaults to arcsec, but we make it explicit here.
# It sets the angular units of the (u,v) intermediate coordinate system.
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
# Now we need to loop over our objects:
for k in range(nobj):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed + k + 1)
# Choose a random RA, Dec around the sky_center.
# Note that for this to come out close to a square shape, we need to account for the
# cos(dec) part of the metric: ds^2 = dr^2 + r^2 d(dec)^2 + r^2 cos^2(dec) d(ra)^2
# So need to calculate dec first.
dec = center_dec + (ud() - 0.5) * image_size_arcsec * galsim.arcsec
ra = center_ra + (
ud() - 0.5) * image_size_arcsec / numpy.cos(dec) * galsim.arcsec
world_pos = galsim.CelestialCoord(ra, dec)
# We will need the image position as well, so use the wcs to get that
image_pos = wcs.toImage(world_pos)
# We also need this in the tangent plane, which we call "world coordinates" here,
# since the PowerSpectrum class is really defined on that plane, not in (ra,dec).
uv_pos = affine.toWorld(image_pos)
# Get the reduced shears and magnification at this point
g1, g2, mu = ps.getLensing(pos=uv_pos)
# Now we will have the COSMOSCatalog make a galaxy profile for us. It can make either
# a RealGalaxy using the original HST image and PSF, or a parametric model based on
# parametric fits to the light distribution of the HST observation. The parametric
# models are either a Sersic fit to the data or a bulge + disk fit according to which
# one gave the better chisq value. We will select a galaxy at random from the catalog.
# One could easily do this by choosing an index = int(ud() * cosmos_cat.nobjects), but
# we will instead allow the catalog to choose a random galaxy for us. It will remove any
# selection effects involved in postage stamp creation using weights that are stored in
# the catalog. (If for some reason you prefer not to do that, you can always choose a
# purely random index yourself using int(ud() * cosmos_cat.nobjects).) We employ this
# random selection by simply failing to specify an index or identifier for a galaxy, in
# which case it chooses a random one.
# First determine whether we will make a real galaxy (`gal_type = 'real'`) or a parametric
# galaxy (`gal_type = 'parametric'`). The real galaxies take longer to render, so for this
# script, we just use them 30% of the time and use parametric galaxies the other 70%.
# We could just use `ud()<0.3` for this, but instead we introduce another Deviate type
# available in GalSim that we haven't used yet: BinomialDeviate.
# It takes an N and p value and returns integers according to a binomial distribution.
# i.e. How many heads you get after N flips if each flip has a chance, p, of being heads.
binom = galsim.BinomialDeviate(ud, N=1, p=0.3)
real = binom()
if real:
# For real galaxies, we will want to whiten the noise in the image (below).
# When whitening the image, we need to make sure the original correlated noise is
# present throughout the whole image, otherwise the whitening will do the wrong thing
# to the parts of the image that don't include the original image. The RealGalaxy
# stores the correct noise profile to use as the gal.noise attribute. This noise
# profile is automatically updated as we shear, dilate, convolve, etc. But we need to
# tell it how large to pad with this noise by hand. This is a bit complicated for the
# code to figure out on its own, so we have to supply the size for noise padding
# with the noise_pad_size parameter.
# The large galaxies will render fine without any noise padding, but the postage stamp
# for the smaller galaxies will be sized appropriately for the PSF, which may make the
# stamp larger than the original galaxy image. The psf image is 40 x 40, although
# the bright part is much more concentrated than that. If we pad out the galaxy image
# to at least 40 x sqrt(2), we should be safe even if the galaxy image is rotated
# with respect to the psf image.
# noise_pad_size = 40 * sqrt(2) * 0.2 arcsec/pixel = 11.3 arcsec
gal = cosmos_cat.makeGalaxy(gal_type='real',
rng=ud,
noise_pad_size=11.3)
else:
gal = cosmos_cat.makeGalaxy(gal_type='parametric', rng=ud)
# Apply a random rotation
theta = ud() * 2.0 * numpy.pi * galsim.radians
gal = gal.rotate(theta)
# Rescale the flux to match our telescope configuration.
# This automatically scales up the noise variance by flux_scaling**2.
gal *= flux_scaling
# Apply the cosmological (reduced) shear and magnification at this position using a single
# GSObject method.
gal = gal.lens(g1, g2, mu)
# Convolve with the PSF.
final = galsim.Convolve(psf, gal)
# Account for the fractional part of the position
# cf. demo9.py for an explanation of this nominal position stuff.
x_nominal = image_pos.x + 0.5
y_nominal = image_pos.y + 0.5
ix_nominal = int(math.floor(x_nominal + 0.5))
iy_nominal = int(math.floor(y_nominal + 0.5))
dx = x_nominal - ix_nominal
dy = y_nominal - iy_nominal
offset = galsim.PositionD(dx, dy)
# We use method='no_pixel' here because the SDSS PSF image that we are using includes the
# pixel response already.
stamp = final.drawImage(wcs=wcs.local(image_pos),
offset=offset,
method='no_pixel')
# Recenter the stamp at the desired position:
stamp.setCenter(ix_nominal, iy_nominal)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# Now, if we are using a real galaxy, we want to ether whiten or at least symmetrize the
# noise on the postage stamp to avoid having to deal with correlated noise in any kind of
# image processing you would want to do on the final image. (Like measure galaxy shapes.)
# Galsim automatically propagates the noise correctly from the initial RealGalaxy object
# through the applied shear, distortion, rotation, and convolution into the final object's
# noise attribute. To make the noise fully white, use the image.whitenNoise() method.
# The returned value is the variance of the Gaussian noise that is present after the
# whitening process.
# However, this is often overkill for many applications. If it is acceptable to merely end
# up with noise with some degree of symmetry (say 4-fold or 8-fold symmetry), then you can
# instead have GalSim just add enough noise to make the resulting noise have this kind of
# symmetry. Usually this requires adding significantly less additional noise, which means
# you can have the resulting total variance be somewhat smaller. The returned variance
# corresponds to the zero-lag value of the noise correlation function, which will still have
# off-diagonal elements. We can do this step using the image.symmetrizeNoise() method.
if real:
if True:
# We use the symmetrizing option here.
new_variance = stamp.symmetrizeNoise(final.noise, 8)
else:
# Here is how you would do it if you wanted to fully whiten the image.
new_variance = stamp.whitenNoise(final.noise)
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += new_variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info('Galaxy %d: position relative to center = %s, t=%f s', k,
str(uv_pos), tot_time)
# We already have some noise in the image, but it isn't uniform. So the first thing to do is
# to make the Gaussian noise uniform across the whole image. We have a special noise class
# that can do this. VariableGaussianNoise takes an image of variance values and applies
# Gaussian noise with the corresponding variance to each pixel.
# So all we need to do is build an image with how much noise to add to each pixel to get us
# up to the maximum value that we already have in the image.
max_current_variance = numpy.max(noise_image.array)
noise_image = max_current_variance - noise_image
vn = galsim.VariableGaussianNoise(rng, noise_image)
full_image.addNoise(vn)
# Now max_current_variance is the noise level across the full image. We don't want to add that
# twice, so subtract off this much from the intended noise that we want to end up in the image.
noise_variance -= max_current_variance
# Now add Gaussian noise with this variance to the final image. We have to do this step
# at the end, rather than adding to individual postage stamps, in order to get the noise
# level right in the overlap regions between postage stamps.
noise = galsim.GaussianNoise(rng, sigma=math.sqrt(noise_variance))
full_image.addNoise(noise)
logger.info('Added noise to final large image')
# Now write the image to disk. It is automatically compressed with Rice compression,
# since the filename we provide ends in .fz.
full_image.write(file_name)
logger.info('Wrote image to %r', file_name)
# Compute some sky positions of some of the pixels to compare with the values of RA, Dec
# that ds9 reports. ds9 always uses (1,1) for the lower left pixel, so the pixel coordinates
# of these pixels are different by 1, but you can check that the RA and Dec values are
# the same as what GalSim calculates.
ra_str = center_ra.hms()
dec_str = center_dec.dms()
logger.info('Center of image is at RA %sh %sm %ss, DEC %sd %sm %ss',
ra_str[0:3], ra_str[3:5], ra_str[5:], dec_str[0:3],
dec_str[3:5], dec_str[5:])
for (x, y) in [(0, 0), (0, image_size - 1), (image_size - 1, 0),
(image_size - 1, image_size - 1)]:
world_pos = wcs.toWorld(galsim.PositionD(x, y))
ra_str = world_pos.ra.hms()
dec_str = world_pos.dec.dms()
logger.info('Pixel (%4d, %4d) is at RA %sh %sm %ss, DEC %sd %sm %ss',
x, y, ra_str[0:3], ra_str[3:5], ra_str[5:], dec_str[0:3],
dec_str[3:5], dec_str[5:])
logger.info(
'ds9 reports these pixels as (1,1), (1,2048), etc. with the same RA, Dec.'
)
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles.
- The real galaxy images include some initial correlated noise from the original HST
observation. However, we whiten the noise of the final image so the final image has
stationary Gaussian noise, rather than correlated noise.
"""
logging.basicConfig(format="%(message)s", level=logging.INFO, stream=sys.stdout)
global logger
logger = logging.getLogger("mock_superbit_data")
# Define some parameters we'll use below.
# Normally these would be read in from some parameter file.
global pixel_scale
pixel_scale = 0.206 # arcsec/pixel
global image_xsize
image_xsize = 6665 # size of image in pixels
global image_ysize
image_ysize = 4453 # size of image in pixels
global image_xsize_arcsec
image_xsize_arcsec = image_xsize*pixel_scale # size of big image in each dimension (arcsec)
global image_ysize_arcsec
image_ysize_arcsec = image_ysize*pixel_scale # size of big image in each dimension (arcsec)
global center_ra
center_ra = 19.3*galsim.hours # The RA, Dec of the center of the image on the sky
global center_dec
center_dec = -33.1*galsim.degrees
global nobj
nobj = 30 # number of galaxies in entire field; this number matches empirical
global nstars
nstars = 1000 # number of stars in the entire field
global flux_scaling # Let's figure out the flux for a 0.5 m class telescope
global tel_diam
tel_diam = 0.5
global psf_fwhm
psf_fwhm = 0.30
global lam
lam = 625 # Central wavelength for an airy disk
global exp_time
exp_time = 300
global noise_variance
global sky_level
psf_path = '/Users/jemcclea/Research/SuperBIT/superbit-ngmix/scripts/outputs/psfex_output'
global nfw # will store the NFWHalo information
global cosmos_cat # will store the COSMOS catalog from which we draw objects
# Set up the NFWHalo:
mass=5E14 # Cluster mass (Msol/h)
nfw_conc = 4 # Concentration parameter = virial radius / NFW scale radius
nfw_z_halo = 0.17 # redshift of the halo --> correct!
nfw_z_source = 0.6 # redshift of the lensed sources; COSMOS galaxies don't have any
omega_m = 0.3 # Omega matter for the background cosmology.
omega_lam = 0.7 # Omega lambda for the background cosmology.
nfw = galsim.NFWHalo(mass=mass, conc=nfw_conc, redshift=nfw_z_halo,
omega_m=omega_m, omega_lam=omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog
"""
cat_file_name = 'real_galaxy_catalog_23.5.fits'
dir = 'data/COSMOS_23.5_training_sample'
#cat_file_name = 'real_galaxy_catalog_23.5_example.fits'
#dir = 'data'
"""
cat_file_name = 'real_galaxy_catalog_25.2.fits'
dir = 'data/COSMOS_25.2_training_sample/'
cosmos_cat = galsim.COSMOSCatalog(cat_file_name, dir=dir)
logger.info('Read in %d galaxies from catalog', cosmos_cat.nobjects)
# The catalog returns objects that are appropriate for HST in 1 second exposures. So for our
# telescope we scale up by the relative area and exposure time.
# Will also multiply by the gain and relative pixel scales...
hst_eff_area = 2.4**2 * (1.-0.33**2)
sbit_eff_area = tel_diam**2 * (1.-0.3840**2)
#sbit_eff_area = tel_diam**2 * (1.-0.1**2)
###
### LOOP OVER PSFs TO MAKE GROUPS OF IMAGES
### WITHIN EACH PSF, ITERATE 5 TIMES TO MAKE 5 SEPARATE IMAGES
###
#all_psfs=glob.glob(psf_path+"/*.psf")
#all_psfs=glob.glob(psf_path+"/*300*.psf")
random_seed = 35609377914
i=0
for psf_filen in range(1):
logger.info('Beginning PSF %s...'% psf_filen)
rng = galsim.BaseDeviate(random_seed)
timescale=str(exp_time)
outname=''.join(['debug_0.3FWHM_gaussStar_',timescale,'_',str(i),'.fits'])
truth_file_name=''.join(['./output-debug/truth_0.3FWHM_gaussStar_',timescale,'_',str(i),'.dat'])
file_name = os.path.join('output-debug',outname)
# Set up the image:
if timescale=='150':
print("Automatically detecting a 150s exposure image, setting flux scale and noise accordingly")
#noise_variance=570 # ADU^2 (Just use simple Gaussian noise here.)
noise_variance=570 # ADU^2 (Just use simple Gaussian noise here.)
sky_level = 51 # ADU
exp_time=150.
else:
print("Automatically detecting a 300s exposure image, setting flux scale and noise accordingly")
#noise_variance=400 # ADU^2 (Just use simple Gaussian noise here.)
noise_variance=400 # ADU^2 (Just use simple Gaussian noise here.)
sky_level = 106 # ADU
exp_time=300.
flux_scaling = (sbit_eff_area/hst_eff_area) * exp_time * 3.33 * (.206/.05)**2
# Setting up a truth catalog
names = [ 'gal_num', 'x_image', 'y_image',
'ra', 'dec', 'g1_nopsf', 'g2_nopsf','g1_meas', 'g2_meas', 'fwhm','final_sigmaSize',
'nopsf_sigmaSize','nfw_g1', 'nfw_g2', 'nfw_mu', 'redshift','flux', 'stamp_sum', 'noisevar']
types = [ int, float, float, float, float, float,
float, float, float, float, float, float,
float, float,float, float, float,float, float]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(image_xsize, image_ysize)
full_image.fill(sky_level)
full_image.setOrigin(0,0)
# We keep track of how much noise is already in the image from the RealGalaxies.
noise_image = galsim.ImageF(image_xsize, image_ysize)
noise_image.setOrigin(0,0)
# Make a slightly non-trivial WCS. We'll use a slightly rotated coordinate system
# and center it at the image center.
theta = 0.17 * galsim.degrees
dudx = numpy.cos(theta) * pixel_scale
dudy = -numpy.sin(theta) * pixel_scale
dvdx = numpy.sin(theta) * pixel_scale
dvdy = numpy.cos(theta) * pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=center_ra, dec=center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
# Loop over galaxy objects:
for k in range(nobj):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed+k+1)
try:
# make single galaxy object
logger.debug("about to make stamp %d...",k)
stamp,truth = make_a_galaxy(ud=ud,wcs=wcs,affine=affine)
logger.debug("stamp %d is made",k)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
logger.debug("stamp %d added to full image",k)
time2 = time.time()
tot_time = time2-time1
logger.info('Galaxy %d positioned relative to center t=%f s',
k, tot_time)
try:
g1_real=stamp.FindAdaptiveMom().observed_shape.g1
g2_real=stamp.FindAdaptiveMom().observed_shape.g2
except:
g1_real=-9999.
g2_real=-9999.
logger.debug("Galaxy %d made it past g1/g2_real stage",k)
sum_flux=numpy.sum(stamp.array)
row = [ k,truth.x, truth.y, truth.ra, truth.dec, truth.g1_nopsf, truth.g2_nopsf, g1_real, g2_real, truth.fwhm, truth.final_sigmaSize, truth.nopsf_sigmaSize,truth.g1,truth.g2, truth.mu, truth.z, truth.flux, sum_flux, truth.variance]
truth_catalog.addRow(row)
logger.debug("row for galaxy %d added to truth catalog\n\n",k)
except:
logger.info('Galaxy %d has failed, skipping...',k)
#pdb.set_trace()
pass
###### Inject cluster galaxy objects:
random_seed=892465352
center_coords = galsim.CelestialCoord(center_ra,center_dec)
centerpix = wcs.toImage(center_coords)
for k in range(40):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed+k+1)
try:
# make single galaxy object
cluster_stamp,truth = make_cluster_galaxy(ud=ud,wcs=wcs,affine=affine,centerpix=centerpix)
# Find the overlapping bounds:
bounds = cluster_stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += cluster_stamp[bounds]
time2 = time.time()
tot_time = time2-time1
logger.info('Cluster galaxy %d positioned relative to center t=%f s',
k, tot_time)
except:
logger.info('Cluster galaxy %d has failed, skipping...',k)
pdb.set_trace()
####
### Now repeat process for stars!
####
random_seed_stars=2308173501873
for k in range(nstars):
time1 = time.time()
ud = galsim.UniformDeviate(random_seed_stars+k+1)
try:
star_stamp,truth=make_a_star(ud=ud,wcs=wcs,affine=affine)
bounds = star_stamp.bounds & full_image.bounds
logger.debug("star stamp & truth catalog made for star %d" %k)
# Add the stamp to the full image.
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2-time1
logger.info('Star %d: positioned relative to center, t=%f s',
k, tot_time)
try:
g1_real=star_stamp.FindAdaptiveMom().observed_shape.g1
g2_real=star_stamp.FindAdaptiveMom().observed_shape.g2
except:
g1_real = -9999.
g2_real = -9999.
this_var = -9999.
sum_flux=numpy.sum(star_stamp.array)
row = [ k,truth.x, truth.y, truth.ra, truth.dec,
truth.g1_nopsf, truth.g2_nopsf, g1_real, g2_real,
truth.fwhm, truth.final_sigmaSize, truth.nopsf_sigmaSize, truth.g1,
truth.g2, truth.mu, truth.z, truth.flux, sum_flux, truth.variance]
truth_catalog.addRow(row)
except:
logger.info('Star %d has failed, skipping...',k)
pdb.set_trace()
# We already have some noise in the image, but it isn't uniform. So the first thing to do is
# to make the Gaussian noise uniform across the whole image.
#max_current_variance = numpy.max(noise_image.array)
#noise_image = max_current_variance - noise_image
vn = galsim.VariableGaussianNoise(rng, noise_image)
full_image.addNoise(vn)
# Now max_current_variance is the noise level across the full image. We don't want to add that
# twice, so subtract off this much from the intended noise that we want to end up in the image.
#noise_variance -= max_current_variance
# Now add Gaussian noise with this variance to the final image.
noise = galsim.GaussianNoise(rng, sigma=math.sqrt(noise_variance))
full_image.addNoise(noise)
logger.info('Added noise to final output image')
# Now write the image to disk.
full_image.write(file_name)
# Add a FLUXSCL keyword for later stacking
this_hdu=astropy.io.fits.open(file_name)
this_hdu[0].header['FLXSCALE'] = 300.0/exp_time
this_hdu.writeto(file_name,overwrite='True')
logger.info('Wrote image to %r',file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
i=i+1
logger.info('completed run %d for psf %s',i,psf_filen)
logger.info('completed all images')
def draw_galaxies(
data_dir=None,
generative_model='https://raw.githubusercontent.com/EiffL/GalSim-Hub/master/modules/generative_model.tar.gz',
batch_size=1024,
n_batches=None,
pool_size=12):
"""
This function will draw in postage stamps a sample of galaxies using both
the real COSMOS galaxy catalog and a given generative model, it outputs
"""
cosmos_cat = galsim.COSMOSCatalog(dir=data_dir)
cosmos_noise = galsim.getCOSMOSNoise()
# Generating galaxies from the model by batch
gal_model = GenerativeGalaxyModel(generative_model)
sims_stamps = []
cosmos_stamps = []
param_stamps = []
idents = []
if n_batches is None:
n_batches = len(cosmos_cat.orig_index) // batch_size
# Process galaxies by batches
for i in range(n_batches):
inds = np.arange(i * batch_size, (i + 1) * batch_size)
print("Batch %d" % i)
# Generate uncovolved light profiles
sim_galaxies = gal_model.sample(
cat=cosmos_cat.param_cat[cosmos_cat.orig_index[inds]])
indices = [(j, k) for j, k in enumerate(inds)]
# Draw galaxies on postage stamps
engine = partial(_draw_galaxies,
cosmos_cat=cosmos_cat,
sim_galaxies=sim_galaxies,
cosmos_noise=cosmos_noise)
if pool_size is None:
res = []
for ind in indices:
res.append(engine(ind))
else:
with Pool(pool_size) as p:
res = p.map(engine, indices)
# Extract the postage stamps into separate lists, discarding the ones
# that failed
for k, im_sims, im_cosmos, im_param, flag in res:
if flag:
sims_stamps.append(im_sims)
cosmos_stamps.append(im_cosmos)
param_stamps.append(im_param)
idents.append(
cosmos_cat.param_cat[cosmos_cat.orig_index[k]]['IDENT'])
# Puts all images into one big table
table = Table([
np.array(idents),
np.stack(cosmos_stamps),
np.stack(sims_stamps),
np.stack(param_stamps)
],
names=['IDENT', 'real', 'mock', 'param'])
# Merge with the cosmos catalog data for convenience
table = join(cosmos_cat.param_cat, table, keys=['IDENT'])
return table
def build_input_pipeline(data_dir,
filename='real_galaxy_catalog_25.2.fits',
conditions=[],
batch_size=128,
stamp_size=64,
pixel_size=0.03,
clip=1.,
input_nprocs=None,
nrepeat=10,
cache_dir=None,
buffer_size=20000,
**kwargs):
"""
This function creates an input pipeline by drawing images from GalSim
Parameters
----------
dir: Directory for the GalSim data
filename: Name of the GalSim real catalog
conditions: List of catalog quantities to use as conditions
nrepeat: Number of times the dataset is randomly rotated
"""
cat = galsim.COSMOSCatalog(dir=data_dir, file_name=filename)
# Extracts the parametric catalog
cat_param = cat.param_cat[cat.orig_index]
from numpy.lib.recfunctions import append_fields
# Adds a few extra fields
bparams = cat_param['bulgefit']
sparams = cat_param['sersicfit']
cat_param = append_fields(cat_param, 'bulge_q', bparams[:, 11])
cat_param = append_fields(cat_param, 'bulge_beta', bparams[:, 15])
cat_param = append_fields(cat_param, 'disk_q', bparams[:, 3])
cat_param = append_fields(cat_param, 'disk_beta', bparams[:, 7])
cat_param = append_fields(cat_param, 'bulge_hlr', cat_param['hlr'][:, 1])
cat_param = append_fields(
cat_param, 'bulge_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 1]), np.zeros(len(cat_param))))
cat_param = append_fields(cat_param, 'disk_hlr', cat_param['hlr'][:, 2])
cat_param = append_fields(
cat_param, 'disk_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 2]), np.zeros(len(cat_param))))
if input_nprocs is not None:
pool = Pool(input_nprocs)
else:
pool = None
def training_fn():
dset = tf.data.Dataset.from_tensor_slices(cat.orig_index)
dset = dset.batch(128).map(
get_postage_stamp_map(cat.real_cat,
stamp_size=stamp_size,
pixel_size=pixel_size,
pool=pool))
dset = dset.flat_map(lambda arg, *rest: tf.data.Dataset.
from_tensor_slices((arg, ) + rest))
dset = dset.repeat(nrepeat)
if cache_dir is not None:
dset = dset.cache(cache_dir)
if len(conditions) > 0:
# Extract from the catalog the desired quantities
dset_cond = tf.data.Dataset.zip(
tuple([
tf.data.Dataset.from_tensor_slices(
cat_param[k].astype('float32')) for k in conditions
]))
dset_cond.repeat(nrepeat)
dset = dset = tf.data.Dataset.zip((dset, dset_cond))
dset = dset.repeat().shuffle(
buffer_size=buffer_size).batch(batch_size).prefetch(16)
iterator = dset.make_one_shot_iterator()
if len(conditions) == 0:
batch_im, batch_psf, batch_ps = iterator.get_next()
return {
'x': tf.clip_by_value(batch_im, -clip, clip),
'psf': batch_psf,
'ps': batch_ps
}, tf.clip_by_value(batch_im, -clip, clip)
else:
(batch_im, batch_psf, batch_ps), batch_cond = iterator.get_next()
return {
'x': tf.clip_by_value(batch_im, -clip, clip),
'psf': batch_psf,
'ps': batch_ps,
'y': {k: batch_cond[i]
for i, k in enumerate(conditions)}
}, tf.clip_by_value(batch_im, -clip, clip)
return training_fn
import galsim
from astropy import wcs as WCS
import numpy as np
import astropy
import os
data_dir='/Users/remy/Desktop/LSST_Project/GalSim/examples/data'
cat0 = galsim.COSMOSCatalog(dir=data_dir, file_name = 'real_galaxy_catalog_23.5_example.fits')
mag_cat_name = os.path.join(os.path.dirname(os.getcwd()), 'data', 'sample_input_catalog.fits')
mag_cat = astropy.table.Table.read(mag_cat_name, format = 'fits')
mags = np.array([mag_cat["u_ab"],
mag_cat["g_ab"],
mag_cat["r_ab"],
mag_cat["i_ab"],
mag_cat["z_ab"],
mag_cat["y_ab"]]).T
l_cat = len(mag_cat)
class Pictures:
""" Class that draws simulated images of the sky at random given a dictionary
"""
def __init__(self, survey, shape, cat = cat0, npsf = 41, pt_fraction = 0.1):
"""
survey: dict
Properties of the imaging survey for which we want to generate images
shape: tuple
2D shape of the output stamp image
cat: Catalog
Galsim catalog of galaxies
def test_cosmos_basic():
"""Check some basic functionality of the COSMOSCatalog class."""
# Note, there's not much here yet. Could try to think of other tests that are more
# interesting.
# Initialize a COSMOSCatalog with all defaults.
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
# Initialize one that doesn't exclude failures. It should be >= the previous one in length.
cat2 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='none')
assert cat2.nobjects >= cat.nobjects
# Check for reasonable exceptions when initializing.
try:
# Can't find data (wrong directory).
np.testing.assert_raises(
IOError,
galsim.COSMOSCatalog,
file_name='real_galaxy_catalog_23.5_example.fits')
except ImportError:
print('The assert_raises tests require nose')
# Try making galaxies
gal_real = cat2.makeGalaxy(index=0, gal_type='real', chromatic=False)
if not isinstance(gal_real, galsim.RealGalaxy):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return an instance of "
"'galsim.RealGalaxy'")
gal_param = cat.makeGalaxy(index=10, gal_type='parametric', chromatic=True)
if not isinstance(gal_param, galsim.ChromaticObject):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return an instance of "
"'galsim.ChromaticObject' for parametric galaxies")
gal_real_list = cat.makeGalaxy(index=[3, 6],
gal_type='real',
chromatic=False)
for gal_real in gal_real_list:
if not isinstance(gal_real, galsim.RealGalaxy):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return a list of instances "
"of 'galsim.RealGalaxy'")
gal_param_list = cat.makeGalaxy(index=[4, 7],
gal_type='parametric',
chromatic=False)
for gal_param in gal_param_list:
if not isinstance(gal_param, galsim.GSObject):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return a list of instances "
"of 'galsim.GSObect'")
# Check for parametric catalog
cat_param = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example_fits.fits',
dir=datapath,
use_real=False)
# Try making galaxies
gal = cat_param.makeGalaxy(index=1)
if not isinstance(gal, galsim.GSObject):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return an instance of "
"'galsim.GSObject when loaded from a fits file.")
gal_list = cat_param.makeGalaxy(index=[2, 3])
for gal in gal_list:
if not isinstance(gal, galsim.GSObject):
raise TypeError(
"COSMOS Catalog makeGalaxy routine does not return a list of instances "
"of 'galsim.GSObject when loaded from a fits file.")
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles. We choose 40% of the galaxies
to use the images, and the other 60% to use the parametric fits
- The real galaxy images include some initial correlated noise from the original HST
observation. However, we whiten the noise of the final image so the final image has
stationary Gaussian noise, rather than correlated noise.
"""
logging.basicConfig(format="%(message)s",
level=logging.INFO,
stream=sys.stdout)
logger = logging.getLogger("mock_superbit_data")
# Define some parameters we'll use below.
# Normally these would be read in from some parameter file.
global pixel_scale
pixel_scale = 0.206 # arcsec/pixel
global image_xsize
image_xsize = 6665 # size of image in pixels
global image_ysize
image_ysize = 4453 # size of image in pixels
global image_xsize_arcsec
image_xsize_arcsec = image_xsize * pixel_scale # size of big image in each dimension (arcsec)
global image_ysize_arcsec
image_ysize_arcsec = image_ysize * pixel_scale # size of big image in each dimension (arcsec)
global center_ra
#center_ra = 19.3*galsim.hours # The RA, Dec of the center of the image on the sky
global center_dec
#center_dec = -33.1*galsim.degrees
global nobj
nobj = 1700 # number of galaxies in entire field -- an adjustment to ensure ~1100 detections
global nstars
nstars = 370 # number of stars in the entire field
#global flux_scaling # Let's figure out the flux for a 0.5 m class telescope
global tel_diam
tel_diam = 0.5
global lam
lam = 587 # Central wavelength
global exp_time
global noise_variance
global sky_level
psf_path = '/Users/jemcclea/Research/SuperBIT_2019/superbit-ngmix/scripts/outputs/psfex_output'
global nfw # will store the NFWHalo information
global cosmos_cat # will store the COSMOS catalog from which we draw objects
# Set up the NFWHalo:
mass = 5E14 # Cluster mass (Msol/h)
nfw_conc = 4 # Concentration parameter = virial radius / NFW scale radius
nfw_z_halo = 0.3 # redshift of the halo
nfw_z_source = 0.6 # redshift of the lensed sources
omega_m = 0.3 # Omega matter for the background cosmology.
omega_lam = 0.7 # Omega lambda for the background cosmology.
nfw = galsim.NFWHalo(mass=mass,
conc=nfw_conc,
redshift=nfw_z_halo,
omega_m=omega_m,
omega_lam=omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog
if True:
# The catalog we distribute with the GalSim code only has 100 galaxies.
# The galaxies will typically be reused several times here.
cat_file_name = 'real_galaxy_catalog_23.5_example.fits'
dir = 'data'
cosmos_cat = galsim.COSMOSCatalog(cat_file_name, dir=dir)
else:
# If you've run galsim_download_cosmos, you can leave out the cat_file_name and dir
# to use the full COSMOS catalog with 56,000 galaxies in it.
cosmos_cat = galsim.COSMOSCatalog()
logger.info('Read in %d galaxies from catalog', cosmos_cat.nobjects)
# The catalog returns objects that are appropriate for HST in 1 second exposures. So for our
# telescope we scale up by the relative area and exposure time.
hst_eff_area = 2.4**2 * (1. - 0.33**2)
sbit_eff_area = tel_diam**2 * (1. - 0.3840**2)
#flux_scaling = (sbit_eff_area/hst_eff_area) * exp_time
###
### LOOP OVER PSFs TO MAKE GROUPS OF IMAGES
### WITHIN EACH PSF, ITERATE 5 TIMES TO MAKE 5 SEPARATE IMAGES
###
all_psfs = glob.glob(psf_path + "/*150*.psf")
logger.info('Beginning loop over jitter/optical psfs')
# random_seed = 24783923
random_seed = 247
i = 0
for psf_filen in all_psfs:
logger.info('Beginning PSF %s...' % psf_filen)
rng = galsim.BaseDeviate(random_seed)
# This is specific to empirical PSFs
try:
timescale = psf_filen.split('target_')[1].split('_WCS')[0]
except:
timescale = psf_filen.split('sci_')[1].split('_WCS')[0]
outname = ''.join(
['mockSuperbit_empiricalPSF_', timescale, '_',
str(i), '.fits'])
truth_file_name = ''.join(
['./output/truth_empiricalPSF_', timescale, '_',
str(i), '.dat'])
file_name = os.path.join('output', outname)
# Set up the image:
if timescale == '150':
print(
"Automatically detecting a 150s exposure image, setting flux scale and noise accordingly"
)
noise_variance = 1.8e3 # ADU^2 (Just use simple Gaussian noise here.) -->150s
sky_level = 51 # ADU / arcsec^2 -->150s
exp_time = 150.
else:
print(
"Automatically detecting a 300s exposure image, setting flux scale and noise accordingly"
)
noise_variance = 2.55e3 # ADU^2 (Just use simple Gaussian noise here.) -->300s
sky_level = 106 # ADU / arcsec^2 -->300s
exp_time = 300.
# Setting up a truth catalog
names = [
'gal_num', 'x_image', 'y_image', 'ra', 'dec', 'g1_meas', 'g2_meas',
'nfw_g1', 'nfw_g2', 'nfw_mu', 'redshift', 'flux', 'var'
]
types = [
int, float, float, float, float, float, float, float, float, float,
float, float, float
]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(image_xsize, image_ysize)
full_image.fill(sky_level)
full_image.setOrigin(0, 0)
# We keep track of how much noise is already in the image from the RealGalaxies.
noise_image = galsim.ImageF(image_xsize, image_ysize)
noise_image.setOrigin(0, 0)
# Make a slightly non-trivial WCS. We'll use a slightly rotated coordinate system
# and center it at the image center.
"""
theta = 0.17 * galsim.degrees
dudx = numpy.cos(theta) * pixel_scale
dudy = -numpy.sin(theta) * pixel_scale
dvdx = numpy.sin(theta) * pixel_scale
dvdy = numpy.cos(theta) * pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=center_ra, dec=center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
"""
this_im_wcs = get_wcs_info(psf_filen)
affine = this_im_wcs.affine(full_image.true_center)
full_image.wcs = this_im_wcs
# Now let's read in the PSFEx PSF model. We read the image directly into an
# InterpolatedImage GSObject, so we can manipulate it as needed
psf_wcs = this_im_wcs
psf_file = os.path.join(psf_path, psf_filen)
psf = galsim.des.DES_PSFEx(psf_file, wcs=psf_wcs)
logger.info('Constructed PSF object from PSFEx file')
# Loop over galaxy objects:
for k in range(nobj):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed + k + 1)
try:
# make single galaxy object
stamp, truth = make_a_galaxy(ud=ud,
this_im_wcs=this_im_wcs,
psf=psf,
affine=affine)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info('Galaxy %d positioned relative to center t=%f s',
k, tot_time)
g1_real = stamp.FindAdaptiveMom().observed_shape.g1
g2_real = stamp.FindAdaptiveMom().observed_shape.g2
#g1_real=-9999.
#g2_real=-9999.
row = [
k, truth.x, truth.y, truth.ra, truth.dec, g1_real, g2_real,
truth.g1, truth.g2, truth.mu, truth.z, truth.flux,
truth.variance
]
truth_catalog.addRow(row)
except:
logger.info('Galaxy %d has failed, skipping...', k)
#pdb.set_trace()
pass
####
### Now repeat process for stars!
####
random_seed_stars = 3221987
for k in range(nstars):
time1 = time.time()
ud = galsim.UniformDeviate(random_seed_stars + k + 1)
star_stamp, truth = make_a_star(ud=ud,
this_im_wcs=this_im_wcs,
psf=psf,
affine=affine)
bounds = star_stamp.bounds & full_image.bounds
# Add the stamp to the full image.
try:
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info('Star %d: positioned relative to center, t=%f s',
k, tot_time)
g1_real = star_stamp.FindAdaptiveMom().observed_shape.g1
g2_real = star_stamp.FindAdaptiveMom().observed_shape.g2
#g1_real = -9999.
#g2_real = -9999.
this_var = -9999.
this_flux = numpy.sum(star_stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, g1_real, g2_real,
truth.g1, truth.g2, truth.mu, truth.z, this_flux, this_var
]
truth_catalog.addRow(row)
except:
logger.info('Star %d has failed, skipping...', k)
pass
# We already have some noise in the image, but it isn't uniform. So the first thing to do is
# to make the Gaussian noise uniform across the whole image.
max_current_variance = numpy.max(noise_image.array)
noise_image = max_current_variance - noise_image
vn = galsim.VariableGaussianNoise(rng, noise_image)
full_image.addNoise(vn)
# Now max_current_variance is the noise level across the full image. We don't want to add that
# twice, so subtract off this much from the intended noise that we want to end up in the image.
noise_variance -= max_current_variance
# Now add Gaussian noise with this variance to the final image.
try:
noise = galsim.GaussianNoise(rng, sigma=math.sqrt(noise_variance))
except:
noise = galsim.GaussianNoise(rng, sigma=math.sqrt(1800))
full_image.addNoise(noise)
logger.info('Added noise to final large image')
# Now write the image to disk. It is automatically compressed with Rice compression,
# since the filename we provide ends in .fz.
full_image.write(file_name)
logger.info('Wrote image to %r', file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
# Compute some sky positions of some of the pixels to compare with the values of RA, Dec
# that ds9 reports. ds9 always uses (1,1) for the lower left pixel, so the pixel coordinates
# of these pixels are different by 1, but you can check that the RA and Dec values are
# the same as what GalSim calculates.
i = i + 1
logger.info(' ')
logger.info('completed run %d for psf %s', i, psf_filen)
logger.info('completed all images')
def main():
parser = argparse.ArgumentParser(
description='MultiProFit HST COSMOS galaxy modelling test')
flags = {
'catalogpath': {
'type': str,
'nargs': '?',
'default': None,
'help': 'GalSim catalog path'
},
'catalogfile': {
'type': str,
'nargs': '?',
'default': None,
'help': 'GalSim catalog filename'
},
'file': {
'type': str,
'nargs': '?',
'default': None,
'help': 'Filename for input/output'
},
'do_fit_fluxfracs': {
'type': mpfutil.str2bool,
'default': False,
'help':
'Fit component flux fractions for galaxies instead of fluxes'
},
'fithsc': {
'type': mpfutil.str2bool,
'default': False,
'help': 'Fit HSC I band image'
},
'fithst': {
'type': mpfutil.str2bool,
'default': False,
'help': 'Fit HST F814W image'
},
'fithst2hsc': {
'type': mpfutil.str2bool,
'default': False,
'help': 'Fit HST F814W image convolved '
'to HSC seeing'
},
'bands_hsc': {
'type': str,
'nargs': '*',
'default': ['HSC-I'],
'help': 'HSC Bands to fit'
},
'hscrepo': {
'type': str,
'default': '/datasets/hsc/repo/rerun/RC/w_2019_26/DM-19560/',
'help': 'Path to HSC processing repository'
},
'model_name_hst2hsc': {
'type': str,
'default': None,
'help': 'HST model fit to use for mock HSC image'
},
'img_plot_maxs': {
'type':
float,
'nargs':
'*',
'default':
None,
'help':
'Max. flux for scaling single-band images. F814W first if fitting HST, '
'then HSC bands.'
},
'img_multi_plot_max': {
'type': float,
'default': None,
'help': 'Max. flux for scaling color images'
},
'indices': {
'type': str,
'nargs': '*',
'default': None,
'help': 'Galaxy catalog index'
},
'loglevel': {
'type': int,
'nargs': '?',
'default': 20,
'help': 'logging.Logger default level'
},
'modelspecfile': {
'type': str,
'default': None,
'help': 'Model specification file'
},
'modellib': {
'type': str,
'nargs': '?',
'default': 'scipy',
'help': 'Optimization libraries'
},
'modellibopts': {
'type': str,
'nargs': '?',
'default': None,
'help': 'Model fitting options'
},
'nwrite': {
'type': int,
'default': 5,
'help': 'Number of galaxies to fit before writing file'
},
#'engines': {'type': str, 'nargs': '*', 'default': 'galsim', 'help': 'Model generation engines'},
'plot': {
'type': mpfutil.str2bool,
'default': False,
'help': 'Toggle plotting of final fits'
},
'print_step': {
'type': int,
'default': 100,
'help': 'Number of fitting steps before printing'
},
#'seed': {'type': int, 'nargs': '?', 'default': 1, 'help': 'Numpy random seed'}
'redo': {
'type': mpfutil.str2bool,
'default': True,
'help': 'Redo existing fits'
},
'redo_psfs': {
'type': mpfutil.str2bool,
'default': False,
'help': 'Redo existing PSF fits'
},
'weights_band': {
'type': float,
'nargs': '*',
'default': None,
'help':
'Multiplicative weights for scaling images in multi-band RGB'
},
'write': {
'type': mpfutil.str2bool,
'default': True,
'help': 'Write file?'
},
}
for key, value in flags.items():
if key in options:
default = options[key]["default"]
else:
default = value['default']
if 'help' in value:
value['help'] += ' (default: ' + str(default) + ')'
value["default"] = default
parser.add_argument('--' + key, **value)
args = parser.parse_args()
args.catalogpath = os.path.expanduser(args.catalogpath)
logging.basicConfig(stream=sys.stdout, level=args.loglevel)
modelspecs = mpffit.get_modelspecs(None if args.modelspecfile is None else
os.path.expanduser(args.modelspecfile))
if args.file is not None:
args.file = os.path.expanduser(args.file)
if args.file is not None and os.path.isfile(args.file):
with open(args.file, 'rb') as f:
data = pickle.load(f)
else:
data = {}
if args.plot:
mpl.rcParams['image.origin'] = 'lower'
srcs = ['hst'] if args.fithst else []
if args.fithsc or args.fithst2hsc:
import lsst.daf.persistence as dafPersist
butler = dafPersist.Butler(args.hscrepo)
skymap = butler.get("deepCoadd_skyMap", dataId={"tract": 9813})
else:
butler = None
skymap = None
if args.fithsc:
srcs += ["hsc"]
if args.fithst2hsc:
srcs += ["hst2hsc"]
bands = (['F814W'] if args.fithst else
[]) + (args.bands_hsc if (args.fithsc or args.fithst2hsc) else [])
for nameattr in ['img_plot_maxs', 'weights_band']:
attr = getattr(args, nameattr)
if attr is not None:
if len(bands) != len(attr):
raise ValueError('len({}={})={} != len(bands={})={}'.format(
nameattr, attr, len(attr), bands, len(bands)))
setattr(args, nameattr,
{key: value
for key, value in zip(bands, attr)})
print('Loading COSMOS catalog at ' +
os.path.join(args.catalogpath, args.catalogfile))
try:
rgcat = gs.RealGalaxyCatalog(args.catalogfile, dir=args.catalogpath)
except RuntimeError as err:
print("Failed to load RealGalaxyCatalog {} in directory {} due to {}".
format(args.catalogfile, args.catalogpath, err))
raise err
if 'hst2hsc' in srcs:
try:
ccat = gs.COSMOSCatalog(args.catalogfile, dir=args.catalogpath)
except RuntimeError as err:
print("Failed to load COSMOSCatalog {} in directory {} due to {}".
format(args.catalogfile, args.catalogpath, err))
raise err
else:
ccat = None
rgcfits = ap.io.fits.open(os.path.join(args.catalogpath,
args.catalogfile))[1].data
nfit = 0
time_mpf_total = 0
time_start = time.time()
for index in args.indices:
idrange = [np.int(x) for x in index.split(",")]
for idnum in range(idrange[0], idrange[0 + (len(idrange) > 1)] + 1):
print("Fitting COSMOS galaxy with ID: {}".format(idnum))
time_now = time.time()
try:
np.random.seed(idnum)
radec = rgcfits[idnum][1:3]
scale_hst = rgcfits[idnum]['PIXEL_SCALE']
fits = fit_galaxy_cosmos(
radec=radec,
id_cosmos_gs=idnum,
srcs=srcs,
modelspecs=modelspecs,
rgcat=rgcat,
ccat=ccat,
butler=butler,
skymap=skymap,
plot=args.plot,
redo=args.redo,
redo_psfs=args.redo_psfs,
reset_images=True,
model_name_hst2hsc=args.model_name_hst2hsc,
bands_hsc=args.bands_hsc,
scale_hst=scale_hst,
modellib=args.modellib,
results=data[idnum] if idnum in data else None,
img_plot_maxs=args.img_plot_maxs,
img_multi_plot_max=args.img_multi_plot_max,
weights_band=args.weights_band,
do_fit_fluxfracs=args.do_fit_fluxfracs,
print_step_interval=args.print_step)
data[idnum] = fits
except Exception as e:
print(f"Error fitting id={idnum}: {e}")
trace = traceback.format_exc()
print(trace)
if idnum not in data:
try:
pickle.loads(pickle.dumps(e))
except Exception as te:
e = RuntimeError(
str(type(e)) + str(e) + "; pickling error:" +
str(te))
data[idnum] = {'error': e, 'trace': trace}
# bit of a hack: galsim won't evaluate the gradient
time_mpf = np.sum([
np.sum([y['time'] for y in x.fits if y['n_eval_grad'] > 0])
for x in data[idnum]['hst']['fits']['galsim'].values()
])
print(
f"Finished fitting COSMOS galaxy ID={idnum} in {time.time() - time_now:.2f} seconds"
f" ({time_mpf:.3f} MultiProFit time)")
time_mpf_total += time_mpf
nfit += 1
if args.write and args.file is not None and (nfit %
args.nwrite) == 0:
with open(args.file, 'wb') as f:
pickle.dump(data, f)
print(
f"Finished fitting {nfit} galaxies in {time.time() - time_start:.2f} seconds"
f" ({time_mpf_total:.3f} MultiProFit time)")
if args.write and args.file is not None:
with open(args.file, 'wb') as f:
pickle.dump(data, f)
if args.plot:
input("Press Enter to finish")
def distributed_strategy(args):
print(
f"Started worker {THIS_WORKER} at {datetime.now().strftime('%y-%m-%d_%H-%M-%S')}"
)
options = tf.io.TFRecordOptions(compression_type=args.compression_type)
catalog = galsim.COSMOSCatalog(sample=args.sample,
dir=args.cosmos_dir,
exclusion_level=args.exclusion_level,
min_flux=args.min_flux)
n_galaxies = catalog.getNObjects()
cat_param = catalog.param_cat[catalog.orig_index]
sparams = cat_param['sersicfit']
cat_param = append_fields(cat_param, 'sersic_q', sparams[:, 3])
cat_param = append_fields(cat_param, 'sersic_n', sparams[:, 2])
with tf.io.TFRecordWriter(
os.path.join(args.output_dir, f"data_{THIS_WORKER}.tfrecords"),
options) as writer:
for index in range((THIS_WORKER - 1), n_galaxies, N_WORKERS):
gal = catalog.makeGalaxy(index,
noise_pad_size=args.pixels *
args.pixel_scale)
psf = gal.original_psf
# We save the corresponding attributes for this galaxy
if hasattr(args, 'attributes'):
params = cat_param[index]
attributes = {k: params[k] for k in args.attributes}
else:
attributes = None
# Apply the PSF
gal = galsim.Convolve(gal, psf)
# Compute sqrt of absolute noise power spectrum, at the resolution and stamp size of target image
ps = gal.noise._get_update_rootps(
(args.pixels, args.pixels),
wcs=galsim.PixelScale(args.pixel_scale))
# We draw the pixel image of the convolved image
im = gal.drawImage(nx=args.pixels,
ny=args.pixels,
scale=args.pixel_scale,
method='no_pixel',
use_true_center=False).array.astype('float32')
# preprocess image
# For correlated noise, we estimate that the sqrt of the Energy Spectral Density of the noise at (f_x=f_y=0)
# is a good estimate of the STD
if args.preprocess:
im = tf.nn.relu(im - ps[0, 0]).numpy(
) # subtract background, fold negative pixels to 0
im /= im.max() # normalize peak to 1
signal_pixels = np.sum(
im > args.signal_threshold
) # how many pixels have a value above a certain threshold
if signal_pixels < args.signal_pixels: # argument used to select only examples that are more distinct galaxy features (it does however bias the dataset in redshift space)
continue
# Draw a kimage of the galaxy, just to figure out what maxk is, there might
# be more efficient ways to do this though...
bounds = galsim.BoundsI(0, args.pixels // 2, -args.pixels // 2,
args.pixels // 2 - 1)
imG = gal.drawKImage(bounds=bounds,
scale=2. * np.pi /
(args.pixels * args.pixels),
recenter=False)
mask = ~(np.fft.fftshift(imG.array, axes=0) == 0)
# Draw the Fourier domain image of the galaxy, using x1 zero padding,
# and x2 subsampling
interp_factor = 2
padding_factor = 1
Nk = args.pixels * interp_factor * padding_factor
bounds = galsim.BoundsI(0, Nk // 2, -Nk // 2, Nk // 2 - 1)
imCp = psf.drawKImage(bounds=bounds,
scale=2. * np.pi /
(Nk * args.pixel_scale / interp_factor),
recenter=False)
# Transform the psf array into proper format, remove the phase
im_psf = np.abs(np.fft.fftshift(imCp.array,
axes=0)).astype('float32')
# The following comes from correlatednoise.py
rt2 = np.sqrt(2.)
shape = (args.pixels, args.pixels)
ps[0, 0] = rt2 * ps[0, 0]
# Then make the changes necessary for even sized arrays
if shape[1] % 2 == 0: # x dimension even
ps[0, shape[1] // 2] = rt2 * ps[0, shape[1] // 2]
if shape[0] % 2 == 0: # y dimension even
ps[shape[0] // 2, 0] = rt2 * ps[shape[0] // 2, 0]
# Both dimensions even
if shape[1] % 2 == 0:
ps[shape[0] // 2,
shape[1] // 2] = rt2 * ps[shape[0] // 2, shape[1] // 2]
# Apply mask to power spectrum so that it is very large outside maxk
ps = np.where(mask, np.log(ps**2), 10).astype('float32')
features = {
"image": _bytes_feature(im.tobytes()),
"height": _int64_feature(im.shape[0]),
"width": _int64_feature(im.shape[1]),
"psf": _bytes_feature(im_psf.tobytes()),
"ps":
_bytes_feature(ps.tobytes()), # power spectrum of the noise
}
# Adding the parameters provided
if attributes is not None:
for k in attributes:
features['attrs_' + k] = _float_feature(attributes[k])
record = tf.train.Example(features=tf.train.Features(
feature=features)).SerializeToString()
writer.write(record)
print(
f"Finished worker {THIS_WORKER} at {datetime.now().strftime('%y-%m-%d_%H-%M-%S')}"
)
value["default"] = default
parser.add_argument('-' + key, **value)
args = parser.parse_args()
args.catalogpath = os.path.expanduser(args.catalogpath)
modelspecs = getmodelspecs(None if args.modelspecfile is None else os.path.expanduser(args.modelspecfile))
try:
rgcat = gs.RealGalaxyCatalog(args.catalogfile, dir=args.catalogpath)
except Exception as e:
print("Failed to load RealGalaxyCatalog {} in directory {}".format(
args.catalogfile, args.catalogpath))
print("Exception:", e)
raise e
try:
ccat = gs.COSMOSCatalog(args.catalogfile, dir=args.catalogpath)
except Exception as e:
print("Failed to load COSMOSCatalog {} in directory {}".format(
args.catalogfile, args.catalogpath))
print("Not using COSMOSCatalog")
ccat = None
if args.fileout is not None and os.path.isfile(os.path.expanduser(args.fileout)):
with open(os.path.expanduser(args.fileout), 'rb') as f:
data = pickle.load(f)
else:
data = {}
if args.plot:
mpl.rcParams['image.origin'] = 'lower'
'/vol/aibn148/data1/bhernandez/PhD/PSF_HST/test_bg3_cdgstridegood.fits',
gsparams=big_fft_params,
scale=0.0165
) #focus/zero_cdgstridegood.fits', gsparams=big_fft_params, scale=0.0165) #1.*0.0231) #Pixel scale from header of image
#psf=psf.rotate(90*galsim.degrees)
print psf.calculateFWHM(scale=pixel_scale)
#psf= galsim.InterpolatedImage('/export/data1/bhernandez/PhD/PSF_HST/Tiny_Tim/result00.fits', gsparams=big_fft_params, scale=0.0231) #1.*0.0231) #Pixel scale from header of image
#print psf.calculateFWHM(scale=pixel_scale)
psf = psf.dilate(
(psf_fwhm) / psf.calculateFWHM(scale=pixel_scale)) #/pixel_scale
#psf = galsim.Gaussian(fwhm = psf_fwhm, gsparams=big_fft_params)
#psf = galsim.Moffat(beta=psf_beta, flux=1., fwhm=psf_fwhm)
#bp_file = os.path.join(galsim.meta_data.share_dir, 'wfc_F814W.dat.gz')
#bandpass = galsim.Bandpass(bp_file, wave_type='ang').thin().withZeropoint(25.94)
cosmos_cat = galsim.COSMOSCatalog(
) #dir=/vol/aibn148/data1/bhernandez/share/galsim/COSMOS_25.2_training_sample)
for n in range(initial_n, 16): #50):# #30):#43):
x = []
y = []
ellipticities = []
betas = []
size = []
flux = []
mag = []
sersic_indices = []
tmp_indices = []
images = []
indices = np.random.random_integers(0,
high=81490,
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from a processed COSMOS 2015 Catalog, scaled to match
anticipated SuperBIT 2021 observations
- The galaxy shape parameters are assigned in a probabilistic way through matching
galaxy fluxes and redshifts to similar GalSim-COSMOS galaxies (see A. Gill+ 2021)
"""
global logger
logging.basicConfig(format="%(message)s", level=logging.DEBUG, stream=sys.stdout)
logger = logging.getLogger("mock_superbit_data")
M = MPIHelper()
# Define some parameters we'll use below.
sbparams = SuperBITParameters(argv=argv)
# Set up the NFWHalo:
nfw = galsim.NFWHalo(mass=sbparams.mass, conc=sbparams.nfw_conc, redshift=sbparams.nfw_z_halo,
omega_m=sbparams.omega_m, omega_lam=sbparams.omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog, as well as catalog containing
# information from COSMOS fits like redshifts, hlr, etc.
# cosmos_cat = galsim.COSMOSCatalog(sbparams.cat_file_name, dir=sbparams.cosmosdir)
# fitcat = Table.read(os.path.join(sbparams.cosmosdir, sbparams.fit_file_name))
cosmos_cat = Table.read(os.path.join(sbparams.cosmosdir,sbparams.cat_file_name))
logger.info('Read in %d galaxies from catalog and associated fit info', len(cosmos_cat))
cluster_cat = galsim.COSMOSCatalog(sbparams.cluster_cat_name)
logger.debug('Read in %d cluster galaxies from catalog' % cosmos_cat.nobjects)
### Now create PSF. First, define Zernicke polynomial component
### note: aberrations were definined for lam = 550, and close to the
### center of the camera. The PSF degrades at the edge of the FOV
lam_over_diam = sbparams.lam * 1.e-9 / sbparams.tel_diam # radians
lam_over_diam *= 206265.
aberrations = numpy.zeros(38) # Set the initial size.
aberrations[0] = 0. # First entry must be zero
aberrations[1] = -0.00305127
aberrations[4] = -0.02474205 # Noll index 4 = Defocus
aberrations[11] = -0.01544329 # Noll index 11 = Spherical
aberrations[22] = 0.00199235
aberrations[26] = 0.00000017
aberrations[37] = 0.00000004
logger.info('Calculated lambda over diam = %f arcsec', lam_over_diam)
# will store the Zernicke component of the PSF
optics = galsim.OpticalPSF(lam=sbparams.lam,diam=sbparams.tel_diam,
obscuration=sbparams.obscuration, nstruts=sbparams.nstruts,
strut_angle=sbparams.strut_angle, strut_thick=sbparams.strut_thick,
aberrations=aberrations)
logger.info('Made telescope PSF profile')
###
### MAKE SIMULATED OBSERVATIONS
### ITERATE n TIMES TO MAKE n SEPARATE IMAGES
###
for i in numpy.arange(1,sbparams.nexp+1):
# get MPI processes in sync at start of each image
M.barrier()
#rng = galsim.BaseDeviate(sbparams.noise_seed+i)
try:
timescale=str(sbparams.exp_time)
outname=''.join(['superbit_gaussJitter_',str(i).zfill(3),'.fits'])
truth_file_name=''.join([sbparams.outdir, '/truth_gaussJitter_', str(i).zfill(3), '.dat'])
file_name = os.path.join(sbparams.outdir, outname)
except galsim.errors.GalSimError:
print("naming failed, check path")
pdb.set_trace()
# Setting up a truth catalog
names = [ 'gal_num', 'x_image', 'y_image',
'ra', 'dec', 'g1_meas', 'g2_meas', 'nfw_mu', 'redshift','flux','truth_fwhm','truth_mom',
'n','hlr','inclination','scale_h_over_r']
types = [ int, float, float, float,float,float,
float, float, float, float, float, float,
float, float, float, float]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(sbparams.image_xsize, sbparams.image_ysize)
sky_level = sbparams.exp_time * sbparams.sky_bkg
full_image.fill(sky_level)
full_image.setOrigin(0,0)
# If you wanted to make a non-trivial WCS system, could set theta to a non-zero number
theta = 0.0 * galsim.degrees
dudx = numpy.cos(theta) * sbparams.pixel_scale
dudy = -numpy.sin(theta) * sbparams.pixel_scale
dvdx = numpy.sin(theta) * sbparams.pixel_scale
dvdy = numpy.cos(theta) * sbparams.pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx, dudy, dvdx, dvdy, origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=sbparams.center_ra, dec=sbparams.center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
## Now let's read in the PSFEx PSF model, if using.
## We read the image directly into an InterpolatedImage GSObject,
## so we can manipulate it as needed
#psf_wcs=wcs
#psf = galsim.des.DES_PSFEx(psf_filen,wcs=psf_wcs)
#logger.info('Constructed PSF object from PSFEx file')
#####
## Loop over galaxy objects:
#####
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nobj)
for k in range(local_start, local_end):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(sbparams.galobj_seed+k+1)
try:
# make single galaxy object
stamp,truth = make_a_galaxy(ud=ud,wcs=wcs,affine=affine,
cosmos_cat=cosmos_cat,optics=optics,nfw=nfw,
sbparams=sbparams)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
# noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2-time1
logger.info('Galaxy %d positioned relative to center t=%f s\n',
k, tot_time)
this_flux=numpy.sum(stamp.array)
row = [ k,truth.x, truth.y, truth.ra, truth.dec, truth.g1, truth.g2, truth.mu,truth.z,
this_flux,truth.fwhm, truth.mom_size,
truth.n, truth.hlr, truth.inclination, truth.scale_h_over_r]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Galaxy %d has failed, skipping...',k)
#####
### Inject cluster galaxy objects:
#####
center_coords = galsim.CelestialCoord(sbparams.center_ra,sbparams.center_dec)
centerpix = wcs.toImage(center_coords)
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nclustergal)
for k in range(local_start, local_end):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(sbparams.cluster_seed+k+1)
try:
# make single galaxy object
cluster_stamp,truth = make_cluster_galaxy(ud=ud,wcs=wcs,affine=affine,
centerpix=centerpix,
cluster_cat=cluster_cat,
optics=optics,
sbparams=sbparams)
# Find the overlapping bounds:
bounds = cluster_stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
#noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += cluster_stamp[bounds]
time2 = time.time()
tot_time = time2-time1
logger.info('Cluster galaxy %d positioned relative to center t=%f s\n',
k, tot_time)
this_flux=numpy.sum(stamp.array)
row = [ k,truth.x, truth.y, truth.ra, truth.dec, truth.g1, truth.g2, truth.mu,truth.z,
this_flux,truth.fwhm,truth.mom_size,
truth.n, truth.hlr, truth.inclination, truth.scale_h_over_r]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Cluster galaxy %d has failed, skipping...',k)
#####
### Now repeat process for stars!
#####
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nstars)
for k in range(local_start, local_end):
time1 = time.time()
ud = galsim.UniformDeviate(sbparams.stars_seed+k+1)
star_stamp,truth = make_a_star(ud=ud, wcs=wcs, affine=affine,
optics=optics, sbparams=sbparams)
bounds = star_stamp.bounds & full_image.bounds
# Add the stamp to the full image.
try:
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2-time1
logger.info('Star %d: positioned relative to center, t=%f s',
k, tot_time)
this_flux=numpy.sum(star_stamp.array)
row = [ k,truth.x, truth.y, truth.ra, truth.dec, truth.g1, truth.g2, truth.mu,
truth.z, this_flux,truth.fwhm,truth.mom_size,
truth.n, truth.hlr, truth.inclination, truth.scale_h_over_r]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Star %d has failed, skipping...',k)
# Gather results from MPI processes, reduce to single result on root
# Using same names on left and right sides is hiding lots of MPI magic
full_image = M.gather(full_image)
truth_catalog = M.gather(truth_catalog)
if M.is_mpi_root():
full_image = reduce(combine_images, full_image)
truth_catalog = reduce(combine_catalogs, truth_catalog)
else:
# do the adding of noise and writing to disk entirely on root
# root and the rest meet again at barrier at start of loop
continue
# The first thing to do is to make the Gaussian noise uniform across the whole image.
# Add dark current
logger.info('Adding Dark current')
dark_noise = sbparams.dark_current * sbparams.exp_time
full_image += dark_noise
# Add ccd noise
logger.info('Adding CCD noise')
noise = galsim.CCDNoise(
sky_level=0, gain=1/sbparams.gain,
read_noise=sbparams.read_noise)
full_image.addNoise(noise)
logger.debug('Added noise to final output image')
if not os.path.exists(os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
full_image.write(file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
logger.info('Wrote image to %r',file_name)
logger.info(' ')
logger.info('completed all images')
logger.info(' ')
def maybe_download_cosmos(target_dir, sample="25.2"):
"""
Checks for already accessible cosmos data, downloads it somewhere otherwise
"""
import logging
logging_level = logging.INFO
# Setup logging to go to sys.stdout or (if requested) to an output file
logging.basicConfig(format="%(message)s",
level=logging_level,
stream=sys.stdout)
logger = logging.getLogger('galaxy2galaxy:galsim')
try:
catalog = galsim.COSMOSCatalog(sample=sample)
return
except:
try:
catalog = galsim.COSMOSCatalog(sample=sample, dir=target_dir)
return
except:
logger.info("Couldn't access dataset, re-downloading")
url = "https://zenodo.org/record/3242143/files/COSMOS_%s_training_sample.tar.gz" % (
sample)
file_name = os.path.basename(url)
target = os.path.join(target_dir, file_name)
unpack_dir = target[:-len('.tar.gz')]
args = argparse.Namespace()
args.quiet = True
args.force = False
args.verbosity = 2
args.save = True
# Download the tarball
new_download, target, meta = galsim.download_cosmos.download(
url, target, unpack_dir, args, logger)
# Usually we unpack if we downloaded the tarball
do_unpack = new_download
# If the unpack dir is missing, then need to unpack
if not os.path.exists(unpack_dir):
do_unpack = True
# But of course if there is no tarball, we can't unpack it
if not os.path.isfile(target):
do_unpack = False
# Unpack the tarball
if do_unpack:
galsim.download_cosmos.unpack(target, target_dir, unpack_dir, meta,
args, logger)
# Usually, we remove the tarball if we unpacked it and command line doesn't specify to save it.
do_remove = do_unpack
# Remove the tarball
if do_remove:
logger.info("Removing the tarball to save space")
os.remove(target)
def image_generator_real(cosmos_cat_dir,
training_or_test,
isolated_or_blended,
constants_dir,
used_idx=None,
nmax_blend=4,
max_try=3,
mag_cut=28.,
method_first_shift='noshift',
do_peak_detection=True):
"""
Return numpy arrays: noiseless and noisy image of single galaxy and of blended galaxies as well as the pandaframe including data about the image and the shifts in the test sample generation configuration
Parameters:
----------
cosmos_cat_dir: COSMOS catalog directory
training_or_test: choice for generating a training or testing dataset
isolated_or_blended: choice for generation of samples of isolated galaxy images or blended galaxies images
constants_dir: directory where normalization constants are saved
used_idx: indexes to use in the catalog (to use different parts of the catalog for training/validation/test)
nmax_blend: maximum number of galaxies in a blended galaxies image
max_try: maximum number of try before leaving the function (to avoir infinite loop)
mag_cut: cut in magnitude to select function below this magnitude
method_first_shift: chosen method for shifting the centered galaxy
do_peak_detection: boolean to do the peak detection
"""
# Import the COSMOS catalog
cosmos_cat = galsim.COSMOSCatalog('real_galaxy_catalog_25.2.fits',
dir=cosmos_cat_dir)
counter = 0
np.random.seed() # important for multiprocessing !
assert training_or_test in ['training', 'validation', 'test']
assert isolated_or_blended in ['blended', 'isolated']
while counter < max_try:
try:
ud = galsim.UniformDeviate()
real_gal_list = []
nb_blended_gal = np.random.randint(nmax_blend) + 1
data = {}
galaxies = []
mag = []
mag_ir = []
j = 0
while j < nb_blended_gal:
# Chose the part of the catalog used for generation
if used_idx is not None:
idx = np.random.choice(used_idx)
else:
idx = np.random.randint(cosmos_cat.nobject)
# Generate galaxy
gal = cosmos_cat.makeGalaxy(idx,
gal_type='parametric',
chromatic=True,
noise_pad_size=0)
# Compute the magnitude of the galaxy
_mag_temp = gal.calculateMagnitude(
filters['r'].withZeropoint(28.13))
# Magnitude cut
if _mag_temp < mag_cut:
gal = gal.rotate(ud() * 360. * galsim.degrees)
galaxies.append(gal)
mag.append(_mag_temp)
mag_ir.append(
gal.calculateMagnitude(
filters['H'].withZeropoint(24.92 -
22.35 * coeff_noise_h)))
j += 1
# Take the real galaxy image only if parametric galaxy is actually created
if len(galaxies) == (len(real_gal_list) + 1):
bp_file = os.path.join(galsim.meta_data.share_dir,
'wfc_F814W.dat.gz')
bandpass = galsim.Bandpass(
bp_file, wave_type='ang').thin().withZeropoint(25.94)
real_gal = cosmos_cat.makeGalaxy(
idx,
gal_type='real',
noise_pad_size=max_stamp_size * pixel_scale_lsst)
real_gal_list.append(real_gal)
# Optionally, find the brightest and put it first in the list
if center_brightest:
_idx = np.argmin(mag)
galaxies.insert(0, galaxies.pop(_idx))
real_gal_list.insert(0, real_gal_list.pop(_idx))
mag.insert(0, mag.pop(_idx))
mag_ir.insert(0, mag_ir.pop(_idx))
# Shift galaxies
shift = np.zeros((nmax_blend, 2))
# Shift the lowest magnitude galaxy
galaxies[0], shift[0] = shift_gal(galaxies[0],
method=method_first_shift,
max_dx=0.1)
# Shift all the other galaxies
for j, gal in enumerate(galaxies[1:]):
galaxies[j + 1], shift[j + 1] = shift_gal(gal,
shift_x0=shift[0, 0],
shift_y0=shift[0, 1],
min_r=0.65 / 2.,
max_r=1.5,
method='annulus')
# Compute distances of the neighbour galaxies to the lowest magnitude galaxy
if nb_blended_gal > 1:
distances = [
shift[j][0]**2 + shift[j][1]**2
for j in range(1, nb_blended_gal)
]
idx_closest_to_peak_galaxy = np.argmin(distances) + 1
else:
idx_closest_to_peak_galaxy = 0
galaxy_noiseless = np.zeros((10, max_stamp_size, max_stamp_size))
blend_noisy = np.zeros((10, max_stamp_size, max_stamp_size))
galaxy_noiseless_real = np.zeros(
(10, max_stamp_size, max_stamp_size))
blend_noisy_real = np.zeros((10, max_stamp_size, max_stamp_size))
# Realize peak detection in r-band filter if asked
if do_peak_detection:
band = 6
galaxies_psf = [
galsim.Convolve([gal * coeff_exp[band], PSF[band]])
for gal in galaxies
]
images, blend_img = draw_images(galaxies_psf, band,
max_stamp_size * 2, 'r',
sky_level_pixel[band])
blend_noisy_temp = blend_img.array.data
peak_detection_output = peak_detection(blend_noisy_temp,
band,
shift,
max_stamp_size * 2,
4,
nb_blended_gal,
training_or_test,
dist_cut=0.65 / 2.)
if not peak_detection_output:
print('No peak detected')
raise RuntimeError
else:
idx_closest_to_peak, idx_closest_to_peak_galaxy, center_pix_x, center_pix_y, center_arc_x, center_arc_y, n_peak = peak_detection_output
# Modify galaxies and shift accordingly
galaxies = [
gal.shift(-center_arc_x, -center_arc_y) for gal in galaxies
]
shift[:nb_blended_gal] -= np.array(
[center_arc_x, center_arc_y])
# shift galaxies for centered configuration 'noshift'
for k, gal in enumerate(real_gal_list):
real_gal_list[k], shift[k] = shift_gal(gal,
shift_x0=shift[k, 0],
shift_y0=shift[k, 1],
method='noshift')
# Draw real images
galaxies_real_psf = [
galsim.Convolve([real_gal * coeff_exp[6], PSF_lsst])
for real_gal in real_gal_list
]
images_real, _ = draw_images(galaxies_real_psf,
6,
max_stamp_size,
'r',
sky_level_pixel[6],
real_or_param='real')
# Now draw image in all bands
for i, filter_name in enumerate(filter_names_all):
galaxies_psf = [
galsim.Convolve([gal * coeff_exp[i], PSF[i]])
for gal in galaxies
]
images, blend_img = draw_images(galaxies_psf, i,
max_stamp_size, filter_name,
sky_level_pixel[i])
if isolated_or_blended == 'isolated' or not do_peak_detection:
idx_closest_to_peak = 0
n_peak = 1
galaxy_noiseless[i] = images[idx_closest_to_peak].array.data
blend_noisy[i] = blend_img.array.data
# Rescale real images by flux
images_real_array = np.zeros(
(len(images_real), max_stamp_size, max_stamp_size))
for jj, image_real in enumerate(images_real):
img_temp = images[jj]
image_real -= np.min(image_real.array)
images_real_array[jj] = image_real.array * np.sum(
img_temp.array) / np.sum(image_real.array)
# real galaxies
galaxy_noiseless_real[i] = images_real_array[
idx_closest_to_peak].data
for image_real_array in images_real_array:
blend_noisy_real[i] += image_real_array\
# Add noise
blend_noisy_real_temp = galsim.Image(blend_noisy_real[i],
dtype=np.float64)
poissonian_noise = galsim.PoissonNoise(
rng, sky_level=sky_level_pixel[i])
blend_noisy_real_temp.addNoise(poissonian_noise)
blend_noisy_real[i] = blend_noisy_real_temp.array.data
# get data for the test sample
if training_or_test == 'test' and filter_name == 'r':
# need psf to compute ellipticities
psf_image = PSF[i].drawImage(nx=max_stamp_size,
ny=max_stamp_size,
scale=pixel_scale[i])
data['redshift'], data['moment_sigma'], data['e1'], data[
'e2'], data['mag'] = get_data(
galaxies[idx_closest_to_peak],
images[idx_closest_to_peak],
psf_image,
param_or_real='param'
) #real_gal_list[idx_closest_to_peak_galaxy], images_real
# Compute data and blendedness
if nb_blended_gal > 1:
data['closest_redshift'], data[
'closest_moment_sigma'], data['closest_e1'], data[
'closest_e2'], data['closest_mag'] = get_data(
galaxies[idx_closest_to_peak_galaxy],
images[idx_closest_to_peak_galaxy],
psf_image,
param_or_real='param'
) #real_gal_list[idx_closest_to_peak_galaxy]
img_central = images[idx_closest_to_peak].array
img_others = np.zeros_like(img_central)
for _h, image in enumerate(images):
if _h != idx_closest_to_peak:
img_others += image.array
img_closest_neighbour = images[
idx_closest_to_peak_galaxy].array
data[
'blendedness_total_lsst'] = utils.compute_blendedness_total(
img_central, img_others)
data[
'blendedness_closest_lsst'] = utils.compute_blendedness_single(
img_central, img_closest_neighbour)
data[
'blendedness_aperture_lsst'] = utils.compute_blendedness_aperture(
img_central, img_others, data['moment_sigma'])
else:
data['closest_redshift'] = np.nan
data['closest_moment_sigma'] = np.nan
data['closest_e1'] = np.nan
data['closest_e2'] = np.nan
data['closest_mag'] = np.nan
data['blendedness_total_lsst'] = np.nan
data['blendedness_closest_lsst'] = np.nan
data['blendedness_aperture_lsst'] = np.nan
break
except RuntimeError as e:
print(e)
# For training/validation, return normalized images only
if training_or_test in ['training', 'validation']:
galaxy_noiseless_real = utils.norm(galaxy_noiseless_real[None, :],
bands=range(10),
path=constants_dir)[0]
blend_noisy_real = utils.norm(blend_noisy_real[None, :],
bands=range(10),
path=constants_dir)[0]
return galaxy_noiseless_real, blend_noisy_real
# For testing, return unormalized images and data
elif training_or_test == 'test':
data['nb_blended_gal'] = nb_blended_gal
data['mag'] = mag[0]
data['mag_ir'] = mag_ir[0]
if nb_blended_gal > 1:
data['closest_mag'] = mag[idx_closest_to_peak_galaxy]
data['closest_mag_ir'] = mag_ir[idx_closest_to_peak_galaxy]
data['closest_x'] = shift[idx_closest_to_peak_galaxy][0]
data['closest_y'] = shift[idx_closest_to_peak_galaxy][1]
else:
data['closest_mag'] = np.nan
data['closest_mag_ir'] = np.nan
data['closest_x'] = np.nan
data['closest_y'] = np.nan
data['idx_closest_to_peak'] = idx_closest_to_peak
data['n_peak_detected'] = n_peak
data['SNR'] = utils.SNR(galaxy_noiseless, sky_level_pixel, band=6)[1]
data['SNR_peak'] = utils.SNR_peak(galaxy_noiseless,
sky_level_pixel,
band=6)[1]
return galaxy_noiseless_real, blend_noisy_real, data, shift
else:
raise ValueError
hst_eff_area = 2.4**2.0*(1.-0.33**2)
fluxScaling = (telesDiam**2/hst_eff_area) * expTime
logger.info("^_^ Working directory: %s", dirname)
logger.info("^_^ Start the simulation ...")
## start time
t0 = time.time()
# number of galaxies to be simulated
skySize = xsize*ysize*pixelScale**2/3600.0
ngal = int(denGal*skySize)
logger.info(" Total %d galaxies to be simulated.", ngal)
# Read in galaxy catalog
cosmosCat = galsim.COSMOSCatalog(cosCatn, dir=cosdir)
logger.info(" Import %d galaxies from COSMOS catalog", cosmosCat.nobjects)
gals_ids = cosmosCat.__dict__['param_cat']['IDENT']
gals_mags = cosmosCat.__dict__['param_cat']['mag_auto']
gals_flux = cosmosCat.__dict__['param_cat']['flux']
gals_zphot = cosmosCat.__dict__['param_cat']['zphot']
# Setup the image:
fullImage = galsim.ImageF(xsize, ysize)
fullImage.setOrigin(0,0)
rng = galsim.BaseDeviate(randomSeed)
noiseImage = galsim.ImageF(xsize, ysize)
noiseImage.setOrigin(0,0)
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles. We choose 40% of the galaxies
to use the images, and the other 60% to use the parametric fits
- The real galaxy images include some initial correlated noise from the original HST
observation. However, we whiten the noise of the final image so the final image has
stationary Gaussian noise, rather than correlated noise.
"""
global logger
logging.basicConfig(format="%(message)s",
level=logging.INFO,
stream=sys.stdout)
logger = logging.getLogger("mock_superbit_data")
# Define some parameters we'll use below.
# Normally these would be read in from some parameter file.
global pixel_scale
pixel_scale = 0.206 # arcsec/pixel
global image_xsize
image_xsize = 6665 # size of image in pixels
global image_ysize
image_ysize = 4453 # size of image in pixels
global image_xsize_arcsec
image_xsize_arcsec = image_xsize * pixel_scale # size of big image in each dimension (arcsec)
global image_ysize_arcsec
image_ysize_arcsec = image_ysize * pixel_scale # size of big image in each dimension (arcsec)
global center_ra
center_ra = 19.3 * galsim.hours # The RA, Dec of the center of the image on the sky
global center_dec
center_dec = -33.1 * galsim.degrees
global exp_time
exp_time = 300
global sky_bkg # mean sky background from AG's paper
sky_bkg = 0.32 # ADU / s / pix
global sky_sigma # standard deviation of sky background
sky_sigma = 0.16 # ADU / s / pix
global nobj
nobj = 22 # number of galaxies in entire field
global nstars
nstars = 300 # number of stars in the entire field
global flux_scaling
global tel_diam
tel_diam = 0.5
global lam
lam = 625 # Central wavelength for Airy disk
global optics
psf_path = '/Users/jemcclea/Research/GalSim/examples/data/fpsc_flight_jitter_psf_oversampled_fixed_10x'
global optics # will store the Zernicke component of the PSF
global nfw # will store the NFWHalo information
global cosmos_cat # will store the COSMOS catalog from which we draw objects
global example_cat # also a COSMOS catalog which will contain cluster galaxies
# Set up the NFWHalo:
mass = 5E14 # Cluster mass (Msol/h)
nfw_conc = 4 # Concentration parameter = virial radius / NFW scale radius
nfw_z_halo = 0.17 # redshift of the halo
omega_m = 0.3 # Omega matter for the background cosmology.
omega_lam = 0.7 # Omega lambda for the background cosmology.
nfw = galsim.NFWHalo(mass=mass,
conc=nfw_conc,
redshift=nfw_z_halo,
omega_m=omega_m,
omega_lam=omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog
cat_file_name = 'real_galaxy_catalog_25.2.fits'
dir = 'data/COSMOS_25.2_training_sample/'
cosmos_cat = galsim.COSMOSCatalog(cat_file_name, dir=dir)
logger.info('Read in %d galaxies from catalog', cosmos_cat.nobjects)
# Also read in example catalog
example_cat_file_name = 'data/real_galaxy_catalog_23.5_example.fits'
example_cat = galsim.COSMOSCatalog(example_cat_file_name)
# The catalog returns objects that are appropriate for HST in 1 second exposures. So for our
# telescope we scale up by the relative area, exposure time and pixel scale
hst_eff_area = 2.4**2 * (1. - 0.33**2)
sbit_eff_area = tel_diam**2 * (1. - 0.380**2)
flux_scaling = (sbit_eff_area / hst_eff_area) * exp_time * (pixel_scale /
.05)**2
### Now create PSF. First, define Zernicke polynomial component
lam_over_diam = lam * 1.e-9 / tel_diam # radians
lam_over_diam *= 206265 # arcsec
aberrations = [0.0] * 12 # Set the initial size.
aberrations[4] = -0.00725859 # Noll index 4 = Defocus
aberrations[5:7] = [0.0, -0.00] # Noll index 5,6 = Astigmatism
aberrations[7:9] = [0.07, 0.00] # Noll index 7,8 = Coma
aberrations[11] = 0.00133254 # Noll index 11 = Spherical
logger.info('Calculated lambda over diam = %f arcsec', lam_over_diam)
optics = galsim.OpticalPSF(lam_over_diam,
obscuration=0.380,
aberrations=aberrations)
logger.info('Made telescope PSF profile')
###
### LOOP OVER PSFs TO MAKE GROUPS OF IMAGES
### WITHIN EACH PSF, ITERATE 5 TIMES TO MAKE 5 SEPARATE IMAGES
###
all_psfs = glob.glob(psf_path + "/*247530*.psf") # this is 121s
logger.info('Beginning loop over jitter/optical psfs')
for psf_filen in all_psfs:
logger.info('Beginning PSF %s...' % psf_filen)
for i in numpy.arange(1, 2):
logger.info('Beginning loop %d' % i)
random_seed = 23058923781
rng = galsim.BaseDeviate(random_seed)
# This is specific to Javier mock PSFs
try:
root = psf_filen.split('data/')[1].split('/')[0]
timescale = psf_filen.split('_10x/')[1].split('.')[0]
outname = ''.join([
'mock_superbit_', root, timescale,
str(i).zfill(3), '.fits'
])
truth_file_name = ''.join([
'./output/truth_', root, timescale,
str(i).zfill(3), '.dat'
])
file_name = os.path.join('output', outname)
except:
pdb.set_trace()
# Setting up a truth catalog
names = [
'gal_num', 'x_image', 'y_image', 'ra', 'dec', 'g1_meas',
'g2_meas', 'nfw_mu', 'redshift', 'flux'
]
types = [
int, float, float, float, float, float, float, float, float,
float
]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(image_xsize, image_ysize)
sky_level = exp_time * sky_bkg
full_image.fill(sky_level)
full_image.setOrigin(0, 0)
# We keep track of how much noise is already in the image from the RealGalaxies.
noise_image = galsim.ImageF(image_xsize, image_ysize)
noise_image.setOrigin(0, 0)
# Make a slightly non-trivial WCS. We'll use a slightly rotated coordinate system
# and center it at the image center.
theta = 0.17 * galsim.degrees
dudx = numpy.cos(theta) * pixel_scale
dudy = -numpy.sin(theta) * pixel_scale
dvdx = numpy.sin(theta) * pixel_scale
dvdy = numpy.cos(theta) * pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx,
dudy,
dvdx,
dvdy,
origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=center_ra, dec=center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
# Now let's read in the PSFEx PSF model. We read the image directly into an
# InterpolatedImage GSObject, so we can manipulate it as needed
psf_wcs = wcs
psf_file = os.path.join(psf_path, psf_filen)
psf = galsim.des.DES_PSFEx(psf_file, wcs=psf_wcs)
logger.info('Constructed PSF object from PSFEx file')
# Loop over galaxy objects:
for k in range(nobj):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed + k + 1)
try:
# make single galaxy object
stamp, truth = make_a_galaxy(ud=ud,
wcs=wcs,
psf=psf,
affine=affine)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Galaxy %d positioned relative to center t=%f s', k,
tot_time)
this_flux = numpy.sum(stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux
]
truth_catalog.addRow(row)
except:
logger.info('Galaxy %d has failed, skipping...', k)
pdb.set_trace()
###### Inject cluster galaxy objects:
random_seed = 892465352
for k in range(50):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed + k + 1)
try:
# make single galaxy object
cluster_stamp, truth = make_cluster_galaxy(ud=ud,
wcs=wcs,
psf=psf,
affine=affine)
# Find the overlapping bounds:
bounds = cluster_stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += cluster_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Cluster galaxy %d positioned relative to center t=%f s',
k, tot_time)
this_flux = numpy.sum(stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux
]
truth_catalog.addRow(row)
except:
logger.info('Cluster galaxy %d has failed, skipping...', k)
pdb.set_trace()
####
### Now repeat process for stars!
####
random_seed_stars = 2308173501873
for k in range(nstars):
time1 = time.time()
ud = galsim.UniformDeviate(random_seed_stars + k + 1)
star_stamp, truth = make_a_star(ud=ud,
wcs=wcs,
psf=psf,
affine=affine)
bounds = star_stamp.bounds & full_image.bounds
# Add the stamp to the full image.
try:
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Star %d: positioned relative to center, t=%f s', k,
tot_time)
this_flux = numpy.sum(star_stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux
]
truth_catalog.addRow(row)
except:
logger.info('Star %d has failed, skipping...', k)
pass
# If real-type COSMOS galaxies are used, the noise across the image won't be uniform. Since this code is
# using parametric-type galaxies, the following section is commented out.
#
# The first thing to do is to make the Gaussian noise uniform across the whole image.
max_current_variance = numpy.max(noise_image.array)
noise_image = max_current_variance - noise_image
vn = galsim.VariableGaussianNoise(rng, noise_image)
full_image.addNoise(vn)
# Now max_current_variance is the noise level across the full image. We don't want to add that
# twice, so subtract off this much from the intended noise that we want to end up in the image.
sky_sigma -= numpy.sqrt(max_current_variance)
# Regardless of galaxy type, add Gaussian noise with this variance to the final image.
this_noise_sigma = sky_sigma * exp_time
noise = galsim.GaussianNoise(rng, sigma=this_noise_sigma)
full_image.addNoise(noise)
logger.debug('Added noise to final output image')
full_image.write(file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
logger.info('Wrote image to %r', file_name)
logger.info(' ')
logger.info('completed run %d for psf %s', i, psf_filen)
i = i + 1
logger.info(' ')
logger.info(' ')
logger.info('completed all images')
logger.info(' ')
def generator(self, data_dir, tmp_dir, dataset_split, task_id=-1):
"""
Generates and yields poattrs2img_cosmos64_euclid_pixscale_1_wmorpho_2stage stamps obtained with GalSim.
"""
p = self.get_hparams()
try:
# try to use default galsim path to the data
catalog = galsim.COSMOSCatalog()
except:
# If that fails, tries to use the specified tmp_dir
catalog = galsim.COSMOSCatalog(dir=tmp_dir +
'/COSMOS_25.2_training_sample')
# Create a list of galaxy indices for this task, remember, there is a task
# per shard, each shard is 1000 galaxies.
assert (task_id > -1)
index = range(
task_id * p.example_per_shard,
min((task_id + 1) * p.example_per_shard, catalog.getNObjects()))
# Extracts additional information about the galaxies
cat_param = catalog.param_cat[catalog.orig_index]
from numpy.lib.recfunctions import append_fields
import numpy as np
bparams = cat_param['bulgefit']
sparams = cat_param['sersicfit']
# Parameters for a 2 component fit
cat_param = append_fields(cat_param, 'bulge_q', bparams[:, 11])
cat_param = append_fields(cat_param, 'bulge_beta', bparams[:, 15])
cat_param = append_fields(cat_param, 'disk_q', bparams[:, 3])
cat_param = append_fields(cat_param, 'disk_beta', bparams[:, 7])
cat_param = append_fields(cat_param, 'bulge_hlr', cat_param['hlr'][:,
1])
cat_param = append_fields(
cat_param, 'bulge_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 1]),
np.zeros(len(cat_param))))
cat_param = append_fields(cat_param, 'disk_hlr', cat_param['hlr'][:,
2])
cat_param = append_fields(
cat_param, 'disk_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 2]),
np.log10(cat_param['flux'][:, 0])))
# Parameters for a single component fit
cat_param = append_fields(cat_param, 'sersic_flux_log10',
np.log10(sparams[:, 0]))
cat_param = append_fields(cat_param, 'sersic_q', sparams[:, 3])
cat_param = append_fields(cat_param, 'sersic_hlr', sparams[:, 1])
cat_param = append_fields(cat_param, 'sersic_n', sparams[:, 2])
cat_param = append_fields(cat_param, 'sersic_beta', sparams[:, 7])
passed = 0
late = 0
irr = 0
for ind in index:
# Draw a galaxy using GalSim, any kind of operation can be done here
# if cat_param['Mph3'][ind] not in [1,2,3] :
# passed += 1
# continue
# if cat_param['Mph3'][ind] == 2:
# if late >= 85 :
# if ind % 200 == 0:
# print("done all the late of the id")
# continue
# late += 1
# if cat_param['Mph3'][ind] == 3:
# if irr >= 85 :
# if ind % 200 == 0:
# print("done all the irr of the id")
# continue
# irr += 1
gal = catalog.makeGalaxy(ind,
noise_pad_size=p.img_len * p.pixel_scale)
# We apply the orginal psf if a different PSF is not requested
if ~hasattr(p, "psf") or p.psf is None:
psf = gal.original_psf
else:
psf = p.psf
# Apply rotation so that the galaxy is at 0 PA
# if hasattr(p, "rotation") and p.rotation:
# rotation_angle = galsim.Angle(-cat_param[ind]['sersic_beta'],
# galsim.radians)
# gal = gal.rotate(rotation_angle)
# psf = psf.rotate(rotation_angle)
# We save the corresponding attributes for this galaxy
if hasattr(p, 'attributes'):
params = cat_param[ind]
attributes = {k: params[k] for k in p.attributes}
else:
attributes = None
flux_r = [1.0]
for i in range(1, self.num_bands):
flux_r.append(
max(
np.random.normal(p.flux_ratio_mean[i],
p.flux_ratio_std[i]), 0))
# Utility function encodes the postage stamp for serialized features
yield galsim_utils.draw_and_encode_stamp(gal,
psf,
stamp_size=p.img_len,
pixel_scale=p.pixel_scale,
attributes=attributes,
flux_r=flux_r,
num_bands=self.num_bands)
def test_cosmos_basic():
"""Check some basic functionality of the COSMOSCatalog class."""
# Note, there's not much here yet. Could try to think of other tests that are more
# interesting.
# Initialize a COSMOSCatalog with all defaults.
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
# Initialize one that doesn't exclude failures. It should be >= the previous one in length.
cat2 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='none')
assert cat2.nobjects >= cat.nobjects
assert len(cat2) == cat2.nobjects == cat2.getNTot() == 100
assert len(cat) == cat.nobjects < cat.getNTot()
# Check other exclusion levels:
cat3 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='bad_stamp')
assert len(cat3) == 97
cat4 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='bad_fits')
assert len(cat4) == 100 # no bad fits in the example file as it happens.
cat5 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='marginal')
assert len(cat5) == 96 # this is actually the default, so == cat
assert cat == cat5
# Check the 25.2 exclusions. We don't have a 25.2 catalog available in Travis runs, so
# mock up the example catalog as though it were 25.2
# Also check the min/max hlr and flux options.
cat2.use_sample = '25.2'
hlr = cat2.param_cat['hlr'][:, 0]
flux = cat2.param_cat['flux'][:, 0]
print("Full range of hlr = ", np.min(hlr), np.max(hlr))
print("Full range of flux = ", np.min(flux), np.max(flux))
cat2._apply_exclusion('marginal',
min_hlr=0.2,
max_hlr=2,
min_flux=50,
max_flux=5000)
print("Size of catalog with hlr and flux exclusions == ", len(cat2))
assert len(cat2) == 47
# Check for reasonable exceptions when initializing.
# Can't find data (wrong directory).
with assert_raises(OSError):
galsim.COSMOSCatalog(file_name='real_galaxy_catalog_23.5_example.fits')
# Try making galaxies
gal_real = cat.makeGalaxy(index=0, gal_type='real', chromatic=False)
assert isinstance(gal_real, galsim.RealGalaxy)
gal_param = cat.makeGalaxy(index=10, gal_type='parametric', chromatic=True)
assert isinstance(gal_param, galsim.ChromaticObject)
# Second time through, don't make the bandpass.
bp = cat._bandpass
sed = cat._sed
assert bp is not None
gal_param2 = cat.makeGalaxy(index=13,
gal_type='parametric',
chromatic=True)
assert isinstance(gal_param2, galsim.ChromaticObject)
assert gal_param != gal_param2
assert cat._bandpass is bp # Not just ==. "is" means the same object.
assert cat._sed is sed
# So far, we've made a bulge+disk and a disky Sersic.
# Do two more to run through two more paths in the code.
gal_param3 = cat.makeGalaxy(index=50,
gal_type='parametric',
chromatic=True)
gal_param4 = cat.makeGalaxy(index=67,
gal_type='parametric',
chromatic=True)
gal_real_list = cat.makeGalaxy(index=[3, 6],
gal_type='real',
chromatic=False)
for gal_real in gal_real_list:
assert isinstance(gal_real, galsim.RealGalaxy)
gal_param_list = cat.makeGalaxy(index=[4, 7],
gal_type='parametric',
chromatic=False)
for gal_param in gal_param_list:
assert isinstance(gal_param, galsim.GSObject)
# Check for parametric catalog
# Can give either the regular name or the _fits name.
cat_param = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
use_real=False)
cat_param2 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example_fits.fits',
dir=datapath,
use_real=False)
assert cat_param2 == cat_param
# Try making galaxies
gal = cat_param.makeGalaxy(index=1)
assert isinstance(gal, galsim.GSObject)
gal_list = cat_param.makeGalaxy(index=[2, 3])
for gal in gal_list:
assert isinstance(gal, galsim.GSObject)
# Check sersic_prec option.
sersic0 = cat_param.makeGalaxy(index=59, sersic_prec=0)
np.testing.assert_almost_equal(sersic0.original.n, 1.14494567108)
sersic1 = cat_param.makeGalaxy(index=59, sersic_prec=0.05) # The default.
np.testing.assert_almost_equal(sersic1.original.n, 1.15)
sersic2 = cat_param.makeGalaxy(index=59, sersic_prec=0.1)
np.testing.assert_almost_equal(sersic2.original.n, 1.1)
sersic3 = cat_param.makeGalaxy(index=59, sersic_prec=0.5)
np.testing.assert_almost_equal(sersic3.original.n, 1.0)
assert_raises(TypeError,
galsim.COSMOSCatalog,
'real_galaxy_catalog_23.5_example.fits',
dir=datapath,
sample='23.5')
assert_raises(ValueError, galsim.COSMOSCatalog, sample='invalid')
assert_raises(ValueError, cat_param.makeGalaxy, gal_type='real')
assert_raises(ValueError, cat_param.makeGalaxy, gal_type='invalid')
assert_raises(ValueError, cat.makeGalaxy, gal_type='invalid')
assert_raises(TypeError, cat_param.makeGalaxy, rng='invalid')
assert_raises(NotImplementedError,
cat.makeGalaxy,
gal_type='real',
chromatic=True)
def generator(self, data_dir, tmp_dir, dataset_split, task_id=-1):
"""
Generates and yields postage stamps obtained with GalSim.
"""
p = self.get_hparams()
try:
# try to use default galsim path to the data
catalog = galsim.COSMOSCatalog()
except:
# If that fails, tries to use the specified tmp_dir
catalog = galsim.COSMOSCatalog(dir=tmp_dir +
'/COSMOS_25.2_training_sample')
# Create a list of galaxy indices for this task, remember, there is a task
# per shard, each shard is 1000 galaxies.
assert (task_id > -1)
index = range(
task_id * p.example_per_shard,
min((task_id + 1) * p.example_per_shard, catalog.getNObjects()))
# Extracts additional information about the galaxies
cat_param = catalog.param_cat[catalog.orig_index]
from numpy.lib.recfunctions import append_fields
import numpy as np
bparams = cat_param['bulgefit']
sparams = cat_param['sersicfit']
# Parameters for a 2 component fit
cat_param = append_fields(cat_param, 'bulge_q', bparams[:, 11])
cat_param = append_fields(cat_param, 'bulge_beta', bparams[:, 15])
cat_param = append_fields(cat_param, 'disk_q', bparams[:, 3])
cat_param = append_fields(cat_param, 'disk_beta', bparams[:, 7])
cat_param = append_fields(cat_param, 'bulge_hlr', cat_param['hlr'][:,
1])
cat_param = append_fields(
cat_param, 'bulge_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 1]),
np.zeros(len(cat_param))))
cat_param = append_fields(cat_param, 'disk_hlr', cat_param['hlr'][:,
2])
cat_param = append_fields(
cat_param, 'disk_flux_log10',
np.where(cat_param['use_bulgefit'] == 1,
np.log10(cat_param['flux'][:, 2]),
np.log10(cat_param['flux'][:, 0])))
# Parameters for a single component fit
cat_param = append_fields(cat_param, 'sersic_flux_log10',
np.log10(sparams[:, 0]))
cat_param = append_fields(cat_param, 'sersic_q', sparams[:, 3])
cat_param = append_fields(cat_param, 'sersic_hlr', sparams[:, 1])
cat_param = append_fields(cat_param, 'sersic_n', sparams[:, 2])
cat_param = append_fields(cat_param, 'sersic_beta', sparams[:, 7])
for ind in index:
# Draw a galaxy using GalSim, any kind of operation can be done here
gal = catalog.makeGalaxy(ind,
noise_pad_size=p.img_len * p.pixel_scale *
2)
# We apply the orginal psf if a different PSF is not requested
if hasattr(p, "psf"):
psf = p.psf
else:
psf = gal.original_psf
# Apply random rotation if requested
if hasattr(p, "rotation") and p.rotation:
rotation_angle = galsim.Angle(-np.random.rand() * 2 * np.pi,
galsim.radians)
gal = gal.rotate(rotation_angle)
psf = psf.rotate(rotation_angle)
# We save the corresponding attributes for this galaxy
if hasattr(p, 'attributes'):
params = cat_param[ind]
attributes = {k: params[k] for k in p.attributes}
else:
attributes = None
# Utility function encodes the postage stamp for serialized features
yield galsim_utils.draw_and_encode_stamp(gal,
psf,
stamp_size=p.img_len,
pixel_scale=p.pixel_scale,
attributes=attributes)
def run(self, exposure, background):
self.log.info("Adding fake galaxies at real positions")
PARENT = lsst.afw.image.PARENT
psf = exposure.getPsf()
md = exposure.getMetadata()
calib = exposure.getCalib()
expBBox = exposure.getBBox(PARENT)
wcs = exposure.getWcs()
skyToPixelMatrix = (wcs.getLinearTransform().invert().getMatrix() /
3600.0)
"""Deal with the skipped ones."""
skipLog = 'runAddFake.skipped'
if not os.path.isfile(skipLog):
os.system('touch ' + skipLog)
if self.config.galType is 'cosmos':
import galsim
exLevel = self.config.exclusionLevel
cosmosCat = galsim.COSMOSCatalog(exclusion_level=exLevel)
else:
cosmosCat = None
for igal, gal in enumerate(self.galData):
try:
galident = gal["ID"]
except KeyError:
galident = igal + 1
try:
flux = calib.getFlux(float(gal['mag']))
except KeyError:
raise KeyError("No mag column in %s" % self.config.galList)
try:
coordAdd = lsst.afw.geom.Point2D(gal['RA'], gal['DEC'])
galCoord = lsst.afw.coord.Coord(coordAdd)
except KeyError:
raise("No RA/DEC column in %s table" % self.config.galList)
galXY = wcs.skyToPixel(galCoord)
bboxI = exposure.getBBox(PARENT)
bboxI.grow(self.config.maxMargin)
if not bboxI.contains(lsst.afw.geom.Point2I(galXY)):
# Will just skip this object
continue
# Check the magnitude
if gal['mag'] <= 0:
self.log.info("Mag <= 0: Skipping %d" % galident)
self.log.info(" mag: %7.3d" % gal['mag'])
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , negMag\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
# This is extrapolating for the PSF, probably not a good idea
# Return an Image of the PSF, in a form suitable for convolution.
# The returned image is normalized to sum to unity.
psfImage = psf.computeKernelImage(galXY)
try:
addShear = self.config.addShear
prec = self.config.sersic_prec
galType = self.config.galType
if self.config.galType is not 'cosmos':
galArray = makeFake.makeGalaxy(flux, gal,
psfImage.getArray(),
galType=galType,
cosmosCat=None,
calib=None,
addShear=addShear,
transform=skyToPixelMatrix)
else:
galArray = makeFake.makeGalaxy(flux, gal,
psfImage.getArray(),
cosmosCat=cosmosCat,
calib=calib,
galType=galType,
addShear=addShear,
sersic_prec=prec,
transform=skyToPixelMatrix)
except IndexError as ierr:
self.log.info("GalSim Index Error: Skipping %d" % galident)
self.log.info(ierr.message)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , galsimI\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
except KeyError as kerr:
self.log.info("GalSim Key Error: Skipping %d" % galident)
self.log.info(kerr.message)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , galsimK\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
except ValueError as verr:
self.log.info("GalSim Value Error: Skipping %d" % galident)
self.log.info(verr.message)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , galsimV\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
except RuntimeError as rerr:
self.log.info("GalSim Runtime Error: Skipping %d" % galident)
self.log.info(rerr.message)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , galsimR\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
except Exception as uerr:
self.log.info("Unexpected Error: Skipping %d" % galident)
self.log.info(uerr.message)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , Unexpected\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
galImage = lsst.afw.image.ImageF(galArray.astype(np.float32))
galBBox = galImage.getBBox(PARENT)
galX0 = (galXY.getX() - galBBox.getWidth()/2.0 + 0.5)
galY0 = (galXY.getY() - galBBox.getHeight()/2.0 + 0.5)
galImage = lsst.afw.math.offsetImage(galImage,
galX0, galY0,
'lanczos3')
galBBox = galImage.getBBox(PARENT)
# Check that we're within the larger exposure, otherwise crop
parentBox = galImage.getBBox(PARENT)
if expBBox.contains(parentBox) is False:
newBBox = galImage.getBBox(PARENT)
newBBox.clip(expBBox)
if newBBox.getArea() <= 0:
self.log.info("BBoxEdge Error: Skipping %d" % galident)
with open(skipLog, "a") as slog:
try:
fcntl.flock(slog, fcntl.LOCK_EX)
slog.write("%8d , bboxEdge\n" % galident)
fcntl.flock(slog, fcntl.LOCK_UN)
except IOError:
continue
continue
self.log.info("Cropping FAKE%d from %s to %s" % (galident,
str(galBBox), str(newBBox)))
galImage = galImage.Factory(galImage, newBBox,
PARENT)
galBBox = newBBox
# Add Noise: Optional?
galMaskedImage = fsl.addNoise(galImage, exposure.getDetector(),
rand_gen=self.npRand)
# Put information of the added fake galaxies into the header
md.set("FAKE%s" % str(galident), "%.3f, %.3f" % (galXY.getX(),
galXY.getY()))
self.log.info("Adding fake %s at: %.1f,%.1f" % (str(galident),
galXY.getX(),
galXY.getY()))
galMaskedImage.getMask().set(self.bitmask)
try:
galMaskedImage.getMask().removeAndClearMaskPlane('FAKE',
True)
except Exception:
pass
try:
galMaskedImage.getMask().removeAndClearMaskPlane('CROSSTALK',
True)
except Exception:
pass
try:
galMaskedImage.getMask().removeAndClearMaskPlane('UNMASKEDNAN',
True)
except Exception:
pass
maskedImage = exposure.getMaskedImage()
try:
maskedImage.getMask().removeAndClearMaskPlane('CROSSTALK',
True)
except Exception:
pass
try:
maskedImage.getMask().removeAndClearMaskPlane('UNMASKEDNAN',
True)
except Exception:
pass
try:
maskedImage.getMask().removeAndClearMaskPlane('FAKE', True)
except Exception:
pass
BBox = galMaskedImage.getBBox(PARENT)
subMaskedImage = maskedImage.Factory(exposure.getMaskedImage(),
BBox,
PARENT)
subMaskedImage += galMaskedImage
"""
def test_dep_scene():
"""Test the deprecated exclude_bad and exclude_fail args to COSMOSCatalog
"""
path, filename = os.path.split(__file__)
datapath = os.path.abspath(os.path.join(path, "../examples/data/"))
# Initialize one that doesn't exclude failures. It should be >= the previous one in length.
cat2 = check_dep(galsim.COSMOSCatalog,
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclude_fail=False,
exclude_bad=False)
# Initialize a COSMOSCatalog with all defaults.
cat = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits', dir=datapath)
assert cat2.nobjects >= cat.nobjects
# Equivalent to current exclusion_level='none'
cat3 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='none')
assert cat2.nobjects == cat3.nobjects
# Just exclude_bad=True
cat2 = check_dep(galsim.COSMOSCatalog,
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclude_fail=False) # i.e. leave exclude_bad=True
assert cat2.nobjects >= cat.nobjects
# Equivalent to current exclusion_level='bad_stamp'
cat3 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='bad_stamp')
assert cat2.nobjects == cat3.nobjects
# Just exclude_fail=True
cat2 = check_dep(galsim.COSMOSCatalog,
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclude_bad=False) # i.e. leave exclude_fail=True
assert cat2.nobjects >= cat.nobjects
# Equivalent to current exclusion_level='bad_fits'
cat3 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='bad_fits')
assert cat2.nobjects == cat3.nobjects
# Both=True
cat2 = check_dep(galsim.COSMOSCatalog,
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclude_fail=True,
exclude_bad=True)
assert cat2.nobjects >= cat.nobjects
# Equivalent to current exclusion_level='marginal'
cat3 = galsim.COSMOSCatalog(
file_name='real_galaxy_catalog_23.5_example.fits',
dir=datapath,
exclusion_level='marginal')
assert cat2.nobjects == cat3.nobjects
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles. We choose 40% of the galaxies
to use the images, and the other 60% to use the parametric fits
- The real galaxy images include some initial correlated noise from the original HST
observation. However, we whiten the noise of the final image so the final image has
stationary Gaussian noise, rather than correlated noise.
"""
logging.basicConfig(format="%(message)s",
level=logging.INFO,
stream=sys.stdout)
logger = logging.getLogger("mock_superbit_data")
# Define some parameters we'll use below.
# Normally these would be read in from some parameter file.
global pixel_scale
pixel_scale = 0.206 # arcsec/pixel
global image_xsize
image_xsize = 6665 # size of image in pixels
global image_ysize
image_ysize = 4453 # size of image in pixels
global image_xsize_arcsec
image_xsize_arcsec = image_xsize * pixel_scale # size of big image in each dimension (arcsec)
global image_ysize_arcsec
image_ysize_arcsec = image_ysize * pixel_scale # size of big image in each dimension (arcsec)
global center_ra
center_ra = 19.3 * galsim.hours # The RA, Dec of the center of the image on the sky
global center_dec
center_dec = -33.1 * galsim.degrees
global exp_time
exp_time = 3000 # exposing for 1500 seconds to match real, observed galaxy/flux count.
global noise_variance
noise_variance = 1.8e3 # ADU^2 (Just use simple Gaussian noise here.) -->150s
#noise_variance = 2.55e3 # ADU^2 (Just use simple Gaussian noise here.) -->300s
global sky_level
sky_level = 51 # ADU / arcsec^2 -->150s
#sky_level = 106 # ADU / arcsec^2 -->300s
global nobj
nobj = 1700 # number of galaxies in entire field -- an adjustment to ensure ~1100 detections
global nstars
nstars = 370 # number of stars in the entire field
global flux_scaling # Let's figure out the flux for a 0.5 m class telescope
global tel_diam
tel_diam = 0.5
global lam
lam = 587 # Central wavelength
psf_path = '/Users/jemcclea/Research/GalSim/examples/data/fpsc_flight_jitter_psf_oversampled_fixed_10x'
global optics # will store the Zernicke component of the PSF
global nfw # will store the NFWHalo information
global cosmos_cat # will store the COSMOS catalog from which we draw objects
# Set up the NFWHalo:
mass = 5E14 # Cluster mass (Msol/h)
nfw_conc = 4 # Concentration parameter = virial radius / NFW scale radius
nfw_z_halo = 0.3 # redshift of the halo
nfw_z_source = 0.6 # redshift of the lensed sources
omega_m = 0.3 # Omega matter for the background cosmology.
omega_lam = 0.7 # Omega lambda for the background cosmology.
field_g1 = 0.03 # The field shear is some cosmic shear applied to the whole field,
field_g2 = 0.01 # taken to be behind the foreground NFW halo (not needed for now)
nfw = galsim.NFWHalo(mass=mass,
conc=nfw_conc,
redshift=nfw_z_halo,
omega_m=omega_m,
omega_lam=omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog
if True:
# The catalog we distribute with the GalSim code only has 100 galaxies.
# The galaxies will typically be reused several times here.
cat_file_name = 'real_galaxy_catalog_23.5_example.fits'
dir = 'data'
cosmos_cat = galsim.COSMOSCatalog(cat_file_name, dir=dir)
else:
# If you've run galsim_download_cosmos, you can leave out the cat_file_name and dir
# to use the full COSMOS catalog with 56,000 galaxies in it.
cosmos_cat = galsim.COSMOSCatalog()
logger.info('Read in %d galaxies from catalog', cosmos_cat.nobjects)
# The catalog returns objects that are appropriate for HST in 1 second exposures. So for our
# telescope we scale up by the relative area and exposure time.
hst_eff_area = 2.4**2 * (1. - 0.33**2)
sbit_eff_area = tel_diam**2 * (
1. - 0.10**2
) # For want of something better, operating with 10% obscuration
flux_scaling = (sbit_eff_area / hst_eff_area) * exp_time
### Now create PSF. First, define Zernicke polynomial component
lam_over_diam = lam * 1.e-9 / tel_diam # radians
lam_over_diam *= 206265 # arcsec
aberrations = [0.0] * 12 # Set the initial size.
aberrations[4] = -0.00725859 # Noll index 4 = Defocus
aberrations[5:7] = [0.0, -0.00] # Noll index 5,6 = Astigmatism
aberrations[7:9] = [0.07, 0.00] # Noll index 7,8 = Coma
aberrations[11] = 0.00133254 # Noll index 11 = Spherical
logger.info('Calculated lambda over diam = %f arcsec', lam_over_diam)
optics = galsim.OpticalPSF(lam_over_diam,
obscuration=0.10,
aberrations=aberrations)
logger.info('Made telescope PSF profile')
###
### LOOP OVER PSFs TO MAKE GROUPS OF IMAGES
### WITHIN EACH PSF, ITERATE 5 TIMES TO MAKE 5 SEPARATE IMAGES
###
all_psfs = glob.glob(psf_path + "/*.psf")
logger.info('Beginning loop over jitter/optical psfs')
for psf_filen in all_psfs:
logger.info('Beginning PSF %s...' % psf_filen)
for i in numpy.arange(1, 6):
logger.info('Beginning loop %d' % i)
random_seed = scipy.random.randint(low=10000000, high=99999999)
rng = galsim.BaseDeviate(random_seed)
# This is specific to Javier mock PSFs
try:
root = psf_filen.split('data/')[1].split('/')[0]
timescale = psf_filen.split('_10x/')[1].split('.')[0]
outname = ''.join([
'mock_superbit_', root, timescale,
str(i).zfill(3), '.fits'
])
truth_file_name = ''.join([
'./output/truth_', root, timescale,
str(i).zfill(3), '.dat'
])
file_name = os.path.join('output', outname)
except:
pdb.set_trace()
# Setting up a truth catalog
names = [
'gal_num', 'x_image', 'y_image', 'ra', 'dec', 'g1_meas',
'g2_meas', 'nfw_g1', 'nfw_g2', 'nfw_mu', 'redshift', 'flux'
]
types = [
int, float, float, float, float, float, float, float, float,
float, float, float
]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(image_xsize, image_ysize)
full_image.fill(sky_level)
full_image.setOrigin(0, 0)
# We keep track of how much noise is already in the image from the RealGalaxies.
noise_image = galsim.ImageF(image_xsize, image_ysize)
noise_image.setOrigin(0, 0)
# Make a slightly non-trivial WCS. We'll use a slightly rotated coordinate system
# and center it at the image center.
theta = 0.17 * galsim.degrees
dudx = numpy.cos(theta) * pixel_scale
dudy = -numpy.sin(theta) * pixel_scale
dvdx = numpy.sin(theta) * pixel_scale
dvdy = numpy.cos(theta) * pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx,
dudy,
dvdx,
dvdy,
origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=center_ra, dec=center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
# Now let's read in the PSFEx PSF model. We read the image directly into an
# InterpolatedImage GSObject, so we can manipulate it as needed
psf_wcs = wcs
psf_file = os.path.join(psf_path, psf_filen)
psf = galsim.des.DES_PSFEx(psf_file, wcs=psf_wcs)
logger.info('Constructed PSF object from PSFEx file')
# Loop over galaxy objects:
for k in range(nobj):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(random_seed + k + 1)
try:
# make single galaxy object
stamp, truth = make_a_galaxy(ud=ud,
wcs=wcs,
psf=psf,
affine=affine)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Galaxy %d positioned relative to center t=%f s', k,
tot_time)
#g1_real=stamp.FindAdaptiveMom().observed_shape.g1
#g2_real=stamp.FindAdaptiveMom().observed_shape.g2
g1_real = -9999.
g2_real = -9999.
this_flux = numpy.sum(stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, g1_real,
g2_real, truth.g1, truth.g2, truth.mu, truth.z,
this_flux
]
truth_catalog.addRow(row)
except:
logger.info('Galaxy %d has failed, skipping...', k)
####
### Now repeat process for stars!
####
random_seed_stars = scipy.random.randint(low=10000000,
high=99999999)
for k in range(nstars):
time1 = time.time()
ud = galsim.UniformDeviate(random_seed_stars + k + 1)
star_stamp, truth = make_a_star(ud=ud,
wcs=wcs,
psf=psf,
affine=affine)
bounds = star_stamp.bounds & full_image.bounds
# Add the stamp to the full image.
try:
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Star %d: positioned relative to center, t=%f s', k,
tot_time)
#g1_real=star_stamp.FindAdaptiveMom().observed_shape.g1 --> no longer positive definite :-?
#g2_real=star_stamp.FindAdaptiveMom().observed_shape.g2
g1_real = -9999.
g2_real = -9999.
this_flux = numpy.sum(star_stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, g1_real,
g2_real, truth.g1, truth.g2, truth.mu, truth.z,
this_flux
]
truth_catalog.addRow(row)
except:
logger.info('Star %d has failed, skipping...', k)
# We already have some noise in the image, but it isn't uniform. So the first thing to do is
# to make the Gaussian noise uniform across the whole image.
max_current_variance = numpy.max(noise_image.array)
noise_image = max_current_variance - noise_image
vn = galsim.VariableGaussianNoise(rng, noise_image)
full_image.addNoise(vn)
# Now max_current_variance is the noise level across the full image. We don't want to add that
# twice, so subtract off this much from the intended noise that we want to end up in the image.
noise_variance -= max_current_variance
# Now add Gaussian noise with this variance to the final image.
noise = galsim.GaussianNoise(rng, sigma=math.sqrt(noise_variance))
full_image.addNoise(noise)
logger.info('Added noise to final large image')
# Now write the image to disk. It is automatically compressed with Rice compression,
# since the filename we provide ends in .fz.
full_image.write(file_name)
logger.info('Wrote image to %r', file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
# Compute some sky positions of some of the pixels to compare with the values of RA, Dec
# that ds9 reports. ds9 always uses (1,1) for the lower left pixel, so the pixel coordinates
# of these pixels are different by 1, but you can check that the RA and Dec values are
# the same as what GalSim calculates.
ra_str = center_ra.hms()
dec_str = center_dec.dms()
logger.info(
'Center of image is at RA %sh %sm %ss, DEC %sd %sm %ss',
ra_str[0:3], ra_str[3:5], ra_str[5:], dec_str[0:3],
dec_str[3:5], dec_str[5:])
for (x, y) in [(0, 0), (0, image_xsize - 1), (image_ysize - 1, 0),
(image_xsize - 1, image_ysize - 1)]:
world_pos = wcs.toWorld(galsim.PositionD(x, y))
ra_str = world_pos.ra.hms()
dec_str = world_pos.dec.dms()
logger.info(
'Pixel (%4d, %4d) is at RA %sh %sm %ss, DEC %sd %sm %ss',
x, y, ra_str[0:3], ra_str[3:5], ra_str[5:], dec_str[0:3],
dec_str[3:5], dec_str[5:])
logger.info(
'ds9 reports these pixels as (1,1), (1,2048), etc. with the same RA, Dec.'
)
i = i + 1
logger.info(' ')
logger.info('completed run %d for psf %s', i, psf_filen)
logger.info('completed all images')
def main(argv):
"""
Make images using model PSFs and galaxy cluster shear:
- The galaxies come from COSMOSCatalog, which can produce either RealGalaxy profiles
(like in demo10) and parametric fits to those profiles. We chose parametric fits since
these are required for chromatic galaxies (ones with filter response included)
- The real galaxy images include some initial correlated noise from the original HST
observation, which would need to be whitened. But we are using parametric galaxies,
so this isn't a concern.
"""
global logger
logging.basicConfig(format="%(message)s",
level=logging.INFO,
stream=sys.stdout)
logger = logging.getLogger("mock_superbit_data")
M = MPIHelper()
# Define some parameters we'll use below.
sbparams = SuperBITParameters(argv=argv)
# Set up the NFWHalo:
nfw = galsim.NFWHalo(mass=sbparams.mass,
conc=sbparams.nfw_conc,
redshift=sbparams.nfw_z_halo,
omega_m=sbparams.omega_m,
omega_lam=sbparams.omega_lam)
logger.info('Set up NFW halo for lensing')
# Read in galaxy catalog, as well as catalog containing
# information from COSMOS fits like redshifts, hlr, etc.
cosmos_cat = galsim.COSMOSCatalog(sbparams.cat_file_name,
dir=sbparams.cosmosdir)
fitcat = Table.read(
os.path.join(os.path.join(sbparams.cosmosdir, sbparams.fit_file_name)))
logger.info('Read in %d galaxies from catalog and associated fit info',
cosmos_cat.nobjects)
cluster_cat = galsim.COSMOSCatalog(sbparams.cluster_cat_name)
print('Read in %d cluster galaxies from catalog' % cosmos_cat.nobjects)
### Now create PSF. First, define Zernicke polynomial component
### note: aberrations were definined for lam = 550, and close to the
### center of the camera. The PSF degrades at the edge of the FOV
lam_over_diam = sbparams.lam * 1.e-9 / sbparams.tel_diam # radians
lam_over_diam *= 206265.
aberrations = numpy.zeros(38) # Set the initial size.
aberrations[0] = 0. # First entry must be zero
aberrations[1] = -0.00305127
aberrations[4] = -0.02474205 # Noll index 4 = Defocus
aberrations[11] = -0.01544329 # Noll index 11 = Spherical
aberrations[22] = 0.00199235
aberrations[26] = 0.00000017
aberrations[37] = 0.00000004
logger.info('Calculated lambda over diam = %f arcsec', lam_over_diam)
# will store the Zernicke component of the PSF
optics = galsim.OpticalPSF(lam=sbparams.lam,
diam=sbparams.tel_diam,
obscuration=sbparams.obscuration,
nstruts=sbparams.nstruts,
strut_angle=sbparams.strut_angle,
strut_thick=sbparams.strut_thick,
aberrations=aberrations)
logger.info('Made telescope PSF profile')
# load SuperBIT bandpass
bandpass = galsim.Bandpass(sbparams.bp_file,
wave_type='nm',
blue_limit=310,
red_limit=1100)
###
### LOOP OVER PSFs TO MAKE GROUPS OF IMAGES
### WITHIN EACH PSF, ITERATE n TIMES TO MAKE n SEPARATE IMAGES
###
#all_psfs=glob.glob(sbparams.psf_path+"/*121*.psf")
logger.info('Beginning loop over jitter/optical psfs')
for im in np.arange(1):
for i in numpy.arange(1, sbparams.nexp + 1):
# get MPI processes in sync at start of each image
M.barrier()
logger.info('Beginning loop %d' % i)
#rng = galsim.BaseDeviate(sbparams.noise_seed+i)
try:
timescale = str(sbparams.exp_time)
outname = ''.join(
['superbit_gaussPSF_',
str(i).zfill(3), '.fits'])
truth_file_name = ''.join([
sbparams.outdir, '/truth_gaussPSF_',
str(i).zfill(3), '.dat'
])
file_name = os.path.join(sbparams.outdir, outname)
except galsim.errors.GalSimError:
print("naming failed, check path")
pdb.set_trace()
# Setting up a truth catalog
names = [
'gal_num', 'x_image', 'y_image', 'ra', 'dec', 'g1_meas',
'g2_meas', 'nfw_mu', 'redshift', 'flux', 'truth_fwhm',
'truth_mom'
]
types = [
int, float, float, float, float, float, float, float, float,
float, float, float
]
truth_catalog = galsim.OutputCatalog(names, types)
# Set up the image:
full_image = galsim.ImageF(sbparams.image_xsize,
sbparams.image_ysize)
sky_level = sbparams.exp_time * sbparams.sky_bkg
# fill with sky_level moved until after MPI results summed
full_image.fill(sky_level)
full_image.setOrigin(0, 0)
# We keep track of how much noise is already in the image from the RealGalaxies.
noise_image = galsim.ImageF(sbparams.image_xsize,
sbparams.image_ysize)
noise_image.setOrigin(0, 0)
# If you wanted to make a non-trivial WCS system, could set theta to a non-zero number
theta = 0.0 * galsim.degrees
dudx = numpy.cos(theta) * sbparams.pixel_scale
dudy = -numpy.sin(theta) * sbparams.pixel_scale
dvdx = numpy.sin(theta) * sbparams.pixel_scale
dvdy = numpy.cos(theta) * sbparams.pixel_scale
image_center = full_image.true_center
affine = galsim.AffineTransform(dudx,
dudy,
dvdx,
dvdy,
origin=full_image.true_center)
sky_center = galsim.CelestialCoord(ra=sbparams.center_ra,
dec=sbparams.center_dec)
wcs = galsim.TanWCS(affine, sky_center, units=galsim.arcsec)
full_image.wcs = wcs
# Now let's read in the PSFEx PSF model. We read the image directly into an
# InterpolatedImage GSObject, so we can manipulate it as needed
psf_wcs = wcs
#psf = galsim.des.DES_PSFEx(psf_filen,wcs=psf_wcs)
logger.info('Constructed PSF object from PSFEx file')
#####
## Loop over galaxy objects:
#####
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nobj)
for k in range(local_start, local_end):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(sbparams.galobj_seed + k + 1)
try:
# make single galaxy object
stamp, truth = make_a_galaxy(ud=ud,
wcs=wcs,
affine=affine,
fitcat=fitcat,
cosmos_cat=cosmos_cat,
optics=optics,
nfw=nfw,
bandpass=bandpass,
sbparams=sbparams)
# Find the overlapping bounds:
bounds = stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
# noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Galaxy %d positioned relative to center t=%f s', k,
tot_time)
this_flux = numpy.sum(stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux, truth.fwhm,
truth.mom_size
]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Galaxy %d has failed, skipping...', k)
#####
### Inject cluster galaxy objects:
#####
center_coords = galsim.CelestialCoord(sbparams.center_ra,
sbparams.center_dec)
centerpix = wcs.toImage(center_coords)
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nclustergal)
for k in range(local_start, local_end):
time1 = time.time()
# The usual random number generator using a different seed for each galaxy.
ud = galsim.UniformDeviate(sbparams.cluster_seed + k + 1)
try:
# make single galaxy object
cluster_stamp, truth = make_cluster_galaxy(
ud=ud,
wcs=wcs,
affine=affine,
centerpix=centerpix,
cluster_cat=cluster_cat,
optics=optics,
bandpass=bandpass,
sbparams=sbparams)
# Find the overlapping bounds:
bounds = cluster_stamp.bounds & full_image.bounds
# We need to keep track of how much variance we have currently in the image, so when
# we add more noise, we can omit what is already there.
#noise_image[bounds] += truth.variance
# Finally, add the stamp to the full image.
full_image[bounds] += cluster_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Cluster galaxy %d positioned relative to center t=%f s',
k, tot_time)
this_flux = numpy.sum(stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux, truth.fwhm,
truth.mom_size
]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Cluster galaxy %d has failed, skipping...', k)
#####
### Now repeat process for stars!
#####
# get local range to iterate over in this process
local_start, local_end = M.mpi_local_range(sbparams.nstars)
for k in range(local_start, local_end):
time1 = time.time()
ud = galsim.UniformDeviate(sbparams.stars_seed + k + 1)
star_stamp, truth = make_a_star(ud=ud,
wcs=wcs,
affine=affine,
optics=optics,
sbparams=sbparams)
bounds = star_stamp.bounds & full_image.bounds
# Add the stamp to the full image.
try:
full_image[bounds] += star_stamp[bounds]
time2 = time.time()
tot_time = time2 - time1
logger.info(
'Star %d: positioned relative to center, t=%f s', k,
tot_time)
this_flux = numpy.sum(star_stamp.array)
row = [
k, truth.x, truth.y, truth.ra, truth.dec, truth.g1,
truth.g2, truth.mu, truth.z, this_flux, truth.fwhm,
truth.mom_size
]
truth_catalog.addRow(row)
except galsim.errors.GalSimError:
logger.info('Star %d has failed, skipping...', k)
# Gather results from MPI processes, reduce to single result on root
# Using same names on left and right sides is hiding lots of MPI magic
full_image = M.gather(full_image)
truth_catalog = M.gather(truth_catalog)
#noise_image = M.gather(noise_image)
if M.is_mpi_root():
full_image = reduce(combine_images, full_image)
truth_catalog = reduce(combine_catalogs, truth_catalog)
#noise_image = reduce(combine_images, noise_image)
else:
# do the adding of noise and writing to disk entirely on root
# root and the rest meet again at barrier at start of loop
continue
# The first thing to do is to make the Gaussian noise uniform across the whole image.
# If real-type COSMOS galaxies are used, the noise across the image won't be uniform. Since this code is
# using parametric-type galaxies, the following section is commented out.
# max_current_variance = numpy.max(noise_image.array)
# noise_image = max_current_variance - noise_image
# The first thing to do is to make the Gaussian noise uniform across the whole image.
# Add dark current
logger.info('Adding Dark current')
dark_noise = sbparams.dark_current * sbparams.exp_time
# np.random.normal(
# sbparams.dark_current, sbparams.dark_current_std,
# size=(sbparams.image_ysize, sbparams.image_xsize)) * sbparams.exp_time
# dark_noise = np.clip(dark_noise, a_min=0, a_max=2**16)
full_image += dark_noise
# Add ccd noise; removed rng in noise
logger.info('Adding CCD noise')
noise = galsim.CCDNoise(sky_level=0,
gain=1 / sbparams.gain,
read_noise=sbparams.read_noise)
full_image.addNoise(noise)
logger.debug('Added noise to final output image')
if not os.path.exists(os.path.dirname(file_name)):
os.makedirs(os.path.dirname(file_name))
full_image.write(file_name)
# Write truth catalog to file.
truth_catalog.write(truth_file_name)
logger.info('Wrote image to %r', file_name)
logger.info(' ')
logger.info('completed run %d', im)
i = i + 1
logger.info(' ')
logger.info(' ')
logger.info('completed all images')
logger.info(' ')
