def __init__(self, file_name=None, image_dir=None, dir=None, preload=False, noise_dir=None,
use_real=True, exclude_fail=True, exclude_bad=True, max_hlr=0.,
_nobjects_only=False):
from galsim._pyfits import pyfits
self.use_real = use_real
if self.use_real:
if not _nobjects_only:
# First, do the easy thing: real galaxies. We make the galsim.RealGalaxyCatalog()
# constructor do most of the work. But note that we don't actually need to
# bother with this if all we care about is the nobjects attribute.
self.real_cat = galsim.RealGalaxyCatalog(
file_name, image_dir=image_dir, dir=dir, preload=preload, noise_dir=noise_dir)
# The fits name has _fits inserted before the .fits ending.
# Note: don't just use k = -5 in case it actually ends with .fits.fz
k = self.real_cat.file_name.find('.fits')
param_file_name = self.real_cat.file_name[:k] + '_fits' + self.real_cat.file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
else:
# Start by doing the same file_name parsing as we did for the real galaxy
param_file_name, _, _ = galsim.real._parse_files_dirs(
file_name, image_dir, dir, noise_dir)
try:
# Read in data.
self.param_cat = pyfits.getdata(param_file_name)
self.param_cat['fit_status']
except KeyError:
# But if that doesn't work, then the name might be the name of the real catalog,
# so try adding _fits to it as above.
k = param_file_name.find('.fits')
param_file_name = param_file_name[:k] + '_fits' + param_file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
# If requested, select galaxies based on existence of a usable fit.
self.orig_index = np.arange(len(self.param_cat))
if exclude_fail or exclude_bad or max_hlr > 0.:
mask = True
if exclude_fail:
sersicfit_status = self.param_cat['fit_status'][:,4]
bulgefit_status = self.param_cat['fit_status'][:,0]
mask &= ( (sersicfit_status > 0) &
(sersicfit_status < 5) &
(bulgefit_status > 0) &
(bulgefit_status < 5) )
if exclude_bad:
hlr = self.param_cat['sersicfit'][:,1]
n = self.param_cat['sersicfit'][:,2]
mask &= ( (n < 5) | (hlr < 1./cosmos_pix_scale) )
# May add more cuts here if we discover other kinds of problematic objects.
if max_hlr > 0.:
hlr = self.param_cat['sersicfit'][:,1]
mask &= (hlr < max_hlr / cosmos_pix_scale)
self.orig_index = self.orig_index[mask]
self.nobjects = len(self.orig_index)
def read_fits(self):
"""Read in a DES_Shapelet stored using the the FITS-file version.
"""
from galsim._pyfits import pyfits
cat = pyfits.getdata(self.file_name,1)
# These fields each only contain one element, hence the [0]'s.
self.psf_order = cat.field('psf_order')[0]
self.psf_size = (self.psf_order+1) * (self.psf_order+2) / 2
self.sigma = cat.field('sigma')[0]
self.fit_order = cat.field('fit_order')[0]
self.fit_size = (self.fit_order+1) * (self.fit_order+2) / 2
self.npca = cat.field('npca')[0]
self.bounds = galsim.BoundsD(
float(cat.field('xmin')[0]), float(cat.field('xmax')[0]),
float(cat.field('ymin')[0]), float(cat.field('ymax')[0]))
self.ave_psf = cat.field('ave_psf')[0]
assert self.ave_psf.shape == (self.psf_size,)
# Note: older pyfits versions don't get the shape right.
# For newer pyfits versions the reshape command should be a no op.
self.rot_matrix = cat.field('rot_matrix')[0].reshape((self.psf_size,self.npca)).T
assert self.rot_matrix.shape == (self.npca, self.psf_size)
self.interp_matrix = cat.field('interp_matrix')[0].reshape((self.npca,self.fit_size)).T
assert self.interp_matrix.shape == (self.fit_size, self.npca)
def read_fits(self):
"""Read in a DES_Shapelet stored using the the FITS-file version.
"""
from galsim._pyfits import pyfits
cat = pyfits.getdata(self.file_name, 1)
# These fields each only contain one element, hence the [0]'s.
self.psf_order = cat.field('psf_order')[0]
self.psf_size = (self.psf_order + 1) * (self.psf_order + 2) / 2
self.sigma = cat.field('sigma')[0]
self.fit_order = cat.field('fit_order')[0]
self.fit_size = (self.fit_order + 1) * (self.fit_order + 2) / 2
self.npca = cat.field('npca')[0]
self.bounds = galsim.BoundsD(float(cat.field('xmin')[0]),
float(cat.field('xmax')[0]),
float(cat.field('ymin')[0]),
float(cat.field('ymax')[0]))
self.ave_psf = cat.field('ave_psf')[0]
assert self.ave_psf.shape == (self.psf_size, )
# Note: older pyfits versions don't get the shape right.
# For newer pyfits versions the reshape command should be a no op.
self.rot_matrix = cat.field('rot_matrix')[0].reshape(
(self.psf_size, self.npca)).T
assert self.rot_matrix.shape == (self.npca, self.psf_size)
self.interp_matrix = cat.field('interp_matrix')[0].reshape(
(self.npca, self.fit_size)).T
assert self.interp_matrix.shape == (self.fit_size, self.npca)
def __init__(self, file_name=None, image_dir=None, dir=None, preload=False,
noise_dir=None, logger=None, _nobjects_only=False):
from galsim._pyfits import pyfits
self.file_name, self.image_dir, self.noise_dir = \
_parse_files_dirs(file_name, image_dir, dir, noise_dir)
self.cat = pyfits.getdata(self.file_name)
self.nobjects = len(self.cat) # number of objects in the catalog
if _nobjects_only: return # Exit early if that's all we needed.
ident = self.cat.field('ident') # ID for object in the training sample
# We want to make sure that the ident array contains all strings.
# Strangely, ident.astype(str) produces a string with each element == '1'.
# Hence this way of doing the conversion:
self.ident = [ "%s"%val for val in ident ]
self.gal_file_name = self.cat.field('gal_filename') # file containing the galaxy image
self.psf_file_name = self.cat.field('PSF_filename') # file containing the PSF image
# Add the directories:
self.gal_file_name = [ os.path.join(self.image_dir,f) for f in self.gal_file_name ]
self.psf_file_name = [ os.path.join(self.image_dir,f) for f in self.psf_file_name ]
# We don't require the noise_filename column. If it is not present, we will use
# Uncorrelated noise based on the variance column.
try:
self.noise_file_name = self.cat.field('noise_filename') # file containing the noise cf
self.noise_file_name = [ os.path.join(self.noise_dir,f) for f in self.noise_file_name ]
except:
self.noise_file_name = None
self.gal_hdu = self.cat.field('gal_hdu') # HDU containing the galaxy image
self.psf_hdu = self.cat.field('PSF_hdu') # HDU containing the PSF image
self.pixel_scale = self.cat.field('pixel_scale') # pixel scale for image (could be different
# if we have training data from other datasets... let's be general here and make it a
# vector in case of mixed training set)
self.variance = self.cat.field('noise_variance') # noise variance for image
self.mag = self.cat.field('mag') # apparent magnitude
self.band = self.cat.field('band') # bandpass in which apparent mag is measured, e.g., F814W
self.weight = self.cat.field('weight') # weight factor to account for size-dependent
# probability
self.saved_noise_im = {}
self.loaded_files = {}
self.logger = logger
# The pyfits commands aren't thread safe. So we need to make sure the methods that
# use pyfits are not run concurrently from multiple threads.
from multiprocessing import Lock
self.gal_lock = Lock() # Use this when accessing gal files
self.psf_lock = Lock() # Use this when accessing psf files
self.loaded_lock = Lock() # Use this when opening new files from disk
self.noise_lock = Lock() # Use this for building the noise image(s) (usually just one)
# Preload all files if desired
if preload: self.preload()
self._preload = preload
def read_fits(self, hdu, _nobjects_only=False):
"""Read in an input catalog from a FITS file.
"""
from galsim._pyfits import pyfits, pyfits_version
raw_data = pyfits.getdata(self.file_name, hdu)
if pyfits_version > '3.0':
self.names = raw_data.columns.names
else:
self.names = raw_data.dtype.names
self.nobjects = len(raw_data.field(self.names[0]))
if (_nobjects_only): return
# The pyfits raw_data is a FITS_rec object, which isn't picklable, so we need to
# copy the fields into a new structure to make sure our Catalog is picklable.
# The simplest is probably a dict keyed by the field names, which we save as self.data.
self.data = {}
for name in self.names:
self.data[name] = raw_data.field(name)
self.ncols = len(self.names)
self.isfits = True
def readFits(self, hdu, _nobjects_only=False):
"""Read in an input catalog from a FITS file.
"""
from galsim._pyfits import pyfits, pyfits_version
raw_data = pyfits.getdata(self.file_name, hdu)
if pyfits_version > '3.0':
self.names = raw_data.columns.names
else:
self.names = raw_data.dtype.names
self.nobjects = len(raw_data.field(self.names[0]))
if (_nobjects_only): return
# The pyfits raw_data is a FITS_rec object, which isn't picklable, so we need to
# copy the fields into a new structure to make sure our Catalog is picklable.
# The simplest is probably a dict keyed by the field names, which we save as self.data.
self.data = {}
for name in self.names:
self.data[name] = raw_data.field(name)
self.ncols = len(self.names)
self.isfits = True
def getNoiseProperties(self, i):
"""Returns the components needed to make the noise correlation function at index `i`.
Specifically, the noise image (or None), the pixel_scale, and the noise variance,
as a tuple (im, scale, var).
"""
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Start getNoise', i)
if self.noise_file_name is None:
im = None
else:
if i >= len(self.noise_file_name):
raise IndexError(
'index %d given to getNoise is out of range (0..%d)' %
(i, len(self.noise_file_name) - 1))
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug(
'RealGalaxyCatalog %d: Got saved noise im', i)
else:
self.noise_lock.acquire()
# Again, a second check in case two processes get here at the same time.
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug(
'RealGalaxyCatalog %d: Got saved noise im', i)
else:
import numpy
from galsim._pyfits import pyfits
array = pyfits.getdata(self.noise_file_name[i])
im = galsim.Image(numpy.ascontiguousarray(
array.astype(numpy.float64)),
scale=self.pixel_scale[i])
self.saved_noise_im[self.noise_file_name[i]] = im
if self.logger:
self.logger.debug(
'RealGalaxyCatalog %d: Built noise im', i)
self.noise_lock.release()
return im, self.pixel_scale[i], self.variance[i]
def getNoiseProperties(self, i):
"""Returns the components needed to make the noise correlation function at index `i`.
Specifically, the noise image (or None), the pixel_scale, and the noise variance,
as a tuple (im, scale, var).
"""
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Start getNoise',i)
if self.noise_file_name is None:
im = None
else:
if i >= len(self.noise_file_name):
raise IndexError(
'index %d given to getNoise is out of range (0..%d)'%(
i,len(self.noise_file_name)-1))
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
self.noise_lock.acquire()
# Again, a second check in case two processes get here at the same time.
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
import numpy
from galsim._pyfits import pyfits
array = pyfits.getdata(self.noise_file_name[i])
im = galsim.Image(numpy.ascontiguousarray(array.astype(numpy.float64)),
scale=self.pixel_scale[i])
self.saved_noise_im[self.noise_file_name[i]] = im
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Built noise im',i)
self.noise_lock.release()
return im, self.pixel_scale[i], self.variance[i]
def test_wfirst_bandpass():
"""Test the WFIRST bandpasses for basic sanity.
"""
import time
t1 = time.time()
from galsim._pyfits import pyfits
# Obtain the bandpasses with AB_zeropoint set
exp_time = 200. # non WFIRST exposure time
bp = galsim.wfirst.getBandpasses(AB_zeropoint=True, exptime=exp_time)
# Check if the zeropoints have been set correctly
AB_spec = lambda x: (3631e-23)*exp_time*(np.pi)*(100.**2)*\
(galsim.wfirst.diameter**2)*(1-galsim.wfirst.obscuration**2)/4.
AB_sed = galsim.SED(spec=AB_spec,flux_type='fnu')
for filter_name, filter_ in bp.iteritems():
mag = AB_sed.calculateMagnitude(bandpass=filter_)
np.testing.assert_almost_equal(mag,0.0,decimal=6,
err_msg="Zeropoint not set accurately enough for bandpass filter \
{0}".format(filter_name))
# Do a slightly less trivial check of bandpass-related calculations:
# Jeff Kruk (at Goddard) took an SED template from the Castelli-Kurucz library, normalized it to
# a particular magnitude in SDSS g band, and checked the count rates he expects for the WFIRST
# bands. I (RM) independently did the same calculation (downloading the templates and bandpass
# myself and using GalSim for all the important bits of the calculation) and my results agree a
# the 5% level. Given that I didn't quite have the same SED, we were very happy with this level
# of agreement. The unit test below reproduces this test, and requires agreement at the 10%
# level.
# Jeff used the C-K template with solar metallicity, T=9550K, surface gravity logg=3.95. I
# downloaded a grid of templates and just used the nearest one, which has solar metallicity,
# T=9500K, surface gravity logg=4.0.
sed_data = pyfits.getdata(os.path.join('wfirst_files','ckp00_9500.fits'))
lam = sed_data.WAVELENGTH.astype(np.float64)
t = sed_data.g40.astype(np.float64)
sed_tab = galsim.LookupTable(x=lam, f=t, interpolant='linear')
sed = galsim.SED(sed_tab, wave_type='A')
# Now take the SDSS g bandpass:
gfile = '/Users/rmandelb/Downloads/g.dat'
bp_dat = np.loadtxt(os.path.join('wfirst_files','g.dat')).transpose()
bp_tab = galsim.LookupTable(x=bp_dat[0,:], f=bp_dat[1,:], interpolant='linear')
bp_ref = galsim.Bandpass(bp_tab, wave_type='A')
# Set an AB zeropoint using WFIRST params:
eff_diam = 100.*galsim.wfirst.diameter*np.sqrt(1.-galsim.wfirst.obscuration**2)
bp_ref = bp_ref.withZeropoint('AB', effective_diameter=eff_diam, exptime=galsim.wfirst.exptime)
# Now get a new SED that has magnitude -0.093 in this filter, since that's the normalization
# that Jeff imposed for his tests.
sed = sed.withMagnitude(-0.093, bp_ref)
# Reference count rates, from Jeff:
reference = {}
reference['Z087'] = 1.98e10
reference['Y106'] = 1.97e10
reference['J129'] = 1.52e10
reference['H158'] = 1.11e10
reference['F184'] = 0.58e10
reference['W149'] = 4.34e10
# Only 10% accuracy required because did not use quite the same stellar template. Fortunately,
# bugs can easily lead to orders of magnitude errors, so this unit test is still pretty
# non-trivial.
for filter_name, filter_ in bp.iteritems():
flux = sed.calculateFlux(filter_)
count_rate = flux / galsim.wfirst.exptime
np.testing.assert_allclose(
count_rate, reference[filter_name], rtol=0.1,
err_msg="Count rate for stellar model not as expected for bandpass "
"{0}".format(filter_name))
t2 = time.time()
print 'time for %s = %.2f'%(funcname(),t2-t1)
def __init__(self, file_name=None, sample=None, image_dir=None, dir=None, preload=False,
noise_dir=None, use_real=True, exclusion_level='marginal', min_hlr=0, max_hlr=0.,
min_flux=0., max_flux=0., _nobjects_only=False, exclude_bad=None,
exclude_fail=None):
if sample is not None and file_name is not None:
raise ValueError("Cannot specify both the sample and file_name!")
# Check for deprecated exclude_bad or exclude_fail args.
if exclude_bad is not None or exclude_fail is not None:
from .deprecated import depr
if exclude_bad is not None:
depr('exclude_bad', 1.4, 'exclusion_level')
if exclude_fail is not None:
depr('exclude_fail', 1.4, 'exclusion_level')
# These aren't equivalent, but probably what the user would want to choose.
if exclude_bad is False and exclude_fail is False:
exclusion_level = 'none'
elif exclude_fail is False: # implying exclude_bad=True is intended
exclusion_level = 'bad_stamp'
elif exclude_bad is False: # implying exclude_fail=True is intended
exclusion_level = 'bad_fits'
# else leave exclusion_level as 'marginal'
from galsim._pyfits import pyfits
self.use_real = use_real
if exclusion_level not in ['none', 'bad_stamp', 'bad_fits', 'marginal']:
raise ValueError("Invalid value of exclusion_level: %s"%exclusion_level)
# Start by parsing the file names, since we'll need the image_dir below.
full_file_name, full_image_dir, _, self.use_sample = \
galsim.real._parse_files_dirs(file_name, image_dir, dir, noise_dir, sample)
if self.use_real and not _nobjects_only:
# First, do the easy thing: real galaxies. We make the galsim.RealGalaxyCatalog()
# constructor do most of the work. But note that we don't actually need to
# bother with this if all we care about is the nobjects attribute.
self.real_cat = galsim.RealGalaxyCatalog(
file_name, sample=sample, image_dir=image_dir, dir=dir, preload=preload,
noise_dir=noise_dir)
# The fits name has _fits inserted before the .fits ending.
# Note: don't just use k = -5 in case it actually ends with .fits.fz
k = self.real_cat.file_name.find('.fits')
param_file_name = self.real_cat.file_name[:k] + '_fits' + self.real_cat.file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
else:
try:
# Read in data.
self.param_cat = pyfits.getdata(full_file_name)
# Check if this was the right file. It should have a 'fit_status' column.
self.param_cat['fit_status']
except KeyError:
# But if that doesn't work, then the name might be the name of the real catalog,
# so try adding _fits to it as above.
k = full_file_name.find('.fits')
param_file_name = full_file_name[:k] + '_fits' + full_file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
# Check for the old-style parameter catalog
if 'fit_dvc_btt' not in self.param_cat.dtype.names:
# This will fail if they try to make a parametric galaxy.
# Don't raise an exception here, since they might not care about that.
# But give them some guidance about the error they will get if they
# do try to make a parametric galaxy.
import warnings
warnings.warn(
'You seem to have an old version of the COSMOS parameter file. '+
'Please run `galsim_download_cosmos` to re-download the COSMOS catalog.')
# NB. The pyfits FITS_Rec class has a bug where it makes a copy of the full
# record array in each record (e.g. in getParametricRecord) and then doesn't
# garbage collect it until the top-level FITS_Record goes out of scope.
# This leads to a memory leak of order 10MB or so each time we make a parametric
# galaxy.
# cf. https://mail.scipy.org/pipermail/astropy/2014-June/003218.html
# also https://github.com/astropy/astropy/pull/520
# The simplest workaround seems to be to convert it to a regular numpy recarray.
# (This also makes it run much faster, as an extra bonus!)
self.param_cat = np.array(self.param_cat, copy=True)
self.orig_index = np.arange(len(self.param_cat))
mask = np.ones(len(self.orig_index), dtype=bool)
if exclusion_level in ['marginal', 'bad_stamp']:
# First, read in what we need to impose selection criteria, if the appropriate
# exclusion_level was chosen.
k = full_file_name.find('.fits')
try:
# This should work if the user passed in (or we defaulted to) the real galaxy
# catalog name:
selection_file_name = full_file_name[:k] + '_selection' + full_file_name[k:]
try:
self.selection_cat = pyfits.getdata(selection_file_name)
except IOError:
# There's one more option: full_file_name might be the parametric fit file, so
# we have to strip off the _fits.fits (instead of just the .fits)
selection_file_name = full_file_name[:k-5] + '_selection' + full_file_name[k:]
self.selection_cat = pyfits.getdata(selection_file_name)
# At this point we've read in the catalog one way or another (otherwise we would
# have gotten tossed out of this part of the code to throw an IOError). So, we can
# proceed to select galaxies in a way that excludes suspect postage stamps (e.g.,
# with deblending issues), suspect parametric model fits, or both of the above plus
# marginal ones. These two options for 'exclusion_level' involve placing cuts on
# the S/N of the object detection in the original postage stamp, and on issues with
# masking that can indicate deblending or detection failures. These cuts were used
# in GREAT3. In the case of the masking cut, in some cases there are messed up ones
# that have a 0 for self.selection_cat['peak_image_pixel_count']. To make sure we
# don't divide by zero (generating a RuntimeWarning), and still eliminate those, we
# will first set that column to 1.e-5. We choose a sample-dependent mask ratio cut,
# since this depends on the peak object flux, which will differ for the two samples
# (and we can't really cut on this for arbitrary user-defined samples).
if self.use_sample == "23.5":
cut_ratio = 0.2
sn_limit = 20.0
else:
cut_ratio = 0.8
sn_limit = 12.0
div_val = self.selection_cat['peak_image_pixel_count']
div_val[div_val == 0.] = 1.e-5
mask &= ( (self.selection_cat['sn_ellip_gauss'] >= sn_limit) &
((self.selection_cat['min_mask_dist_pixels'] > 11.0) |
(self.selection_cat['average_mask_adjacent_pixel_count'] / \
div_val < cut_ratio)) )
except IOError:
# We can't make any of the above cuts (or any later ones that depend on the
# selection catalog) because we couldn't find the selection catalog. Bummer. Warn
# the user, and move on.
self.selection_cat = None
import warnings
warnings.warn(
'File with GalSim selection criteria not found! '+
'Not all of the requested exclusions will be performed. '+
'Run the program galsim_download_cosmos to get the necessary selection file.')
# Finally, impose a cut that the total flux in the postage stamp should be positive,
# which excludes a tiny number of galaxies (of order 10 in each sample) with some sky
# subtraction or deblending errors. Some of these are eliminated by other cuts when
# using exclusion_level='marginal'.
if hasattr(self,'real_cat'):
if hasattr(self.real_cat, 'stamp_flux'):
mask &= self.real_cat.stamp_flux > 0
else:
import warnings
warnings.warn(
'This version of the COSMOS catalog does not have info about total flux in '+
'the galaxy postage stamps. Exclusion of negative-flux stamps in advance '+
'cannot be done. '+
'Run the program galsim_download_cosmos to get the updated catalog with this '+
'information precomputed.')
if exclusion_level in ['bad_fits', 'marginal']:
# This 'exclusion_level' involves eliminating failed parametric fits (bad fit status
# flags). In this case we only get rid of those with failed bulge+disk AND failed
# Sersic fits, so there is no viable parametric model for the galaxy.
sersicfit_status = self.param_cat['fit_status'][:,4]
bulgefit_status = self.param_cat['fit_status'][:,0]
mask &= ( ((sersicfit_status > 0) &
(sersicfit_status < 5)) |
((bulgefit_status > 0) &
(bulgefit_status < 5)) )
if exclusion_level == 'marginal':
# We have already placed some cuts (above) in this case, but we'll do some more. For
# example, a failed bulge+disk fit often indicates difficulty in fit convergence due to
# noisy surface brightness profiles, so we might want to toss out those that have a
# failure in EITHER fit.
mask &= ( ((sersicfit_status > 0) &
(sersicfit_status < 5)) &
((bulgefit_status > 0) &
(bulgefit_status < 5)) )
# Some fit parameters can indicate a likely sky subtraction error: very high sersic n
# AND abnormally large half-light radius (>1 arcsec).
if 'hlr' not in self.param_cat.dtype.names:
# This is the circularized HLR in arcsec, which we have to compute from the stored
# parametric fits.
hlr = cosmos_pix_scale * self.param_cat['sersicfit'][:,1] * \
np.sqrt(self.param_cat['sersicfit'][:,2])
else:
# This is the pre-computed circularized HLR in arcsec.
hlr = self.param_cat['hlr'][:,0]
n = self.param_cat['sersicfit'][:,2]
mask &= ( (n < 5) | (hlr < 1.) )
# Major flux differences in the parametric model vs. the COSMOS catalog can indicate fit
# issues, deblending problems, etc.
if self.selection_cat is not None:
mask &= ( np.abs(self.selection_cat['dmag']) < 0.8)
if min_hlr > 0. or max_hlr > 0. or min_flux > 0. or max_flux > 0.:
if 'hlr' not in self.param_cat.dtype.names:
# Check if they have a new version of the selection catalog that has precomputed
# fluxes etc. If not, do the calculations, which include some approximations in
# getting the flux.
import warnings
warnings.warn(
'You seem to have an old version of the COSMOS parameter file. '+
'Please run `galsim_download_cosmos` to re-download the COSMOS catalog ' +
'to get faster and more accurate selection.')
sparams = self.param_cat['sersicfit']
hlr_pix = sparams[:,1]
n = sparams[:,2]
q = sparams[:,3]
hlr = cosmos_pix_scale*hlr_pix*np.sqrt(q)
if min_flux > 0. or max_flux > 0.:
flux_hlr = sparams[:,0]
# The prefactor for n=4 is 3.607. For n=1, it is 1.901.
# It's not linear in these values, but for the sake of efficiency and the
# ability to work on the whole array at once, just linearly interpolate.
# This was improved as part of issue #693, for which part of the work involved
# updating the catalogs to include precomputed fluxes and radii. So as shown
# below, if that info is in the catalog we just use it directly instead of using
# this approximate calculation.
#prefactor = ( (n-1.)*3.607 + (4.-n)*1.901 ) / (4.-1.)
prefactor = ((3.607-1.901)/3.) * n + (4.*1.901 - 1.*3.607)/3.
flux = 2.0*np.pi*prefactor*(hlr**2)*flux_hlr/cosmos_pix_scale**2
else:
hlr = self.param_cat['hlr'][:,0] # sersic half-light radius
flux = self.param_cat['flux'][:,0]
if min_hlr > 0.:
mask &= (hlr > min_hlr)
if max_hlr > 0.:
mask &= (hlr < max_hlr)
if min_flux > 0.:
mask &= (flux > min_flux)
if max_flux > 0.:
mask &= (flux < max_flux)
self.orig_index = self.orig_index[mask]
self.nobjects = len(self.orig_index)
def __init__(self, file_name=None, sample=None, image_dir=None, dir=None, preload=False,
noise_dir=None, logger=None, _nobjects_only=False):
if sample is not None and file_name is not None:
raise ValueError("Cannot specify both the sample and file_name!")
from galsim._pyfits import pyfits
self.file_name, self.image_dir, self.noise_dir, _ = \
_parse_files_dirs(file_name, image_dir, dir, noise_dir, sample)
self.cat = pyfits.getdata(self.file_name)
self.nobjects = len(self.cat) # number of objects in the catalog
if _nobjects_only: return # Exit early if that's all we needed.
ident = self.cat.field('ident') # ID for object in the training sample
# We want to make sure that the ident array contains all strings.
# Strangely, ident.astype(str) produces a string with each element == '1'.
# Hence this way of doing the conversion:
self.ident = [ "%s"%val for val in ident ]
self.gal_file_name = self.cat.field('gal_filename') # file containing the galaxy image
self.psf_file_name = self.cat.field('PSF_filename') # file containing the PSF image
# Add the directories:
# Note the strip call. Sometimes the filenames have an extra space at the end.
# This gets rid of that space.
self.gal_file_name = [ os.path.join(self.image_dir,f.strip()) for f in self.gal_file_name ]
self.psf_file_name = [ os.path.join(self.image_dir,f.strip()) for f in self.psf_file_name ]
# We don't require the noise_filename column. If it is not present, we will use
# Uncorrelated noise based on the variance column.
try:
self.noise_file_name = self.cat.field('noise_filename') # file containing the noise cf
self.noise_file_name = [ os.path.join(self.noise_dir,f) for f in self.noise_file_name ]
except:
self.noise_file_name = None
self.gal_hdu = self.cat.field('gal_hdu') # HDU containing the galaxy image
self.psf_hdu = self.cat.field('PSF_hdu') # HDU containing the PSF image
self.pixel_scale = self.cat.field('pixel_scale') # pixel scale for image (could be different
# if we have training data from other datasets... let's be general here and make it a
# vector in case of mixed training set)
self.variance = self.cat.field('noise_variance') # noise variance for image
self.mag = self.cat.field('mag') # apparent magnitude
self.band = self.cat.field('band') # bandpass in which apparent mag is measured, e.g., F814W
self.weight = self.cat.field('weight') # weight factor to account for size-dependent
# probability
self.saved_noise_im = {}
self.loaded_files = {}
self.logger = logger
# The pyfits commands aren't thread safe. So we need to make sure the methods that
# use pyfits are not run concurrently from multiple threads.
from multiprocessing import Lock
self.gal_lock = Lock() # Use this when accessing gal files
self.psf_lock = Lock() # Use this when accessing psf files
self.loaded_lock = Lock() # Use this when opening new files from disk
self.noise_lock = Lock() # Use this for building the noise image(s) (usually just one)
# Preload all files if desired
if preload: self.preload()
self._preload = preload
def __init__(self,
file_name=None,
image_dir=None,
dir=None,
preload=False,
noise_dir=None,
use_real=True,
exclude_fail=True,
exclude_bad=True,
max_hlr=0.,
_nobjects_only=False):
from galsim._pyfits import pyfits
self.use_real = use_real
if self.use_real:
if not _nobjects_only:
# First, do the easy thing: real galaxies. We make the galsim.RealGalaxyCatalog()
# constructor do most of the work. But note that we don't actually need to
# bother with this if all we care about is the nobjects attribute.
self.real_cat = galsim.RealGalaxyCatalog(file_name,
image_dir=image_dir,
dir=dir,
preload=preload,
noise_dir=noise_dir)
# The fits name has _fits inserted before the .fits ending.
# Note: don't just use k = -5 in case it actually ends with .fits.fz
k = self.real_cat.file_name.find('.fits')
param_file_name = self.real_cat.file_name[:
k] + '_fits' + self.real_cat.file_name[
k:]
self.param_cat = pyfits.getdata(param_file_name)
else:
# Start by doing the same file_name parsing as we did for the real galaxy
param_file_name, _, _ = galsim.real._parse_files_dirs(
file_name, image_dir, dir, noise_dir)
try:
# Read in data.
self.param_cat = pyfits.getdata(param_file_name)
self.param_cat['fit_status']
except KeyError:
# But if that doesn't work, then the name might be the name of the real catalog,
# so try adding _fits to it as above.
k = param_file_name.find('.fits')
param_file_name = param_file_name[:
k] + '_fits' + param_file_name[
k:]
self.param_cat = pyfits.getdata(param_file_name)
# If requested, select galaxies based on existence of a usable fit.
self.orig_index = np.arange(len(self.param_cat))
if exclude_fail or exclude_bad or max_hlr > 0.:
mask = True
if exclude_fail:
sersicfit_status = self.param_cat['fit_status'][:, 4]
bulgefit_status = self.param_cat['fit_status'][:, 0]
mask &= ((sersicfit_status > 0) & (sersicfit_status < 5) &
(bulgefit_status > 0) & (bulgefit_status < 5))
if exclude_bad:
hlr = self.param_cat['sersicfit'][:, 1]
n = self.param_cat['sersicfit'][:, 2]
mask &= ((n < 5) | (hlr < 1. / cosmos_pix_scale))
# May add more cuts here if we discover other kinds of problematic objects.
if max_hlr > 0.:
hlr = self.param_cat['sersicfit'][:, 1]
mask &= (hlr < max_hlr / cosmos_pix_scale)
self.orig_index = self.orig_index[mask]
self.nobjects = len(self.orig_index)
def test_wfirst_bandpass():
"""Test the WFIRST bandpasses for basic sanity.
"""
import time
t1 = time.time()
from galsim._pyfits import pyfits
# Obtain the bandpasses with AB_zeropoint set
exp_time = 200. # non WFIRST exposure time
bp = galsim.wfirst.getBandpasses(AB_zeropoint=True, exptime=exp_time)
# Check if the zeropoints have been set correctly
AB_spec = lambda x: (3631e-23)*exp_time*(np.pi)*(100.**2)*\
(galsim.wfirst.diameter**2)*(1-galsim.wfirst.obscuration**2)/4.
AB_sed = galsim.SED(spec=AB_spec, wave_type='nm', flux_type='fnu')
for filter_name, filter_ in bp.iteritems():
mag = AB_sed.calculateMagnitude(bandpass=filter_)
np.testing.assert_almost_equal(
mag,
0.0,
decimal=6,
err_msg="Zeropoint not set accurately enough for bandpass filter \
{0}".format(filter_name))
# Do a slightly less trivial check of bandpass-related calculations:
# Jeff Kruk (at Goddard) took an SED template from the Castelli-Kurucz library, normalized it to
# a particular magnitude in SDSS g band, and checked the count rates he expects for the WFIRST
# bands. I (RM) independently did the same calculation (downloading the templates and bandpass
# myself and using GalSim for all the important bits of the calculation) and my results agree a
# the 5% level. Given that I didn't quite have the same SED, we were very happy with this level
# of agreement. The unit test below reproduces this test, and requires agreement at the 10%
# level.
# Jeff used the C-K template with solar metallicity, T=9550K, surface gravity logg=3.95. I
# downloaded a grid of templates and just used the nearest one, which has solar metallicity,
# T=9500K, surface gravity logg=4.0.
sed_data = pyfits.getdata(os.path.join('wfirst_files', 'ckp00_9500.fits'))
lam = sed_data.WAVELENGTH.astype(np.float64)
t = sed_data.g40.astype(np.float64)
sed_tab = galsim.LookupTable(x=lam, f=t, interpolant='linear')
sed = galsim.SED(sed_tab, wave_type='A', flux_type='flambda')
# Now take the SDSS g bandpass:
gfile = '/Users/rmandelb/Downloads/g.dat'
bp_dat = np.loadtxt(os.path.join('wfirst_files', 'g.dat')).transpose()
bp_tab = galsim.LookupTable(x=bp_dat[0, :],
f=bp_dat[1, :],
interpolant='linear')
bp_ref = galsim.Bandpass(bp_tab, wave_type='A')
# Set an AB zeropoint using WFIRST params:
eff_diam = 100. * galsim.wfirst.diameter * np.sqrt(
1. - galsim.wfirst.obscuration**2)
bp_ref = bp_ref.withZeropoint('AB',
effective_diameter=eff_diam,
exptime=galsim.wfirst.exptime)
# Now get a new SED that has magnitude -0.093 in this filter, since that's the normalization
# that Jeff imposed for his tests.
sed = sed.withMagnitude(-0.093, bp_ref)
# Reference count rates, from Jeff:
reference = {}
reference['Z087'] = 1.98e10
reference['Y106'] = 1.97e10
reference['J129'] = 1.52e10
reference['H158'] = 1.11e10
reference['F184'] = 0.58e10
reference['W149'] = 4.34e10
# Only 10% accuracy required because did not use quite the same stellar template. Fortunately,
# bugs can easily lead to orders of magnitude errors, so this unit test is still pretty
# non-trivial.
for filter_name, filter_ in bp.iteritems():
flux = sed.calculateFlux(filter_)
count_rate = flux / galsim.wfirst.exptime
np.testing.assert_allclose(
count_rate,
reference[filter_name],
rtol=0.1,
err_msg="Count rate for stellar model not as expected for bandpass "
"{0}".format(filter_name))
t2 = time.time()
print 'time for %s = %.2f' % (funcname(), t2 - t1)