def SavePreviewRGC(config,filename_rgc,n_gals_preview=10):
"""
Function for eyeballing the contents of the created mock RGC catalogs.
Arguments
---------
config config dict
filename_rgc filename of the newly created real galaxy catalog fits
n_gals_preview how many plots to produce (default=10)
"""
# open the RGC
table = pyfits.open(filename_rgc)[1].data
# get the image and PSF filenames
fits_gal = table[0]['GAL_FILENAME']
fits_psf = table[0]['PSF_FILENAME']
import pylab
# loop over galaxies and save plots
for n in range(n_gals_preview):
img_gal = pyfits.getdata(fits_gal,ext=n)
img_psf = pyfits.getdata(fits_psf,ext=n)
pylab.subplot(1,2,1)
pylab.imshow(img_gal,interpolation='nearest')
pylab.title('galaxy')
pylab.subplot(1,2,2)
pylab.imshow(img_psf,interpolation='nearest')
pylab.title('PSF')
filename_fig = 'fig.previewRGC.%s.%d.png' % (config['args'].filename_config,n)
pylab.savefig(filename_fig)
def __init__(self, file_name=None, image_dir=None, dir=None, preload=False, noise_dir=None,
use_real=True, exclude_fail=True, exclude_bad=True, max_hlr=0.,
_nobjects_only=False):
self.use_real = use_real
if self.use_real:
if not _nobjects_only:
# First, do the easy thing: real galaxies. We make the galsim.RealGalaxyCatalog()
# constructor do most of the work. But note that we don't actually need to
# bother with this if all we care about is the nobjects attribute.
self.real_cat = galsim.RealGalaxyCatalog(
file_name, image_dir=image_dir, dir=dir, preload=preload, noise_dir=noise_dir)
# The fits name has _fits inserted before the .fits ending.
# Note: don't just use k = -5 in case it actually ends with .fits.fz
k = self.real_cat.file_name.find('.fits')
param_file_name = self.real_cat.file_name[:k] + '_fits' + self.real_cat.file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
else:
# Start by doing the same file_name parsing as we did for the real galaxy
param_file_name, _, _ = galsim.real._parse_files_dirs(
file_name, image_dir, dir, noise_dir)
try:
# Read in data.
self.param_cat = pyfits.getdata(param_file_name)
self.param_cat['fit_status']
except KeyError:
# But if that doesn't work, then the name might be the name of the real catalog,
# so try adding _fits to it as above.
k = param_file_name.find('.fits')
param_file_name = param_file_name[:k] + '_fits' + param_file_name[k:]
self.param_cat = pyfits.getdata(param_file_name)
# If requested, select galaxies based on existence of a usable fit.
self.orig_index = np.arange(len(self.param_cat))
if exclude_fail or exclude_bad or max_hlr > 0.:
mask = True
if exclude_fail:
sersicfit_status = self.param_cat['fit_status'][:,4]
bulgefit_status = self.param_cat['fit_status'][:,0]
mask &= ( (sersicfit_status > 0) &
(sersicfit_status < 5) &
(bulgefit_status > 0) &
(bulgefit_status < 5) )
if exclude_bad:
hlr = self.param_cat['sersicfit'][:,1]
n = self.param_cat['sersicfit'][:,2]
mask &= ( (n < 5) | (hlr < 1./cosmos_pix_scale) )
# May add more cuts here if we discover other kinds of problematic objects.
if max_hlr > 0.:
hlr = self.param_cat['sersicfit'][:,1]
mask &= (hlr < max_hlr / cosmos_pix_scale)
self.orig_index = self.orig_index[mask]
self.nobjects = len(self.orig_index)
def read_fits(self):
"""Read in a DES_Shapelet stored using the the FITS-file version.
"""
from galsim import pyfits
cat = pyfits.getdata(self.file_name,1)
# These fields each only contain one element, hence the [0]'s.
self.psf_order = cat.field('psf_order')[0]
self.psf_size = (self.psf_order+1) * (self.psf_order+2) / 2
self.sigma = cat.field('sigma')[0]
self.fit_order = cat.field('fit_order')[0]
self.fit_size = (self.fit_order+1) * (self.fit_order+2) / 2
self.npca = cat.field('npca')[0]
self.bounds = galsim.BoundsD(
float(cat.field('xmin')[0]), float(cat.field('xmax')[0]),
float(cat.field('ymin')[0]), float(cat.field('ymax')[0]))
self.ave_psf = cat.field('ave_psf')[0]
assert self.ave_psf.shape == (self.psf_size,)
# Note: older pyfits versions don't get the shape right.
# For newer pyfits versions the reshape command should be a no op.
self.rot_matrix = cat.field('rot_matrix')[0].reshape((self.psf_size,self.npca)).T
assert self.rot_matrix.shape == (self.npca, self.psf_size)
self.interp_matrix = cat.field('interp_matrix')[0].reshape((self.npca,self.fit_size)).T
assert self.interp_matrix.shape == (self.fit_size, self.npca)
def __init__(self, file_name=None, image_dir=None, dir=None, preload=False,
noise_dir=None, logger=None, _nobjects_only=False):
self.file_name, self.image_dir, self.noise_dir = \
_parse_files_dirs(file_name, image_dir, dir, noise_dir)
self.cat = pyfits.getdata(self.file_name)
self.nobjects = len(self.cat) # number of objects in the catalog
if _nobjects_only: return # Exit early if that's all we needed.
ident = self.cat.field('ident') # ID for object in the training sample
# We want to make sure that the ident array contains all strings.
# Strangely, ident.astype(str) produces a string with each element == '1'.
# Hence this way of doing the conversion:
self.ident = [ "%s"%val for val in ident ]
self.gal_file_name = self.cat.field('gal_filename') # file containing the galaxy image
self.psf_file_name = self.cat.field('PSF_filename') # file containing the PSF image
# Add the directories:
self.gal_file_name = [ os.path.join(self.image_dir,f) for f in self.gal_file_name ]
self.psf_file_name = [ os.path.join(self.image_dir,f) for f in self.psf_file_name ]
# We don't require the noise_filename column. If it is not present, we will use
# Uncorrelated noise based on the variance column.
try:
self.noise_file_name = self.cat.field('noise_filename') # file containing the noise cf
self.noise_file_name = [ os.path.join(self.noise_dir,f) for f in self.noise_file_name ]
except:
self.noise_file_name = None
self.gal_hdu = self.cat.field('gal_hdu') # HDU containing the galaxy image
self.psf_hdu = self.cat.field('PSF_hdu') # HDU containing the PSF image
self.pixel_scale = self.cat.field('pixel_scale') # pixel scale for image (could be different
# if we have training data from other datasets... let's be general here and make it a
# vector in case of mixed training set)
self.variance = self.cat.field('noise_variance') # noise variance for image
self.mag = self.cat.field('mag') # apparent magnitude
self.band = self.cat.field('band') # bandpass in which apparent mag is measured, e.g., F814W
self.weight = self.cat.field('weight') # weight factor to account for size-dependent
# probability
self.saved_noise_im = {}
self.loaded_files = {}
self.logger = logger
# The pyfits commands aren't thread safe. So we need to make sure the methods that
# use pyfits are not run concurrently from multiple threads.
from multiprocessing import Lock
self.gal_lock = Lock() # Use this when accessing gal files
self.psf_lock = Lock() # Use this when accessing psf files
self.loaded_lock = Lock() # Use this when opening new files from disk
self.noise_lock = Lock() # Use this for building the noise image(s) (usually just one)
# Preload all files if desired
if preload: self.preload()
self._preload = preload
def SavePreviewRGC(config, filename_rgc, n_gals_preview=10):
"""
Function for eyeballing the contents of the created mock RGC catalogs.
Arguments
---------
config config dict
filename_rgc filename of the newly created real galaxy catalog fits
n_gals_preview how many plots to produce (default=10)
"""
# open the RGC
table = pyfits.open(filename_rgc)[1].data
# get the image and PSF filenames
fits_gal = table[0]['GAL_FILENAME']
fits_psf = table[0]['PSF_FILENAME']
import pylab
# loop over galaxies and save plots
for n in range(n_gals_preview):
img_gal = pyfits.getdata(fits_gal, ext=n)
img_psf = pyfits.getdata(fits_psf, ext=n)
pylab.subplot(1, 2, 1)
pylab.imshow(img_gal, interpolation='nearest')
pylab.title('galaxy')
pylab.subplot(1, 2, 2)
pylab.imshow(img_psf, interpolation='nearest')
pylab.title('PSF')
filename_fig = 'fig.previewRGC.%s.%d.png' % (
config['args'].filename_config, n)
pylab.savefig(filename_fig)
def read_fits(self, hdu, nobjects_only):
"""Read in an input catalog from a FITS file.
"""
from galsim import pyfits, pyfits_version
raw_data = pyfits.getdata(self.file_name, hdu)
if pyfits_version > '3.0':
self.names = raw_data.columns.names
else:
self.names = raw_data.dtype.names
self.nobjects = len(raw_data.field(self.names[0]))
if (nobjects_only): return
# The pyfits raw_data is a FITS_rec object, which isn't picklable, so we need to
# copy the fields into a new structure to make sure our Catalog is picklable.
# The simplest is probably a dict keyed by the field names, which we save as self.data.
self.data = {}
for name in self.names:
self.data[name] = raw_data.field(name)
self.ncols = len(self.names)
self.isfits = True
def getNoiseProperties(self, i):
"""Returns the components needed to make the noise correlation function at index `i`.
Specifically, the noise image (or None), the pixel_scale, and the noise variance,
as a tuple (im, scale, var).
"""
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Start getNoise',i)
if self.noise_file_name is None:
im = None
else:
if i >= len(self.noise_file_name):
raise IndexError(
'index %d given to getNoise is out of range (0..%d)'%(
i,len(self.noise_file_name)-1))
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
self.noise_lock.acquire()
# Again, a second check in case two processes get here at the same time.
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
import numpy
array = pyfits.getdata(self.noise_file_name[i])
im = galsim.Image(numpy.ascontiguousarray(array.astype(numpy.float64)),
scale=self.pixel_scale[i])
self.saved_noise_im[self.noise_file_name[i]] = im
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Built noise im',i)
self.noise_lock.release()
return im, self.pixel_scale[i], self.variance[i]
def getNoiseProperties(self, i):
"""Returns the components needed to make the noise correlation function at index `i`.
Specifically, the noise image (or None), the pixel_scale, and the noise variance,
as a tuple (im, scale, var).
"""
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Start getNoise',i)
if self.noise_file_name is None:
im = None
else:
if i >= len(self.noise_file_name):
raise IndexError(
'index %d given to getNoise is out of range (0..%d)'%(
i,len(self.noise_file_name)-1))
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
self.noise_lock.acquire()
# Again, a second check in case two processes get here at the same time.
if self.noise_file_name[i] in self.saved_noise_im:
im = self.saved_noise_im[self.noise_file_name[i]]
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Got saved noise im',i)
else:
import numpy
array = pyfits.getdata(self.noise_file_name[i])
im = galsim.Image(numpy.ascontiguousarray(array.astype(numpy.float64)))
self.saved_noise_im[self.noise_file_name[i]] = im
if self.logger:
self.logger.debug('RealGalaxyCatalog %d: Built noise im',i)
self.noise_lock.release()
return im, self.pixel_scale[i], self.variance[i]
def __init__(self, file_name, image_dir=None, dir=None, preload=False, nobjects_only=False,
noise_dir=None, logger=None):
import os
# First build full file_name
if dir is None:
self.file_name = file_name
if image_dir is None:
self.image_dir = os.path.dirname(file_name)
elif os.path.dirname(image_dir) == '':
self.image_dir = os.path.join(os.path.dirname(self.file_name),image_dir)
else:
self.image_dir = image_dir
else:
self.file_name = os.path.join(dir,file_name)
if image_dir is None:
self.image_dir = dir
else:
self.image_dir = os.path.join(dir,image_dir)
if not os.path.isdir(self.image_dir):
raise RuntimeError(self.image_dir+' directory does not exist!')
if noise_dir is None:
self.noise_dir = self.image_dir
else:
if not os.path.isdir(noise_dir):
raise RuntimeError(noise_dir+' directory does not exist!')
self.noise_dir = noise_dir
cat = pyfits.getdata(self.file_name)
self.nobjects = len(cat) # number of objects in the catalog
if nobjects_only: return # Exit early if that's all we needed.
ident = cat.field('ident') # ID for object in the training sample
# We want to make sure that the ident array contains all strings.
# Strangely, ident.astype(str) produces a string with each element == '1'.
# Hence this way of doing the conversion:
self.ident = [ "%s"%val for val in ident ]
self.gal_file_name = cat.field('gal_filename') # file containing the galaxy image
self.psf_file_name = cat.field('PSF_filename') # file containing the PSF image
# Add the directories:
self.gal_file_name = [ os.path.join(self.image_dir,f) for f in self.gal_file_name ]
self.psf_file_name = [ os.path.join(self.image_dir,f) for f in self.psf_file_name ]
# We don't require the noise_filename column. If it is not present, we will use
# Uncorrelated noise based on the variance column.
try:
self.noise_file_name = cat.field('noise_filename') # file containing the noise cf
self.noise_file_name = [ os.path.join(self.noise_dir,f) for f in self.noise_file_name ]
except:
self.noise_file_name = None
self.gal_hdu = cat.field('gal_hdu') # HDU containing the galaxy image
self.psf_hdu = cat.field('PSF_hdu') # HDU containing the PSF image
self.pixel_scale = cat.field('pixel_scale') # pixel scale for image (could be different
# if we have training data from other datasets... let's be general here and make it a
# vector in case of mixed training set)
self.variance = cat.field('noise_variance') # noise variance for image
self.mag = cat.field('mag') # apparent magnitude
self.band = cat.field('band') # bandpass in which apparent mag is measured, e.g., F814W
self.weight = cat.field('weight') # weight factor to account for size-dependent
# probability
self.saved_noise_im = {}
self.loaded_files = {}
self.logger = logger
# The pyfits commands aren't thread safe. So we need to make sure the methods that
# use pyfits are not run concurrently from multiple threads.
from multiprocessing import Lock
self.gal_lock = Lock() # Use this when accessing gal files
self.psf_lock = Lock() # Use this when accessing psf files
self.loaded_lock = Lock() # Use this when opening new files from disk
self.noise_lock = Lock() # Use this for building the noise image(s) (usually just one)
# Preload all files if desired
if preload: self.preload()