def SavePreviewRGC(config,filename_rgc,n_gals_preview=10):
"""
Function for eyeballing the contents of the created mock RGC catalogs.
Arguments
---------
config config dict
filename_rgc filename of the newly created real galaxy catalog fits
n_gals_preview how many plots to produce (default=10)
"""
# open the RGC
table = pyfits.open(filename_rgc)[1].data
# get the image and PSF filenames
fits_gal = table[0]['GAL_FILENAME']
fits_psf = table[0]['PSF_FILENAME']
import pylab
# loop over galaxies and save plots
for n in range(n_gals_preview):
img_gal = pyfits.getdata(fits_gal,ext=n)
img_psf = pyfits.getdata(fits_psf,ext=n)
pylab.subplot(1,2,1)
pylab.imshow(img_gal,interpolation='nearest')
pylab.title('galaxy')
pylab.subplot(1,2,2)
pylab.imshow(img_psf,interpolation='nearest')
pylab.title('PSF')
filename_fig = 'fig.previewRGC.%s.%d.png' % (config['args'].filename_config,n)
pylab.savefig(filename_fig)
def SavePreviewRGC(config,filename_rgc,n_gals_preview=10):
"""
Function for eyeballing the contents of the created mock RGC catalogs.
Arguments
---------
config config dict
filename_rgc filename of the newly created real galaxy catalog fits
n_gals_preview how many plots to produce (default=10)
"""
# open the RGC
table = pyfits.open(filename_rgc)[1].data
# get the image and PSF filenames
fits_gal = table[0]['GAL_FILENAME']
fits_psf = table[0]['PSF_FILENAME']
import pylab
# loop over galaxies and save plots
for n in range(n_gals_preview):
img_gal = pyfits.getdata(fits_gal,ext=n)
img_psf = pyfits.getdata(fits_psf,ext=n)
pylab.subplot(1,2,1)
pylab.imshow(img_gal,interpolation='nearest')
pylab.title('galaxy')
pylab.subplot(1,2,2)
pylab.imshow(img_psf,interpolation='nearest')
pylab.title('PSF')
filename_fig = 'fig.previewRGC.%s.%d.png' % (config['args'].filename_config,n)
pylab.savefig(filename_fig)
def preload(self):
"""Preload the files into memory.
There are memory implications to this, so we don't do this by default. However, it can be
a big speedup if memory isn't an issue. Especially if many (or all) of the images are
stored in the same file as different HDUs.
"""
import numpy
from multiprocessing import Lock
if self.logger:
self.logger.debug('RealGalaxyCatalog: start preload')
for file_name in numpy.concatenate((self.gal_file_name , self.psf_file_name)):
# numpy sometimes add a space at the end of the string that is not present in
# the original file. Stupid. But this next line removes it.
file_name = file_name.strip()
if file_name not in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: preloading %s',file_name)
# I use memmap=False, because I was getting problems with running out of
# file handles in the great3 real_gal run, which uses a lot of rgc files.
# I think there must be a bug in pyfits that leaves file handles open somewhere
# when memmap = True. Anyway, I don't know what the performance implications
# are (since I couldn't finish the run with the default memmap=True), but I
# don't think there is much impact either way with memory mapping in our case.
self.loaded_files[file_name] = pyfits.open(file_name,memmap=False)
def preload(self):
"""Preload the files into memory.
There are memory implications to this, so we don't do this by default. However, it can be
a big speedup if memory isn't an issue. Especially if many (or all) of the images are
stored in the same file as different HDUs.
"""
import numpy
from multiprocessing import Lock
if self.logger:
self.logger.debug('RealGalaxyCatalog: start preload')
for file_name in numpy.concatenate((self.gal_file_name , self.psf_file_name)):
# numpy sometimes add a space at the end of the string that is not present in
# the original file. Stupid. But this next line removes it.
file_name = file_name.strip()
if file_name not in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: preloading %s',file_name)
# I use memmap=False, because I was getting problems with running out of
# file handles in the great3 real_gal run, which uses a lot of rgc files.
# I think there must be a bug in pyfits that leaves file handles open somewhere
# when memmap = True. Anyway, I don't know what the performance implications
# are (since I couldn't finish the run with the default memmap=True), but I
# don't think there is much impact either way with memory mapping in our case.
self.loaded_files[file_name] = pyfits.open(file_name,memmap=False)
def _getFile(self, file_name):
from multiprocessing import Lock
if file_name in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: File %s is already open',file_name)
f = self.loaded_files[file_name]
else:
self.loaded_lock.acquire()
# Check again in case two processes both hit the else at the same time.
if file_name in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: File %s is already open',file_name)
f = self.loaded_files[file_name]
else:
if self.logger:
self.logger.debug('RealGalaxyCatalog: open file %s',file_name)
f = pyfits.open(file_name,memmap=False)
self.loaded_files[file_name] = f
self.loaded_lock.release()
return f
def _getFile(self, file_name):
from multiprocessing import Lock
if file_name in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: File %s is already open',file_name)
f = self.loaded_files[file_name]
else:
self.loaded_lock.acquire()
# Check again in case two processes both hit the else at the same time.
if file_name in self.loaded_files:
if self.logger:
self.logger.debug('RealGalaxyCatalog: File %s is already open',file_name)
f = self.loaded_files[file_name]
else:
if self.logger:
self.logger.debug('RealGalaxyCatalog: open file %s',file_name)
f = pyfits.open(file_name,memmap=False)
self.loaded_files[file_name] = f
self.loaded_lock.release()
return f
def time_gzip():
"""Time different functions for gzip"""
import time
for file, gzfile, size, n_iter in [ (big_im_file, big_im_file_gz, 5000, 1),
(medium_im_file, medium_im_file_gz, 1000, 20),
(small_im_file, small_im_file_gz, 200, 400) ]:
infile_name = os.path.join(dir,file)
hdu_list = pyfits.open(infile_name)
outfile_name = os.path.join(dir,gzfile)
t1 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_in_mem(hdu_list, outfile_name)
except:
# Only report error the first time.
if n_iter == 1:
import traceback
print('gzip_in_mem failed with exception:')
traceback.print_exc()
t2 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_tmp(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('gzip_tmp failed with exception:')
traceback.print_exc()
t3 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_call(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('gzip_call failed with exception:')
traceback.print_exc()
t4 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_call2(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('gzip_call2 failed with exception:')
traceback.print_exc()
t5 = time.time()
print('Times for %d iterations of writing to %s (%d x %d): '%(n_iter, gzfile, size, size))
print(' time for gzip_in_mem = %.2f'%(t2-t1))
print(' time for gzip_tmp = %.2f'%(t3-t2))
print(' time for gzip_call = %.2f'%(t4-t3))
print(' time for gzip_call2 = %.2f'%(t5-t4))
hdu_list.close()
default_order = [ f.__name__ for f in galsim.fits._write_file.gz_methods ]
print('The current default order for gzip write is ',default_order)
print()
import shutil
import subprocess
# Setup various file and directory names:
work_dir = '/gpfs01/astro/workarea/mjarvis'
tile_name = 'DES0436-5748'
out_dir = os.path.join(work_dir,tile_name)
print 'tile = ',tile_name
print 'out_dir = ',out_dir
meds_dir = os.path.join('/astro/u/astrodat/data/DES/meds/011/20130820000021_DES0436-5748')
meds_file = tile_name + '-r-meds-011.fits.fz'
print 'meds_file = ',meds_file
# Open the current meds file:
meds = pyfits.open(os.path.join(meds_dir,meds_file))
print 'Opened meds file'
# First pull out some information from the meta catalog
meta = meds[3].data[0]
magzp_ref = meta['magzp_ref'] # 30
se_hdu = meta['se_hdu'] # 2
se_wt_hdu = meta['se_wt_hdu'] # 4
se_badpix_hdu = meta['se_badpix_hdu'] # 3
sky_hdu = meta['sky_hdu'] # 2
seg_hdu = meta['seg_hdu'] # 2
coadd_hdu = meta['coadd_hdu'] # 2
coadd_wt_hdu = meta['coadd_wt_hdu'] # 3
coadd_seg_hdu = meta['coadd_seg_hdu'] # 0 NB This is a bug.
fake_coadd_seg = meta['fake_coadd_seg'] # 0
des_root = meta['DESDATA'] # /astro/u/astrodat/data/DES
def time_gzip():
"""Time different functions for gzip"""
import time
for file, gzfile, size, n_iter in [
(big_im_file, big_im_file_gz, 5000, 1),
(medium_im_file, medium_im_file_gz, 1000, 20),
(small_im_file, small_im_file_gz, 200, 400)
]:
infile_name = os.path.join(dir, file)
hdu_list = pyfits.open(infile_name)
outfile_name = os.path.join(dir, gzfile)
t1 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_in_mem(hdu_list, outfile_name)
except:
# Only report error the first time.
if n_iter == 1:
import traceback
print 'gzip_in_mem failed with exception:'
traceback.print_exc()
t2 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_tmp(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'gzip_tmp failed with exception:'
traceback.print_exc()
t3 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_call(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'gzip_call failed with exception:'
traceback.print_exc()
t4 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.gzip_call2(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'gzip_call2 failed with exception:'
traceback.print_exc()
t5 = time.time()
print 'Times for %d iterations of writing to %s (%d x %d): ' % (
n_iter, gzfile, size, size)
print ' time for gzip_in_mem = %.2f' % (t2 - t1)
print ' time for gzip_tmp = %.2f' % (t3 - t2)
print ' time for gzip_call = %.2f' % (t4 - t3)
print ' time for gzip_call2 = %.2f' % (t5 - t4)
default_order = [f.__name__ for f in galsim.fits._write_file.gz_methods]
print 'The current default order for gzip write is ', default_order
print
def time_bzip2():
"""Time different functions for bzip2"""
import time
for file, bz2file, size, n_iter in [ (big_im_file, big_im_file_bz2, 5000, 1),
(medium_im_file, medium_im_file_bz2, 1000, 20),
(small_im_file, small_im_file_bz2, 200, 400) ]:
infile_name = os.path.join(dir,file)
hdu_list = pyfits.open(infile_name)
outfile_name = os.path.join(dir,bz2file)
t1 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bz2_in_mem(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'bz2_in_mem failed with exception:'
traceback.print_exc()
t2 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bz2_tmp(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'bz2_tmp failed with exception:'
traceback.print_exc()
t3 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bzip2_call(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'bzip2_call failed with exception:'
traceback.print_exc()
t4 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bzip2_call2(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print 'bzip2_call2 failed with exception:'
traceback.print_exc()
t5 = time.time()
print 'Times for %d iterations of writing to %s (%d x %d): '%(n_iter, bz2file, size, size)
print ' time for bz2_in_mem = %.2f'%(t2-t1)
print ' time for bz2_tmp = %.2f'%(t3-t2)
print ' time for bzip2_call = %.2f'%(t4-t3)
print ' time for bzip2_call2 = %.2f'%(t5-t4)
default_order = [ f.__name__ for f in galsim.fits._write_file.bz2_methods ]
print 'The current default order for bzip2 write is ',default_order
print
def __call__(self, file, hdus, clobber, file_compress, pyfits_compress):
import os
if os.path.isfile(file):
if clobber:
os.remove(file)
else:
raise IOError('File %r already exists'%file)
if not file_compress:
hdus.writeto(file)
else:
if self.in_mem:
try:
# The compression routines work better if we first write to an internal buffer
# and then output that to a file.
import io
buf = io.BytesIO()
hdus.writeto(buf)
data = buf.getvalue()
except:
self.in_mem = False
return self(file,hdus,clobber,file_compress)
else:
# However, pyfits versions before 2.3 do not support writing to a buffer, so the
# abover code with fail. We need to use a temporary in that case.
tmp = file + '.tmp'
# It would be pretty odd for this filename to already exist, but just in case...
while os.path.isfile(tmp):
tmp = tmp + '.tmp'
hdus.writeto(tmp)
with open(tmp,"r") as buf:
data = buf.read()
os.remove(tmp)
if file_compress == 'gzip':
import gzip
# There is a compresslevel option (for both gzip and bz2), but we just use the
# default.
fout = gzip.GzipFile(file, 'wb')
fout.write(data)
fout.close()
elif file_compress == 'bzip2':
import bz2
fout = bz2.BZ2File(file, 'wb')
fout.write(data)
fout.close()
else:
raise ValueError("Unknown file_compression")
# There is a bug in pyfits where they don't add the size of the variable length array
# to the TFORMx header keywords. They should have size at the end of them.
# This bug has been fixed in version 3.1.2.
# (See http://trac.assembla.com/pyfits/ticket/199)
if pyfits_compress and pyfits_version < '3.1.2':
with pyfits.open(file,'update',disable_image_compression=True) as hdus:
for hdu in hdus[1:]: # Skip PrimaryHDU
# Find the maximum variable array length
max_ar_len = max([ len(ar[0]) for ar in hdu.data ])
# Add '(N)' to the TFORMx keywords for the variable array items
s = '(%d)'%max_ar_len
for key in hdu.header.keys():
if key.startswith('TFORM'):
tform = hdu.header[key]
# Only update if the form is a P (= variable length data)
# and the (*) is not there already.
if 'P' in tform and '(' not in tform:
hdu.header[key] = tform + s
# Workaround for a bug in some pyfits 3.0.x versions
# It was fixed in 3.0.8. I'm not sure when the bug was
# introduced, but I believe it was 3.0.3.
if (pyfits_version > '3.0' and pyfits_version < '3.0.8' and
'COMPRESSION_ENABLED' in pyfits.hdu.compressed.__dict__):
pyfits.hdu.compressed.COMPRESSION_ENABLED = True
def storePSFImages(PSF_dict, filename, bandpass_list=None, clobber=False):
"""
This is a routine to store images of chromatic WFIRST PSFs in different bands for each SCA. It
takes an output dict of PSFs (`PSF_dict`) directly from getPSF(). The output will be a file
(`filename`) that has all the images, along with an HDU that contains a FITS table indicating
the bandpasses, SCAs, and other information needed to reconstruct the PSF information.
This routine is not meant to work for PSFs from getPSF() that are completely achromatic. The
reason for this is that those PSFs are quite fast to generate, so there is little benefit to
storing them.
@param PSF_dict A dict of PSF objects for each SCA, in the same format as output by
the getPSF() routine (though it can take versions that have been
modified, for example in the inclusion of an SED).
@param filename The name of the file to which the images and metadata should be
written; extension should be *.fits.
@param bandpass_list A list of bandpass names for which images should be generated and
stored. If None, all WFIRST imaging passbands are used.
[default: None]
@param clobber Should the routine clobber `filename` (if they already exist)?
[default: False]
"""
# Check for sane input PSF_dict.
if len(PSF_dict) == 0 or len(PSF_dict) > galsim.wfirst.n_sca or \
min(PSF_dict.keys()) < 1 or max(PSF_dict.keys()) > galsim.wfirst.n_sca:
raise ValueError("PSF_dict must come from getPSF()!")
# Check if file already exists and warn about clobbering.
if os.path.exists(filename):
if clobber is False:
raise ValueError("Output file already exists, and clobber is not set!")
else:
import warnings
warnings.warn("Output file already exists, and will be clobbered.")
# Check that bandpass list input is okay. It should be strictly a subset of the default list of
# bandpasses.
if bandpass_list is None:
bandpass_list = default_bandpass_list
else:
if not isinstance(bandpass_list[0], str):
raise ValueError("Expected input list of bandpass names!")
if not set(bandpass_list).issubset(default_bandpass_list):
err_msg = ''
for item in default_bandpass_list:
err_msg += item+' '
raise ValueError("Bandpass list must be a subset of the default list, containing %s"
%err_msg)
# Get all the WFIRST bandpasses.
bandpass_dict = galsim.wfirst.getBandpasses()
# Loop through making images and lists of their relevant parameters.
im_list = []
bp_name_list = []
SCA_index_list = []
for SCA in PSF_dict.keys():
PSF = PSF_dict[SCA]
if not isinstance(PSF, galsim.ChromaticOpticalPSF):
raise RuntimeError("Error, PSFs are not ChromaticOpticalPSFs.")
star = galsim.Gaussian(sigma=1.e-8, flux=1.)
for bp_name in bandpass_list:
bandpass = bandpass_dict[bp_name]
star_sed = galsim.SED(lambda x:1).withFlux(1, bandpass)
obj = galsim.Convolve(star*star_sed, PSF)
im = obj.drawImage(bandpass, scale=0.5*galsim.wfirst.pixel_scale,
method='no_pixel')
im_list.append(im)
bp_name_list.append(bp_name)
SCA_index_list.append(SCA)
# Save images to file.
n_ims = len(im_list)
galsim.fits.writeMulti(im_list, filename, clobber=clobber)
# Add data to file, after constructing a FITS table. Watch out for clobbering.
bp_names = pyfits.Column(name='bandpass', format='A10', array=np.array(bp_name_list))
SCA_indices = pyfits.Column(name='SCA', format='J', array=np.array(SCA_index_list))
cols = pyfits.ColDefs([bp_names, SCA_indices])
tbhdu = pyfits.BinTableHDU.from_columns(cols)
f = pyfits.open(filename, mode='update')
f.append(tbhdu)
f.flush()
f.close()
def read(self):
from galsim import pyfits
hdu = pyfits.open(self.file_name)[1]
# Number of parameters used for the interpolation. We require this to be 2.
pol_naxis = hdu.header['POLNAXIS']
if pol_naxis != 2:
raise IOError("PSFEx: Expected POLNAXIS == 2, got %d"%pol_naxis)
# These are the names of the two axes. Should be X_IMAGE, Y_IMAGE.
# If they aren't, then the way we use the interpolation will be wrong.
# Well, really they can also be XWIN_IMAGE, etc. So just check that it
# starts with X and ends with IMAGE.
pol_name1 = hdu.header['POLNAME1']
if not (pol_name1.startswith('X') and pol_name1.endswith('IMAGE')):
raise IOError("PSFEx: Expected POLNAME1 == X*_IMAGE, got %s"%pol_name1)
pol_name2 = hdu.header['POLNAME2']
if not (pol_name2.startswith('Y') and pol_name2.endswith('IMAGE')):
raise IOError("PSFEx: Expected POLNAME2 == Y*_IMAGE, got %s"%pol_name2)
# Zero points and scale. Interpolation is in terms of (x-x0)/xscale, (y-y0)/yscale
pol_zero1 = hdu.header['POLZERO1']
pol_zero2 = hdu.header['POLZERO2']
pol_scal1 = hdu.header['POLSCAL1']
pol_scal2 = hdu.header['POLSCAL2']
# This defines the number of "context groups".
# Here is Emmanuel's explanation:
#
# POLNGRP is the number of "context groups". A group represents a set of variables
# (SExtractor measurements or FITS header parameters if preceded with ":") which share
# the same maximum polynomial degree. For instance if x and y are in group 1, and the
# degree of that group is 2, and z is in group 2 with degree 1, the polynomial will
# consist of:
# 1, x, x^2, y, y.x, y^2, z, z.x^2, z.y, z.y.x, z.y^2
# (see eq 14 in https://www.astromatic.net/pubsvn/software/psfex/trunk/doc/psfex.pdf )
# By default, POLNGRP is 1 and the group contains X_IMAGE and Y_IMAGE measurements
# from SExtractor.
#
# For now, we require this to be 1, since I didn't have any files with POLNGRP != 1 to
# test on.
pol_ngrp = hdu.header['POLNGRP']
if pol_ngrp != 1:
raise IOError("PSFEx: Current implementation requires POLNGRP == 1, got %d"%pol_ngrp)
# Which group each item is in. We require group 1.
pol_group1 = hdu.header['POLGRP1']
if pol_group1 != 1:
raise IOError("PSFEx: Expected POLGRP1 == 1, got %s"%pol_group1)
pol_group2 = hdu.header['POLGRP2']
if pol_group2 != 1:
raise IOError("PSFEx: Expected POLGRP2 == 1, got %s"%pol_group2)
# The degree of the polynomial. E.g. POLDEG1 = 2 means the values will be:
# 1, x, x^2, y, xy, y^2
# If we start allowing POLNGRP > 1, there is a separate degree for each group.
pol_deg = hdu.header['POLDEG1']
# The number of axes in the basis object. We require this to be 3.
psf_naxis = hdu.header['PSFNAXIS']
if psf_naxis != 3:
raise IOError("PSFEx: Expected PSFNAXIS == 3, got %d"%psfnaxis)
# The first two axes are the image size of the PSF postage stamp.
psf_axis1 = hdu.header['PSFAXIS1']
psf_axis2 = hdu.header['PSFAXIS2']
# The third axis is the direction of the polynomial interpolation. So it should
# be equal to (d+1)(d+2)/2.
psf_axis3 = hdu.header['PSFAXIS3']
if psf_axis3 != ((pol_deg+1)*(pol_deg+2))/2:
raise IOError("PSFEx: POLDEG and PSFAXIS3 disagree")
# This is the PSF "sample size". Again, from Emmanuel:
#
# PSF_SAMP is the sampling step of the PSF. PSF_SAMP=0.5 means that the PSF model has
# two samples per original image pixel (superresolution, so in automatic mode it is a
# sign that the original images were undersampled)
#
# In other words, it can be thought of as a unit conversion:
# "image pixels" / "psfex pixels"
# So 1 image pixel = (1/psf_samp) psfex pixels.
psf_samp = hdu.header['PSF_SAMP']
# The basis object is a data cube (assuming PSFNAXIS==3)
# Note: older pyfits versions don't get the shape right.
# For newer pyfits versions the reshape command should be a no op.
basis = hdu.data.field('PSF_MASK')[0].reshape(psf_axis3,psf_axis2,psf_axis1)
# Make sure this turned out right.
if basis.shape[0] != psf_axis3:
raise IOError("PSFEx: PSFAXIS3 disagrees with actual basis size")
if basis.shape[1] != psf_axis2:
raise IOError("PSFEx: PSFAXIS2 disagrees with actual basis size")
if basis.shape[2] != psf_axis1:
raise IOError("PSFEx: PSFAXIS1 disagrees with actual basis size")
# Save some of these values for use in building the interpolated images
self.basis = basis
self.fit_order = pol_deg
self.fit_size = psf_axis3
self.x_zero = pol_zero1
self.y_zero = pol_zero2
self.x_scale = pol_scal1
self.y_scale = pol_scal2
self.sample_scale = psf_samp
def time_bzip2():
"""Time different functions for bzip2"""
import time
for file, bz2file, size, n_iter in [ (big_im_file, big_im_file_bz2, 5000, 1),
(medium_im_file, medium_im_file_bz2, 1000, 20),
(small_im_file, small_im_file_bz2, 200, 400) ]:
infile_name = os.path.join(dir,file)
hdu_list = pyfits.open(infile_name)
outfile_name = os.path.join(dir,bz2file)
t1 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bz2_in_mem(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('bz2_in_mem failed with exception:')
traceback.print_exc()
t2 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bz2_tmp(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('bz2_tmp failed with exception:')
traceback.print_exc()
t3 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bzip2_call(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('bzip2_call failed with exception:')
traceback.print_exc()
t4 = time.time()
try:
for iter in range(n_iter):
if os.path.isfile(outfile_name):
os.remove(outfile_name)
galsim.fits._write_file.bzip2_call2(hdu_list, outfile_name)
except:
if n_iter == 1:
import traceback
print('bzip2_call2 failed with exception:')
traceback.print_exc()
t5 = time.time()
print('Times for %d iterations of writing to %s (%d x %d): '%(n_iter, bz2file, size, size))
print(' time for bz2_in_mem = %.2f'%(t2-t1))
print(' time for bz2_tmp = %.2f'%(t3-t2))
print(' time for bzip2_call = %.2f'%(t4-t3))
print(' time for bzip2_call2 = %.2f'%(t5-t4))
hdu_list.close()
default_order = [ f.__name__ for f in galsim.fits._write_file.bz2_methods ]
print('The current default order for bzip2 write is ',default_order)
print()
def read(self):
from galsim import pyfits
hdu = pyfits.open(self.file_name)[1]
# Number of parameters used for the interpolation. We require this to be 2.
pol_naxis = hdu.header['POLNAXIS']
if pol_naxis != 2:
raise IOError("PSFEx: Expected POLNAXIS == 2, got %d" % pol_naxis)
# These are the names of the two axes. Should be X_IMAGE, Y_IMAGE.
# If they aren't, then the way we use the interpolation will be wrong.
# Well, really they can also be XWIN_IMAGE, etc. So just check that it
# starts with X and ends with IMAGE.
pol_name1 = hdu.header['POLNAME1']
if not (pol_name1.startswith('X') and pol_name1.endswith('IMAGE')):
raise IOError("PSFEx: Expected POLNAME1 == X*_IMAGE, got %s" %
pol_name1)
pol_name2 = hdu.header['POLNAME2']
if not (pol_name2.startswith('Y') and pol_name2.endswith('IMAGE')):
raise IOError("PSFEx: Expected POLNAME2 == Y*_IMAGE, got %s" %
pol_name2)
# Zero points and scale. Interpolation is in terms of (x-x0)/xscale, (y-y0)/yscale
pol_zero1 = hdu.header['POLZERO1']
pol_zero2 = hdu.header['POLZERO2']
pol_scal1 = hdu.header['POLSCAL1']
pol_scal2 = hdu.header['POLSCAL2']
# This defines the number of "context groups".
# Here is Emmanuel's explanation:
#
# POLNGRP is the number of "context groups". A group represents a set of variables
# (SExtractor measurements or FITS header parameters if preceded with ":") which share
# the same maximum polynomial degree. For instance if x and y are in group 1, and the
# degree of that group is 2, and z is in group 2 with degree 1, the polynomial will
# consist of:
# 1, x, x^2, y, y.x, y^2, z, z.x^2, z.y, z.y.x, z.y^2
# (see eq 14 in https://www.astromatic.net/pubsvn/software/psfex/trunk/doc/psfex.pdf )
# By default, POLNGRP is 1 and the group contains X_IMAGE and Y_IMAGE measurements
# from SExtractor.
#
# For now, we require this to be 1, since I didn't have any files with POLNGRP != 1 to
# test on.
pol_ngrp = hdu.header['POLNGRP']
if pol_ngrp != 1:
raise IOError(
"PSFEx: Current implementation requires POLNGRP == 1, got %d" %
pol_ngrp)
# Which group each item is in. We require group 1.
pol_group1 = hdu.header['POLGRP1']
if pol_group1 != 1:
raise IOError("PSFEx: Expected POLGRP1 == 1, got %s" % pol_group1)
pol_group2 = hdu.header['POLGRP2']
if pol_group2 != 1:
raise IOError("PSFEx: Expected POLGRP2 == 1, got %s" % pol_group2)
# The degree of the polynomial. E.g. POLDEG1 = 2 means the values will be:
# 1, x, x^2, y, xy, y^2
# If we start allowing POLNGRP > 1, there is a separate degree for each group.
pol_deg = hdu.header['POLDEG1']
# The number of axes in the basis object. We require this to be 3.
psf_naxis = hdu.header['PSFNAXIS']
if psf_naxis != 3:
raise IOError("PSFEx: Expected PSFNAXIS == 3, got %d" % psfnaxis)
# The first two axes are the image size of the PSF postage stamp.
psf_axis1 = hdu.header['PSFAXIS1']
psf_axis2 = hdu.header['PSFAXIS2']
# The third axis is the direction of the polynomial interpolation. So it should
# be equal to (d+1)(d+2)/2.
psf_axis3 = hdu.header['PSFAXIS3']
if psf_axis3 != ((pol_deg + 1) * (pol_deg + 2)) / 2:
raise IOError("PSFEx: POLDEG and PSFAXIS3 disagree")
# This is the PSF "sample size". Again, from Emmanuel:
#
# PSF_SAMP is the sampling step of the PSF. PSF_SAMP=0.5 means that the PSF model has
# two samples per original image pixel (superresolution, so in automatic mode it is a
# sign that the original images were undersampled)
#
# In other words, it can be thought of as a unit conversion:
# "image pixels" / "psfex pixels"
# So 1 image pixel = (1/psf_samp) psfex pixels.
psf_samp = hdu.header['PSF_SAMP']
# The basis object is a data cube (assuming PSFNAXIS==3)
# Note: older pyfits versions don't get the shape right.
# For newer pyfits versions the reshape command should be a no op.
basis = hdu.data.field('PSF_MASK')[0].reshape(psf_axis3, psf_axis2,
psf_axis1)
# Make sure this turned out right.
if basis.shape[0] != psf_axis3:
raise IOError("PSFEx: PSFAXIS3 disagrees with actual basis size")
if basis.shape[1] != psf_axis2:
raise IOError("PSFEx: PSFAXIS2 disagrees with actual basis size")
if basis.shape[2] != psf_axis1:
raise IOError("PSFEx: PSFAXIS1 disagrees with actual basis size")
# Save some of these values for use in building the interpolated images
self.basis = basis
self.fit_order = pol_deg
self.fit_size = psf_axis3
self.x_zero = pol_zero1
self.y_zero = pol_zero2
self.x_scale = pol_scal1
self.y_scale = pol_scal2
self.sample_scale = psf_samp
def __call__(self, file, file_compress):
if not file_compress:
hdus = pyfits.open(file, 'readonly')
return hdus, None
elif file_compress == 'gzip':
import gzip
if self.gzip_in_mem:
try:
fin = gzip.GzipFile(file, 'rb')
hdus = pyfits.open(fin, 'readonly')
# Sometimes this doesn't work. The symptoms may be that this raises an
# exception, or possibly the hdus list comes back empty, in which case the
# next line will raise an exception.
hdu = hdus[0]
# pyfits doesn't actually read the file yet, so we can't close fin here.
# Need to pass it back to the caller and let them close it when they are
# done with hdus.
return hdus, fin
except:
# Mark that we can't do this the efficient way so next time (and afterward)
# it will use the below code instead.
self.gzip_in_mem = False
return self(file,file_compress)
else:
try:
# This usually works, although pyfits internally uses a temporary file,
# which is why we prefer the above code if it works.
hdus = pyfits.open(file, 'readonly')
return hdus, None
except:
# But just in case, here is an implementation that should always work.
fin = gzip.GzipFile(file, 'rb')
data = fin.read()
tmp = file + '.tmp'
# It would be pretty odd for this filename to already exist, but just in case...
while os.path.isfile(tmp):
tmp = tmp + '.tmp'
with open(tmp,"w") as tmpout:
tmpout.write(data)
hdus = pyfits.open(tmp)
return hdus, tmp
elif file_compress == 'bzip2':
import bz2
if self.bz2_in_mem:
try:
# This normally works. But it might not on old versions of pyfits.
fin = bz2.BZ2File(file, 'rb')
hdus = pyfits.open(fin, 'readonly')
hdu = hdus[0]
return hdus, fin
except:
# Mark that we can't do this the efficient way so next time (and afterward)
# it will use the below code instead.
self.bz2_in_mem = False
return self(file,file_compress)
else:
fin = bz2.BZ2File(file, 'rb')
data = fin.read()
tmp = file + '.tmp'
# It would be pretty odd for this filename to already exist, but just in case...
while os.path.isfile(tmp):
tmp = tmp + '.tmp'
with open(tmp,"w") as tmpout:
tmpout.write(data)
hdus = pyfits.open(tmp)
return hdus, tmp
else:
raise ValueError("Unknown file_compression")
def test_read():
"""Test reading a FitsHeader from an existing FITS file
"""
import time
t1 = time.time()
def check_tpv(header):
"""Check that the header object has correct values from the tpv.fits file
"""
# Check using a few different access methods.
assert header['TIME-OBS'] == '04:28:14.105'
assert header.get('FILTER') == 'I'
assert header['AIRMASS'] == 1.185
assert len(header) == 215
assert 'ADC' in header
assert ('FILPOS',6) in header.items()
assert ('FILPOS',6) in header.iteritems()
assert 'OBSERVAT' in header.keys()
assert 'OBSERVAT' in header.iterkeys()
assert 54384.18627436 in header.values() # MJD-OBS
assert 54384.18627436 in header.itervalues()
file_name = 'tpv.fits'
dir = 'fits_files'
# First option: give a file_name
header = galsim.FitsHeader(file_name=os.path.join(dir,file_name))
check_tpv(header)
do_pickle(header)
# Let the FitsHeader init handle the dir
header = galsim.FitsHeader(file_name=file_name, dir=dir)
check_tpv(header)
do_pickle(header)
# If the first arg is a str, then it should be interpreted as a file name
header = galsim.FitsHeader(file_name, dir=dir)
check_tpv(header)
# If you pass in a pyfits hdulist, that should also work
hdu_list = pyfits.open(os.path.join(dir,file_name))
header = galsim.FitsHeader(hdu_list=hdu_list)
check_tpv(header)
do_pickle(header)
# Can explicitly give an hdu number to use. In this case, there is only 1, so need to use 0.
header = galsim.FitsHeader(hdu_list=hdu_list, hdu=0)
check_tpv(header)
do_pickle(header)
# If you pass in a pyfits Header object, that should also work
header = galsim.FitsHeader(header=hdu_list[0].header)
check_tpv(header)
do_pickle(header)
# The header is the first parameter, so don't need to name it.
header = galsim.FitsHeader(hdu_list[0].header)
check_tpv(header)
# Remove an item from the header
# Start with file_name constructor, to test that the repr is changed by the edit.
header = galsim.FitsHeader(file_name=os.path.join(dir,file_name))
del header['AIRMASS']
assert 'AIRMASS' not in header
assert len(header) == 214
do_pickle(header)
# Should be able to get with a default value if the key is not present
assert header.get('AIRMASS', 2.0) == 2.0
# key should still not be in the header
assert 'AIRMASS' not in header
assert len(header) == 214
# Add items to a header
header['AIRMASS'] = 2
assert header.get('AIRMASS') == 2
# Overwrite an existing value
header['AIRMASS'] = 1.7
assert header.get('AIRMASS') == 1.7
# Set with a comment field
header['AIRMASS'] = (1.9, 'The airmass of the observation')
assert header.get('AIRMASS') == 1.9
# Update with a dict
d = { 'AIRMASS' : 1.185 }
header.update(d)
assert header.get('AIRMASS') == 1.185
# We are essentially back to where we started, except the len won't be right.
# Deleting a key removed an item, but setting it overwrote a blank item.
# But if we add back another one of these, we should be back to the original values.
header.append('','', useblanks=False)
check_tpv(header)
do_pickle(header)
# Clear all values
header.clear()
assert 'AIRMASS' not in header
assert 'FILTER' not in header
assert len(header) == 0
do_pickle(header)
t2 = time.time()
print 'time for %s = %.2f'%(funcname(),t2-t1)
