def applyNpolCorr(fname, unpolcorr):
"""
Determines whether non-polynomial distortion lookup tables should be added
as extensions to the science file based on the 'NPOLFILE' keyword in the
primary header and NPOLEXT kw in the first extension.
This is a default correction and will always run in the pipeline.
The file used to generate the extensions is
recorded in the NPOLEXT keyword in the first science extension.
If 'NPOLFILE' in the primary header is different from 'NPOLEXT' in the
extension header and the file exists on disk and is a 'new type' npolfile,
then the lookup tables will be updated as 'WCSDVARR' extensions.
Parameters
----------
fname : `~astropy.io.fits.HDUList`
Input FITS file object.
"""
applyNPOLCorr = True
try:
# get NPOLFILE kw from primary header
fnpol0 = fname[0].header['NPOLFILE']
if fnpol0 == 'N/A':
utils.remove_distortion(fname, "NPOLFILE")
return False
fnpol0 = fileutil.osfn(fnpol0)
if not fileutil.findFile(fnpol0):
msg = '"NPOLFILE" exists in primary header but file {0} not found.'
'Non-polynomial distortion correction will not be applied.'.format(fnpol0)
logger.critical(msg)
raise IOError("NPOLFILE {0} not found".format(fnpol0))
try:
# get NPOLEXT kw from first extension header
fnpol1 = fname[1].header['NPOLEXT']
fnpol1 = fileutil.osfn(fnpol1)
if fnpol1 and fileutil.findFile(fnpol1):
if fnpol0 != fnpol1:
applyNPOLCorr = True
else:
msg = """\n\tNPOLEXT with the same value as NPOLFILE found in first extension.
NPOL correction will not be applied."""
logger.info(msg)
applyNPOLCorr = False
else:
# npl file defined in first extension may not be found
# but if a valid kw exists in the primary header, non-polynomial
# distortion correction should be applied.
applyNPOLCorr = True
except KeyError:
# the case of "NPOLFILE" kw present in primary header but "NPOLEXT" missing
# in first extension header
applyNPOLCorr = True
except KeyError:
logger.info('\n\t"NPOLFILE" keyword not found in primary header')
applyNPOLCorr = False
return applyNPOLCorr
if isOldStyleDGEO(fname, fnpol0):
applyNPOLCorr = False
return (applyNPOLCorr and unpolcorr)
def applyNpolCorr(fname, unpolcorr):
"""
Determines whether non-polynomial distortion lookup tables should be added
as extensions to the science file based on the 'NPOLFILE' keyword in the
primary header and NPOLEXT kw in the first extension.
This is a default correction and will always run in the pipeline.
The file used to generate the extensions is
recorded in the NPOLEXT keyword in the first science extension.
If 'NPOLFILE' in the primary header is different from 'NPOLEXT' in the
extension header and the file exists on disk and is a 'new type' npolfile,
then the lookup tables will be updated as 'WCSDVARR' extensions.
"""
applyNPOLCorr = True
try:
# get NPOLFILE kw from primary header
fnpol0 = fits.getval(fname, 'NPOLFILE')
if fnpol0 == 'N/A':
utils.remove_distortion(fname, "NPOLFILE")
return False
fnpol0 = fileutil.osfn(fnpol0)
if not fileutil.findFile(fnpol0):
msg = """\n\tKw "NPOLFILE" exists in primary header but file %s not found
Non-polynomial distortion correction will not be applied\n
""" % fnpol0
logger.critical(msg)
raise IOError("NPOLFILE {0} not found".format(fnpol0))
try:
# get NPOLEXT kw from first extension header
fnpol1 = fits.getval(fname, 'NPOLEXT', ext=1)
fnpol1 = fileutil.osfn(fnpol1)
if fnpol1 and fileutil.findFile(fnpol1):
if fnpol0 != fnpol1:
applyNPOLCorr = True
else:
msg = """\n\tNPOLEXT with the same value as NPOLFILE found in first extension.
NPOL correction will not be applied."""
logger.info(msg)
applyNPOLCorr = False
else:
# npl file defined in first extension may not be found
# but if a valid kw exists in the primary header, non-polynomial
#distortion correction should be applied.
applyNPOLCorr = True
except KeyError:
# the case of "NPOLFILE" kw present in primary header but "NPOLEXT" missing
# in first extension header
applyNPOLCorr = True
except KeyError:
logger.info('\n\t"NPOLFILE" keyword not found in primary header')
applyNPOLCorr = False
return applyNPOLCorr
if isOldStyleDGEO(fname, fnpol0):
applyNPOLCorr = False
return (applyNPOLCorr and unpolcorr)
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
toclose = False
if isinstance(f, str):
f = fits.open(f)
toclose = True
try:
pav3 = f[0].header['PA_V3']
except KeyError:
rootname = f[0].header['ROOTNAME']
sptfile = rootname + '_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
f[0].header['PA_V3'] = pav3
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
if toclose:
f.close()
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t', f)
return removed_files
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
toclose = False
if isinstance(f, str):
f = fits.open(f)
toclose = True
try:
pav3 = f[0].header['PA_V3']
except KeyError:
rootname = f[0].header['ROOTNAME']
sptfile = rootname+'_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
f[0].header['PA_V3'] = pav3
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f.filename() or "")
if toclose:
f.close()
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t',f)
return removed_files
def _getExposure(img, output, frame, idckey):
# setup Exposure object with distortion model and WCS info
if frame == 'input':
if img.find('[') < 0:
img += '[sci]'
_val = fileutil.getKeyword(img, 'CD1_1')
if _val == None:
img[:-7] += '[0]'
if fileutil.getKeyword(img, 'CD1_1') == None:
print("Input %s not valid!" % img)
raise Exception
_exp = pydrizzle.Exposure(img, idckey)
_wcs = _exp.geometry.wcslin
else:
# Use output frame for converting pixel shifts
# to units of arcseconds...
# Make sure we have a recognized file:
if not fileutil.findFile(output):
if fileutil.findFile(output + '.fits'):
output += '.fits'
else:
print("Can NOT find output file %s!" % output)
raise Exception
if output.find('[') < 0:
output += '[0]'
_val = fileutil.getKeyword(output, 'CD1_1')
if _val == None:
output[:-3] += '[sci,1]'
if fileutil.getKeyword(output, 'CD1_1') == None:
print("Output %s not valid!" % output)
raise Exception
_exp = pydrizzle.Exposure(output, idckey)
_wcs = _exp.geometry.wcs
return _wcs
def applyD2ImCorr(fname, d2imcorr):
applyD2IMCorr = True
try:
# get D2IMFILE kw from primary header
fd2im0 = fits.getval(fname, 'D2IMFILE')
if fd2im0 == 'N/A':
utils.remove_distortion(fname, "D2IMFILE")
return False
fd2im0 = fileutil.osfn(fd2im0)
if not fileutil.findFile(fd2im0):
msg = """\n\tKw D2IMFILE exists in primary header but file %s not found\n
Detector to image correction will not be applied\n""" % fd2im0
logger.critical(msg)
print(msg)
raise IOError("D2IMFILE {0} not found".format(fd2im0))
try:
# get D2IMEXT kw from first extension header
fd2imext = fits.getval(fname, 'D2IMEXT', ext=1)
fd2imext = fileutil.osfn(fd2imext)
if fd2imext and fileutil.findFile(fd2imext):
if fd2im0 != fd2imext:
applyD2IMCorr = True
else:
applyD2IMCorr = False
else:
# D2IM file defined in first extension may not be found
# but if a valid kw exists in the primary header,
# detector to image correction should be applied.
applyD2IMCorr = True
except KeyError:
# the case of D2IMFILE kw present in primary header but D2IMEXT missing
# in first extension header
applyD2IMCorr = True
except KeyError:
print('D2IMFILE keyword not found in primary header')
applyD2IMCorr = False
return applyD2IMCorr
def applyD2ImCorr(fname, d2imcorr):
applyD2IMCorr = True
try:
# get D2IMFILE kw from primary header
fd2im0 = fits.getval(fname, 'D2IMFILE')
if fd2im0 == 'N/A':
utils.remove_distortion(fname, "D2IMFILE")
return False
fd2im0 = fileutil.osfn(fd2im0)
if not fileutil.findFile(fd2im0):
msg = """\n\tKw D2IMFILE exists in primary header but file %s not found\n
Detector to image correction will not be applied\n""" % fd2im0
logger.critical(msg)
print(msg)
raise IOError("D2IMFILE {0} not found".format(fd2im0))
try:
# get D2IMEXT kw from first extension header
fd2imext = fits.getval(fname, 'D2IMEXT', ext=1)
fd2imext = fileutil.osfn(fd2imext)
if fd2imext and fileutil.findFile(fd2imext):
if fd2im0 != fd2imext:
applyD2IMCorr = True
else:
applyD2IMCorr = False
else:
# D2IM file defined in first extension may not be found
# but if a valid kw exists in the primary header,
# detector to image correction should be applied.
applyD2IMCorr = True
except KeyError:
# the case of D2IMFILE kw present in primary header but D2IMEXT missing
# in first extension header
applyD2IMCorr = True
except KeyError:
print('D2IMFILE keyword not found in primary header')
applyD2IMCorr = False
return applyD2IMCorr
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
try:
pav3 = fits.getval(f, 'PA_V3')
except KeyError:
rootname = fits.getval(f, 'ROOTNAME')
sptfile = rootname+'_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
fits.setval(f, 'PA_V3', value=pav3)
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t',f)
return removed_files
def checkPA_V3(fnames):
removed_files = []
for f in fnames:
try:
pav3 = fits.getval(f, 'PA_V3')
except KeyError:
rootname = fits.getval(f, 'ROOTNAME')
sptfile = rootname + '_spt.fits'
if fileutil.findFile(sptfile):
try:
pav3 = fits.getval(sptfile, 'PA_V3')
except KeyError:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
fits.setval(f, 'PA_V3', value=pav3)
else:
print("Warning: Files without keyword PA_V3 detected")
removed_files.append(f)
if removed_files != []:
print("Warning: Removing the following files from input list")
for f in removed_files:
print('\t', f)
return removed_files
def apply_d2im_correction(fname, d2imcorr):
"""
Logic to decide whether to apply the D2IM correction.
Parameters
----------
fname : `~astropy.io.fits.HDUList` or str
Input FITS science file object.
d2imcorr : bool
Flag indicating if D2IM is should be enabled if allowed.
Return
------
applyD2IMCorr : bool
Flag whether to apply the correction.
The D2IM correction is applied to a science file if it is in the
allowed corrections for the instrument. The name of the file
with the correction is saved in the ``D2IMFILE`` keyword in the
primary header. When the correction is applied the name of the
file is saved in the ``D2IMEXT`` keyword in the 1st extension header.
"""
fname, toclose = _toclose(fname)
applyD2IMCorr = True
if not d2imcorr:
logger.info("D2IM correction not requested - not applying it.")
return False
# get D2IMFILE kw from primary header
try:
fd2im0 = fname[0].header['D2IMFILE']
except KeyError:
logger.info("D2IMFILE keyword is missing - D2IM correction will not be applied.")
return False
if fd2im0 == 'N/A':
utils.remove_distortion(fname, "D2IMFILE")
return False
fd2im0 = fileutil.osfn(fd2im0)
if not fileutil.findFile(fd2im0):
message = "D2IMFILE {0} not found.".format(fd2im0)
logger.critical(message)
raise IOError(message)
try:
# get D2IMEXT kw from first extension header
fd2imext = fname[1].header['D2IMEXT']
except KeyError:
# the case of D2IMFILE kw present in primary header but D2IMEXT missing
# in first extension header
return True
fd2imext = fileutil.osfn(fd2imext)
if fd2imext and fileutil.findFile(fd2imext):
if fd2im0 != fd2imext:
applyD2IMCorr = True
else:
applyD2IMCorr = False
else:
# D2IM file defined in first extension may not be found
# but if a valid kw exists in the primary header,
# detector to image correction should be applied.
applyD2IMCorr = True
if toclose:
fname.close()
return applyD2IMCorr
def writeFITS(self,
template,
sciarr,
whtarr,
ctxarr=None,
versions=None,
overwrite=yes,
blend=True,
virtual=False,
rules_file=None,
logfile=None):
"""
Generate PyFITS objects for each output extension
using the file given by 'template' for populating
headers.
The arrays will have the size specified by 'shape'.
"""
if not isinstance(template, list):
template = [template]
if fileutil.findFile(self.output):
if overwrite:
log.info('Deleting previous output product: %s' % self.output)
fileutil.removeFile(self.output)
else:
log.warning('Output file %s already exists and overwrite not '
'specified!' % self.output)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
# initialize output value for this method
outputFITS = {}
# Default value for NEXTEND when 'build'== True
nextend = 3
if not self.build:
nextend = 0
if self.outweight:
if overwrite:
if fileutil.findFile(self.outweight):
log.info('Deleting previous output WHT product: %s' %
self.outweight)
fileutil.removeFile(self.outweight)
else:
log.warning('Output file %s already exists and overwrite '
'not specified!' % self.outweight)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
if self.outcontext:
if overwrite:
if fileutil.findFile(self.outcontext):
log.info('Deleting previous output CTX product: %s' %
self.outcontext)
fileutil.removeFile(self.outcontext)
else:
log.warning('Output file %s already exists and overwrite '
'not specified!' % self.outcontext)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
# Get default headers from multi-extension FITS file
# If only writing out single drizzle product, blending needs to be
# forced off as there is only 1 input to report, no blending needed
if self.single:
blend = False
# If input data is not in MEF FITS format, it will return 'None'
# and those headers will have to be generated from drizzle output
# file FITS headers.
# NOTE: These are HEADER objects, not HDUs
self.fullhdrs, intab = getTemplates(template,
blend=False,
rules_file=rules_file)
newhdrs, newtab = getTemplates(template,
blend=blend,
rules_file=rules_file)
if newtab is not None: nextend += 1 # account for new table extn
prihdr = newhdrs[0]
scihdr = newhdrs[1]
errhdr = newhdrs[2]
dqhdr = newhdrs[3]
# Setup primary header as an HDU ready for appending to output FITS file
prihdu = fits.PrimaryHDU(header=prihdr, data=None)
# Start by updating PRIMARY header keywords...
prihdu.header.set('EXTEND', value=True, after='NAXIS')
prihdu.header['NEXTEND'] = nextend
prihdu.header['FILENAME'] = self.output
prihdu.header['PROD_VER'] = 'DrizzlePac {}'.format(__version__)
prihdu.header['DRIZPARS'] = (logfile, "Logfile for processing")
# Update the ROOTNAME with the new value as well
_indx = self.output.find('_drz')
if _indx < 0:
rootname_val = self.output
else:
rootname_val = self.output[:_indx]
prihdu.header['ROOTNAME'] = rootname_val
# Get the total exposure time for the image
# If not calculated by PyDrizzle and passed through
# the pardict, then leave value from the template image.
if self.texptime:
prihdu.header['EXPTIME'] = self.texptime
prihdu.header.set('TEXPTIME', value=self.texptime, after='EXPTIME')
prihdu.header['EXPSTART'] = self.expstart
prihdu.header['EXPEND'] = self.expend
# Update ASN_MTYPE to reflect the fact that this is a product
# Currently hard-wired to always output 'PROD-DTH' as MTYPE
prihdu.header['ASN_MTYP'] = 'PROD-DTH'
# Update DITHCORR calibration keyword if present
# Remove when we can modify FITS headers in place...
if 'DRIZCORR' in prihdu.header:
prihdu.header['DRIZCORR'] = 'COMPLETE'
if 'DITHCORR' in prihdu.header:
prihdu.header['DITHCORR'] = 'COMPLETE'
prihdu.header['NDRIZIM'] = (len(self.parlist),
'Drizzle, No. images drizzled onto output')
# Only a subset of these keywords makes sense for the new WCS based
# transformations. They need to be reviewed to decide what to keep
# and what to leave out.
if not self.blot:
self.addDrizKeywords(prihdu.header, versions)
if scihdr:
scihdr.pop('OBJECT', None)
if 'CCDCHIP' in scihdr: scihdr['CCDCHIP'] = '-999'
if 'NCOMBINE' in scihdr:
scihdr['NCOMBINE'] = self.parlist[0]['nimages']
# If BUNIT keyword was found and reset, then
bunit_last_kw = self.find_kwupdate_location(scihdr, 'bunit')
if self.bunit is not None:
comment_str = "Units of science product"
if self.bunit.lower()[:5] == 'count':
comment_str = "counts * gain = electrons"
scihdr.set('BUNIT',
value=self.bunit,
comment=comment_str,
after=bunit_last_kw)
else:
# check to see whether to update already present BUNIT comment
if 'bunit' in scihdr and scihdr['bunit'].lower(
)[:5] == 'count':
comment_str = "counts * gain = electrons"
scihdr.set('BUNIT',
value=scihdr['bunit'],
comment=comment_str,
after=bunit_last_kw)
# Add WCS keywords to SCI header
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(scihdr, 'CD1_1')
addWCSKeywords(self.wcs,
scihdr,
blot=self.blot,
single=self.single,
after=pre_wcs_kw)
# Recompute this after removing distortion kws
pre_wcs_kw = self.find_kwupdate_location(scihdr, 'CD1_1')
##########
# Now, build the output file
##########
if self.build:
print('-Generating multi-extension output file: ', self.output)
fo = fits.HDUList()
# Add primary header to output file...
fo.append(prihdu)
if self.single and self.compress:
hdu = fits.CompImageHDU(data=sciarr,
header=scihdr,
name=EXTLIST[0])
else:
hdu = fits.ImageHDU(data=sciarr,
header=scihdr,
name=EXTLIST[0])
last_kw = self.find_kwupdate_location(scihdr, 'EXTNAME')
hdu.header.set('EXTNAME', value='SCI', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
fo.append(hdu)
# Build WHT extension here, if requested...
if errhdr:
errhdr['CCDCHIP'] = '-999'
if self.single and self.compress:
hdu = fits.CompImageHDU(data=whtarr,
header=errhdr,
name=EXTLIST[1])
else:
hdu = fits.ImageHDU(data=whtarr,
header=errhdr,
name=EXTLIST[1])
last_kw = self.find_kwupdate_location(errhdr, 'EXTNAME')
hdu.header.set('EXTNAME', value='WHT', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header, 'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,
hdu.header,
blot=self.blot,
single=self.single,
after=pre_wcs_kw)
fo.append(hdu)
# Build CTX extension here
# If there is only 1 plane, write it out as a 2-D extension
if self.outcontext:
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
else:
_ctxarr = None
if self.single and self.compress:
hdu = fits.CompImageHDU(data=_ctxarr,
header=dqhdr,
name=EXTLIST[2])
else:
hdu = fits.ImageHDU(data=_ctxarr,
header=dqhdr,
name=EXTLIST[2])
last_kw = self.find_kwupdate_location(dqhdr, 'EXTNAME')
hdu.header.set('EXTNAME', value='CTX', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header, 'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,
hdu.header,
blot=self.blot,
single=self.single,
after=pre_wcs_kw)
fo.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fo, [1])
# add table of combined header keyword values to FITS file
if newtab is not None:
fo.append(newtab)
if not virtual:
print('Writing out to disk:', self.output)
# write out file to disk
fo.writeto(self.output)
fo.close()
del fo, hdu
fo = None
# End 'if not virtual'
outputFITS[self.output] = fo
else:
print('-Generating simple FITS output: %s' % self.outdata)
fo = fits.HDUList()
hdu_header = prihdu.header.copy()
del hdu_header['nextend']
# Append remaining unique header keywords from template DQ
# header to Primary header...
if scihdr:
for _card in scihdr.cards:
if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu_header:
hdu_header.append(_card)
hdu_header['filename'] = self.outdata
if self.compress:
hdu = fits.CompImageHDU(data=sciarr, header=hdu_header)
wcs_ext = [1]
else:
hdu = fits.PrimaryHDU(data=sciarr, header=hdu_header)
wcs_ext = [0]
# explicitly set EXTEND to FALSE for simple FITS files.
dim = len(sciarr.shape)
hdu.header.set('extend', value=False, after='NAXIS%s' % dim)
# explicitly remove EXTNAME, EXTVER from header
# since this header may have been used
# to create a CompImageHDU instance instead of a PrimaryHDU instance
for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']:
hdu.header.pop(kw, None)
hdu.header.set('filetype',
'SCI',
before='TELESCOP',
comment='Type of data in array')
# Add primary header to output file...
fo.append(hdu)
if not self.blot:
# remove all alternate WCS solutions from headers of this product
logutil.logging.disable(logutil.logging.INFO)
wcs_functions.removeAllAltWCS(fo, wcs_ext)
logutil.logging.disable(logutil.logging.NOTSET)
# add table of combined header keyword values to FITS file
if newtab is not None:
fo.append(newtab)
if not virtual or "single_sci" in self.outdata:
print('Writing out image to disk:', self.outdata)
# write out file to disk
fo.writeto(self.outdata, overwrite=True)
del hdu
if "single_sci" not in self.outdata:
del fo
fo = None
# End 'if not virtual'
outputFITS[self.outdata] = fo
if self.outweight and whtarr is not None:
# We need to build new PyFITS objects for each WHT array
fwht = fits.HDUList()
if errhdr:
errhdr['CCDCHIP'] = '-999'
if self.compress:
hdu = fits.CompImageHDU(data=whtarr, header=prihdu.header)
else:
hdu = fits.PrimaryHDU(data=whtarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if errhdr:
for _card in errhdr.cards:
if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu.header:
hdu.header.append(_card)
hdu.header['filename'] = self.outweight
hdu.header['CCDCHIP'] = '-999'
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(
hdu.header, 'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,
hdu.header,
blot=self.blot,
single=self.single,
after=pre_wcs_kw)
# explicitly remove EXTNAME, EXTVER from header
# since this header may have been used
# to create a CompImageHDU instance instead of a PrimaryHDU instance
for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']:
hdu.header.pop(kw, None)
hdu.header.set('filetype',
'WHT',
before='TELESCOP',
comment='Type of data in array')
# Add primary header to output file...
fwht.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fwht, wcs_ext)
if not virtual:
print('Writing out image to disk:', self.outweight)
fwht.writeto(self.outweight, overwrite=True)
del fwht, hdu
fwht = None
# End 'if not virtual'
outputFITS[self.outweight] = fwht
# If a context image was specified, build a PyFITS object
# for it as well...
if self.outcontext and ctxarr is not None:
fctx = fits.HDUList()
# If there is only 1 plane, write it out as a 2-D extension
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
if self.compress:
hdu = fits.CompImageHDU(data=_ctxarr, header=prihdu.header)
else:
hdu = fits.PrimaryHDU(data=_ctxarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if dqhdr:
for _card in dqhdr.cards:
if ((_card.keyword not in RESERVED_KEYS)
and _card.keyword not in hdu.header):
hdu.header.append(_card)
hdu.header['filename'] = self.outcontext
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(
hdu.header, 'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,
hdu.header,
blot=self.blot,
single=self.single,
after=pre_wcs_kw)
# explicitly remove EXTNAME, EXTVER from header
# since this header may have been used
# to create a CompImageHDU instance instead of a PrimaryHDU instance
for kw in ['EXTNAME', 'EXTVER', 'PCOUNT', 'GCOUNT']:
hdu.header.pop(kw, None)
hdu.header.set('filetype',
'CTX',
before='TELESCOP',
comment='Type of data in array')
fctx.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fctx, wcs_ext)
if not virtual:
print('Writing out image to disk:', self.outcontext)
fctx.writeto(self.outcontext, overwrite=True)
del fctx, hdu
fctx = None
# End 'if not virtual'
outputFITS[self.outcontext] = fctx
return outputFITS
def run_driz(imageObjectList,output_wcs,paramDict,single,build,wcsmap=None):
""" Perform drizzle operation on input to create output.
The input parameters originally was a list
of dictionaries, one for each input, that matches the
primary parameters for an ``IRAF`` `drizzle` task.
This method would then loop over all the entries in the
list and run `drizzle` for each entry.
Parameters required for input in paramDict:
build,single,units,wt_scl,pixfrac,kernel,fillval,
rot,scale,xsh,ysh,blotnx,blotny,outnx,outny,data
"""
# Insure that input imageObject is a list
if not isinstance(imageObjectList, list):
imageObjectList = [imageObjectList]
#
# Setup the versions info dictionary for output to PRIMARY header
# The keys will be used as the name reported in the header, as-is
#
_versions = {'AstroDrizzle':__version__,
'PyFITS':util.__fits_version__,
'Numpy':util.__numpy_version__}
# Set sub-sampling rate for drizzling
#stepsize = 2.0
log.info(' **Using sub-sampling value of %s for kernel %s' %
(paramDict['stepsize'], paramDict['kernel']))
maskval = interpret_maskval(paramDict)
outwcs = copy.deepcopy(output_wcs)
# Check for existance of output file.
if single == False and build == True and fileutil.findFile(
imageObjectList[0].outputNames['outFinal']):
log.info('Removing previous output product...')
os.remove(imageObjectList[0].outputNames['outFinal'])
# print out parameters being used for drizzling
log.info("Running Drizzle to create output frame with WCS of: ")
output_wcs.printwcs()
# Will we be running in parallel?
pool_size = util.get_pool_size(paramDict.get('num_cores'), len(imageObjectList))
will_parallel = single and pool_size > 1
if will_parallel:
log.info('Executing %d parallel workers' % pool_size)
else:
if single: # not yet an option for final drizzle, msg would confuse
log.info('Executing serially')
# Set parameters for each input and run drizzle on it here.
#
# Perform drizzling...
numctx = 0
for img in imageObjectList:
numctx += img._nmembers
_numctx = {'all':numctx}
# if single:
# Determine how many chips make up each single image
for img in imageObjectList:
for chip in img.returnAllChips(extname=img.scienceExt):
plsingle = chip.outputNames['outSingle']
if plsingle in _numctx: _numctx[plsingle] += 1
else: _numctx[plsingle] = 1
# Compute how many planes will be needed for the context image.
_nplanes = int((_numctx['all']-1) / 32) + 1
# For single drizzling or when context is turned off,
# minimize to 1 plane only...
if single or imageObjectList[0][1].outputNames['outContext'] in [None,'',' ']:
_nplanes = 1
#
# An image buffer needs to be setup for converting the input
# arrays (sci and wht) from FITS format to native format
# with respect to byteorder and byteswapping.
# This buffer should be reused for each input if possible.
#
_outsci = _outwht = _outctx = _hdrlist = None
if (not single) or \
( (single) and (not will_parallel) and (not imageObjectList[0].inmemory) ):
# Note there are four cases/combinations for single drizzle alone here:
# (not-inmem, serial), (not-inmem, parallel), (inmem, serial), (inmem, parallel)
#_outsci=np.zeros((output_wcs._naxis2,output_wcs._naxis1),dtype=np.float32)
_outsci=np.empty((output_wcs._naxis2,output_wcs._naxis1),dtype=np.float32)
_outsci.fill(maskval)
_outwht=np.zeros((output_wcs._naxis2,output_wcs._naxis1),dtype=np.float32)
# initialize context to 3-D array but only pass appropriate plane to drizzle as needed
_outctx=np.zeros((_nplanes,output_wcs._naxis2,output_wcs._naxis1),dtype=np.int32)
_hdrlist = []
# Keep track of how many chips have been processed
# For single case, this will determine when to close
# one product and open the next.
_chipIdx = 0
# Remember the name of the 1st image that goes into this particular product
# Insure that the header reports the proper values for the start of the
# exposure time used to make this; in particular, TIME-OBS and DATE-OBS.
template = None
#
# Work on each image
#
subprocs = []
for img in imageObjectList:
chiplist = img.returnAllChips(extname=img.scienceExt)
# How many inputs should go into this product?
num_in_prod = _numctx['all']
if single:
num_in_prod = _numctx[chiplist[0].outputNames['outSingle']]
# The name of the 1st image
fnames = []
for chip in chiplist:
fnames.append(chip.outputNames['data'])
if _chipIdx == 0:
template = fnames
else:
template.extend(fnames)
# Work each image, possibly in parallel
if will_parallel:
# use multiprocessing.Manager only if in parallel and in memory
if img.inmemory:
manager = multiprocessing.Manager()
dproxy = manager.dict(img.virtualOutputs) # copy & wrap it in proxy
img.virtualOutputs = dproxy
# parallelize run_driz_img (currently for separate drizzle only)
p = multiprocessing.Process(target=run_driz_img,
name='adrizzle.run_driz_img()', # for err msgs
args=(img,chiplist,output_wcs,outwcs,template,paramDict,
single,num_in_prod,build,_versions,_numctx,_nplanes,
_chipIdx,None,None,None,None,wcsmap))
subprocs.append(p)
else:
# serial run_driz_img run (either separate drizzle or final drizzle)
run_driz_img(img,chiplist,output_wcs,outwcs,template,paramDict,
single,num_in_prod,build,_versions,_numctx,_nplanes,
_chipIdx,_outsci,_outwht,_outctx,_hdrlist,wcsmap)
# Increment/reset master chip counter
_chipIdx += len(chiplist)
if _chipIdx == num_in_prod:
_chipIdx = 0
# do the join if we spawned tasks
if will_parallel:
mputil.launch_and_wait(subprocs, pool_size) # blocks till all done
del _outsci,_outwht,_outctx,_hdrlist
def buildShadowMaskImage(rootname,detnum,extnum,maskname,replace=yes,bitvalue=None,binned=1):
""" Builds mask image from WFPC2 shadow calibrations.
detnum - string value for 'DETECTOR' detector
"""
# insure detnum is a string
if type(detnum) != type(''):
detnum = repr(detnum)
_funcroot = '_func_Shadow_WF'
# build template shadow mask's filename
_mask = 'wfpc2_inmask'+detnum+'.fits'
"""
if rootname != None:
maskname = buildMaskName(fileutil.buildNewRootname(rootname),detnum)
else:
maskname = None
"""
# If an old version of the maskfile was present, remove it and rebuild it.
if fileutil.findFile(maskname) and replace:
fileutil.removeFile(maskname)
# Read in info from .c1h file to add flagged bad pixels to final mask
_indx = rootname.find('.c1h')
if _indx < 0: _indx = len(rootname)
if rootname.find('.fits') < 0:
_dqname = rootname[:_indx]+'.c1h'
else:
_dqname = rootname
_use_inmask = False
if fileutil.findFile(_dqname) != yes or bitvalue == None:
_use_inmask = True
# Check to see if file exists...
if _use_inmask and not fileutil.findFile(_mask):
# If not, create the file.
# This takes a long time to run, so it should be done
# only when absolutely necessary...
try:
_funcx = _funcroot+detnum+'x'
_funcy = _funcroot+detnum+'y'
_xarr = np.clip(np.fromfunction(eval(_funcx),(800,800)),0.0,1.0).astype(np.uint8)
_yarr = np.clip(np.fromfunction(eval(_funcy),(800,800)),0.0,1.0).astype(np.uint8)
maskarr = _xarr * _yarr
if binned !=1:
print('in buildmask', binned)
bmaskarr = maskarr[::2,::2]
bmaskarr *= maskarr[1::2,::2]
bmaskarr *= maskarr[::2,1::2]
bmaskarr *= maskarr[1::2,1::2]
maskarr = bmaskarr.copy()
del bmaskarr
#Write out the mask file as simple FITS file
fmask = pyfits.open(_mask,'append')
maskhdu = pyfits.PrimaryHDU(data=maskarr)
fmask.append(maskhdu)
#Close files
fmask.close()
del fmask
except:
return None
# Check for existance of input .c1h file for use in making inmask file
if fileutil.findFile(_dqname) != yes:
print('DQ file ',_dqname,' NOT found...')
print('Copying ',_mask,'to ',maskname,' as input mask file.')
# Now, copy template mask file to output file, if necessary
fileutil.copyFile(_mask,maskname,replace=yes)
elif bitvalue == None:
# If bitvalue was not set, then do not use anything but shadow mask
fileutil.copyFile(_mask,maskname,replace=yes)
else:
#
# Build full mask based on .c1h and shadow mask
#
fdq = fileutil.openImage(_dqname)
#fsmask = pyfits.open(_mask,memmap=1,mode='readonly')
try:
# Read in DQ array from .c1h and from shadow mask files
dqarr = fdq[int(extnum)].data
#maskarr = fsmask[0].data
# Build mask array from DQ array
dqmaskarr = buildMask(dqarr,bitvalue)
#Write out the mask file as simple FITS file
fdqmask = pyfits.open(maskname,'append')
maskhdu = pyfits.PrimaryHDU(data=dqmaskarr)
fdqmask.append(maskhdu)
#Close files
fdqmask.close()
del fdqmask
fdq.close()
del fdq
except:
fdq.close()
del fdq
# Safeguard against leaving behind an incomplete file
if fileutil.findFile(maskname):
os.remove(maskname)
_errstr = "\nWarning: Problem creating DQMASK file for "+rootname+".\n"
#raise IOError, _errstr
print(_errstr)
return None
# Return the name of the mask image written out
return maskname
def writeFITS(self, template, sciarr, whtarr, ctxarr=None, versions=None, extlist=EXTLIST, overwrite=yes):
""" Generate PyFITS objects for each output extension
using the file given by 'template' for populating
headers.
The arrays will have the size specified by 'shape'.
"""
if fileutil.findFile(self.output):
if overwrite:
print('Deleting previous output product: ',self.output)
fileutil.removeFile(self.output)
else:
print('WARNING: Output file ',self.output,' already exists and overwrite not specified!')
print('Quitting... Please remove before resuming operations.')
raise IOError
# Default value for NEXTEND when 'build'== True
nextend = 3
if not self.build:
nextend = 0
if self.outweight:
if overwrite:
if fileutil.findFile(self.outweight):
print('Deleting previous output WHT product: ',self.outweight)
fileutil.removeFile(self.outweight)
else:
print('WARNING: Output file ',self.outweight,' already exists and overwrite not specified!')
print('Quitting... Please remove before resuming operations.')
raise IOError
if self.outcontext:
if overwrite:
if fileutil.findFile(self.outcontext):
print('Deleting previous output CTX product: ',self.outcontext)
fileutil.removeFile(self.outcontext)
else:
print('WARNING: Output file ',self.outcontext,' already exists and overwrite not specified!')
print('Quitting... Please remove before resuming operations.')
raise IOError
# Get default headers from multi-extension FITS file
# If input data is not in MEF FITS format, it will return 'None'
# and those headers will have to be generated from drizzle output
# file FITS headers.
# NOTE: These are HEADER objects, not HDUs
prihdr,scihdr,errhdr,dqhdr = getTemplates(template,extlist)
if prihdr == None:
# Use readgeis to get header for use as Primary header.
_indx = template.find('[')
if _indx < 0:
_data = template
else:
_data = template[:_indx]
fpri = readgeis.readgeis(_data)
prihdr = fpri[0].header.copy()
fpri.close()
del fpri
# Setup primary header as an HDU ready for appending to output FITS file
prihdu = pyfits.PrimaryHDU(header=prihdr,data=None)
# Start by updating PRIMARY header keywords...
prihdu.header.update('EXTEND',pyfits.TRUE,after='NAXIS')
prihdu.header.update('NEXTEND',nextend)
prihdu.header.update('FILENAME', self.output)
# Update the ROOTNAME with the new value as well
_indx = self.output.find('_drz')
if _indx < 0:
prihdu.header.update('ROOTNAME', self.output)
else:
prihdu.header.update('ROOTNAME', self.output[:_indx])
# Get the total exposure time for the image
# If not calculated by PyDrizzle and passed through
# the pardict, then leave value from the template image.
if self.texptime:
prihdu.header.update('EXPTIME', self.texptime)
prihdu.header.update('EXPSTART', self.expstart)
prihdu.header.update('EXPEND', self.expend)
#Update ASN_MTYPE to reflect the fact that this is a product
# Currently hard-wired to always output 'PROD-DTH' as MTYPE
prihdu.header.update('ASN_MTYP', 'PROD-DTH')
# Update DITHCORR calibration keyword if present
# Remove when we can modify FITS headers in place...
if 'DRIZCORR' in prihdu.header:
prihdu.header.update('DRIZCORR','COMPLETE')
if 'DITHCORR' in prihdu.header:
prihdu.header.update('DITHCORR','COMPLETE')
prihdu.header.update('NDRIZIM',len(self.parlist),
comment='Drizzle, No. images drizzled onto output')
self.addDrizKeywords(prihdu.header,versions)
if scihdr:
del scihdr['OBJECT']
if 'CCDCHIP' in scihdr: scihdr.update('CCDCHIP','-999')
if 'NCOMBINE' in scihdr:
scihdr.update('NCOMBINE', self.parlist[0]['nimages'])
# If BUNIT keyword was found and reset, then
if self.bunit is not None:
scihdr.update('BUNIT',self.bunit,comment="Units of science product")
if self.wcs:
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
scihdr.update('ORIENTAT',self.wcs.orient)
scihdr.update('CD1_1',self.wcs.cd11)
scihdr.update('CD1_2',self.wcs.cd12)
scihdr.update('CD2_1',self.wcs.cd21)
scihdr.update('CD2_2',self.wcs.cd22)
scihdr.update('CRVAL1',self.wcs.crval1)
scihdr.update('CRVAL2',self.wcs.crval2)
scihdr.update('CRPIX1',self.wcs.crpix1)
scihdr.update('CRPIX2',self.wcs.crpix2)
scihdr.update('VAFACTOR',1.0)
# Remove any reference to TDD correction
if 'TDDALPHA' in scihdr:
del scihdr['TDDALPHA']
del scihdr['TDDBETA']
# Remove '-SIP' from CTYPE for output product
if scihdr['ctype1'].find('SIP') > -1:
scihdr.update('ctype1', scihdr['ctype1'][:-4])
scihdr.update('ctype2',scihdr['ctype2'][:-4])
# Remove SIP coefficients from DRZ product
for k in scihdr.items():
if (k[0][:2] in ['A_','B_']) or (k[0][:3] in ['IDC','SCD'] and k[0] != 'IDCTAB') or \
(k[0][:6] in ['SCTYPE','SCRVAL','SNAXIS','SCRPIX']):
del scihdr[k[0]]
self.addPhotKeywords(scihdr,prihdu.header)
##########
# Now, build the output file
##########
if self.build:
print('-Generating multi-extension output file: ',self.output)
fo = pyfits.HDUList()
# Add primary header to output file...
fo.append(prihdu)
hdu = pyfits.ImageHDU(data=sciarr,header=scihdr,name=extlist[0])
fo.append(hdu)
# Build WHT extension here, if requested...
if errhdr:
errhdr.update('CCDCHIP','-999')
hdu = pyfits.ImageHDU(data=whtarr,header=errhdr,name=extlist[1])
hdu.header.update('EXTVER',1)
if self.wcs:
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
hdu.header.update('ORIENTAT',self.wcs.orient)
hdu.header.update('CD1_1',self.wcs.cd11)
hdu.header.update('CD1_2',self.wcs.cd12)
hdu.header.update('CD2_1',self.wcs.cd21)
hdu.header.update('CD2_2',self.wcs.cd22)
hdu.header.update('CRVAL1',self.wcs.crval1)
hdu.header.update('CRVAL2',self.wcs.crval2)
hdu.header.update('CRPIX1',self.wcs.crpix1)
hdu.header.update('CRPIX2',self.wcs.crpix2)
hdu.header.update('VAFACTOR',1.0)
fo.append(hdu)
# Build CTX extension here
# If there is only 1 plane, write it out as a 2-D extension
if self.outcontext:
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
else:
_ctxarr = None
hdu = pyfits.ImageHDU(data=_ctxarr,header=dqhdr,name=extlist[2])
hdu.header.update('EXTVER',1)
if self.wcs:
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
hdu.header.update('ORIENTAT',self.wcs.orient)
hdu.header.update('CD1_1',self.wcs.cd11)
hdu.header.update('CD1_2',self.wcs.cd12)
hdu.header.update('CD2_1',self.wcs.cd21)
hdu.header.update('CD2_2',self.wcs.cd22)
hdu.header.update('CRVAL1',self.wcs.crval1)
hdu.header.update('CRVAL2',self.wcs.crval2)
hdu.header.update('CRPIX1',self.wcs.crpix1)
hdu.header.update('CRPIX2',self.wcs.crpix2)
hdu.header.update('VAFACTOR',1.0)
fo.append(hdu)
fo.writeto(self.output)
fo.close()
del fo, hdu
else:
print('-Generating simple FITS output: ',self.outdata)
fo = pyfits.HDUList()
hdu = pyfits.PrimaryHDU(data=sciarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if scihdr:
for _card in scihdr.ascard:
if (_card.key not in RESERVED_KEYS and
_card.key not in hdu.header):
hdu.header.ascard.append(_card)
del hdu.header['PCOUNT']
del hdu.header['GCOUNT']
self.addPhotKeywords(hdu.header, prihdu.header)
hdu.header.update('filename', self.outdata)
# Add primary header to output file...
fo.append(hdu)
fo.writeto(self.outdata)
del fo,hdu
if self.outweight and whtarr != None:
# We need to build new PyFITS objects for each WHT array
fwht = pyfits.HDUList()
if errhdr:
errhdr.update('CCDCHIP','-999')
hdu = pyfits.PrimaryHDU(data=whtarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if errhdr:
for _card in errhdr.ascard:
if (_card.key not in RESERVED_KEYS and
_card.key not in hdu.header):
hdu.header.ascard.append(_card)
hdu.header.update('filename', self.outweight)
hdu.header.update('CCDCHIP','-999')
if self.wcs:
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
hdu.header.update('ORIENTAT',self.wcs.orient)
hdu.header.update('CD1_1',self.wcs.cd11)
hdu.header.update('CD1_2',self.wcs.cd12)
hdu.header.update('CD2_1',self.wcs.cd21)
hdu.header.update('CD2_2',self.wcs.cd22)
hdu.header.update('CRVAL1',self.wcs.crval1)
hdu.header.update('CRVAL2',self.wcs.crval2)
hdu.header.update('CRPIX1',self.wcs.crpix1)
hdu.header.update('CRPIX2',self.wcs.crpix2)
hdu.header.update('VAFACTOR',1.0)
# Add primary header to output file...
fwht.append(hdu)
fwht.writeto(self.outweight)
del fwht,hdu
# If a context image was specified, build a PyFITS object
# for it as well...
if self.outcontext and ctxarr != None:
fctx = pyfits.HDUList()
# If there is only 1 plane, write it out as a 2-D extension
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
hdu = pyfits.PrimaryHDU(data=_ctxarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if dqhdr:
for _card in dqhdr.ascard:
if (_card.key not in RESERVED_KEYS and
_card.key not in hdu.header):
hdu.header.ascard.append(_card)
hdu.header.update('filename', self.outcontext)
if self.wcs:
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
hdu.header.update('ORIENTAT',self.wcs.orient)
hdu.header.update('CD1_1',self.wcs.cd11)
hdu.header.update('CD1_2',self.wcs.cd12)
hdu.header.update('CD2_1',self.wcs.cd21)
hdu.header.update('CD2_2',self.wcs.cd22)
hdu.header.update('CRVAL1',self.wcs.crval1)
hdu.header.update('CRVAL2',self.wcs.crval2)
hdu.header.update('CRPIX1',self.wcs.crpix1)
hdu.header.update('CRPIX2',self.wcs.crpix2)
hdu.header.update('VAFACTOR',1.0)
fctx.append(hdu)
fctx.writeto(self.outcontext)
del fctx,hdu
def buildShadowMaskImage(dqfile,detnum,extnum,maskname,bitvalue=None,binned=1):
""" Builds mask image from WFPC2 shadow calibrations.
detnum - string value for 'DETECTOR' detector
"""
# insure detnum is a string
if type(detnum) != type(''):
detnum = repr(detnum)
_funcroot = '_func_Shadow_WF'
# build template shadow mask's filename
# If an old version of the maskfile was present, remove it and rebuild it.
if fileutil.findFile(maskname):
fileutil.removeFile(maskname)
_use_inmask = False
if fileutil.findFile(dqfile) != True or bitvalue == None:
_use_inmask = True
# Check for existance of input .c1h file for use in making inmask file
if fileutil.findFile(dqfile) != True or bitvalue is None:
#_mask = 'wfpc2_inmask'+detnum+'.fits'
_mask = maskname
# Check to see if file exists...
if _use_inmask and not fileutil.findFile(_mask):
# If not, create the file.
# This takes a long time to run, so it should be done
# only when absolutely necessary...
try:
_funcx = _funcroot+detnum+'x'
_funcy = _funcroot+detnum+'y'
_xarr = np.clip(np.fromfunction(eval(_funcx),(800,800)),0.0,1.0).astype(np.uint8)
_yarr = np.clip(np.fromfunction(eval(_funcy),(800,800)),0.0,1.0).astype(np.uint8)
maskarr = _xarr * _yarr
if binned !=1:
bmaskarr = maskarr[::2,::2]
bmaskarr *= maskarr[1::2,::2]
bmaskarr *= maskarr[::2,1::2]
bmaskarr *= maskarr[1::2,1::2]
maskarr = bmaskarr.copy()
del bmaskarr
#Write out the mask file as simple FITS file
fmask = fits.open(_mask,'append')
maskhdu = fits.PrimaryHDU(data=maskarr)
fmask.append(maskhdu)
#Close files
fmask.close()
del fmask
except:
return None
else:
#
# Build full mask based on .c1h and shadow mask
#
fdq = fileutil.openImage(dqfile)
#fsmask = fits.open(_mask,memmap=1,mode='readonly')
try:
# Read in DQ array from .c1h and from shadow mask files
dqarr = fdq[int(extnum)].data
#maskarr = fsmask[0].data
# Build mask array from DQ array
dqmaskarr = buildMask(dqarr,bitvalue)
#Write out the mask file as simple FITS file
fdqmask = fits.open(maskname,'append')
maskhdu = fits.PrimaryHDU(data=dqmaskarr)
fdqmask.append(maskhdu)
#Close files
fdqmask.close()
del fdqmask
fdq.close()
del fdq
except:
fdq.close()
del fdq
# Safeguard against leaving behind an incomplete file
if fileutil.findFile(maskname):
os.remove(maskname)
_errstr = "\nWarning: Problem creating DQMASK file for "+rootname+".\n"
#raise IOError, _errstr
print(_errstr)
return None
# Return the name of the mask image written out
return maskname
def buildMaskImage(rootname,bitvalue,output,extname='DQ',extver=1):
""" Builds mask image from rootname's DQ array
If there is no valid 'DQ' array in image, then return
an empty string.
"""
# If no bitvalue is set or rootname given, assume no mask is desired
# However, this name would be useful as the output mask from
# other processing, such as MultiDrizzle, so return it anyway.
#if bitvalue == None or rootname == None:
# return None
# build output name
maskname = output
# If an old version of the maskfile was present, remove it and rebuild it.
if fileutil.findFile(maskname):
fileutil.removeFile(maskname)
# Open input file with DQ array
fdq = fileutil.openImage(rootname,memmap=0,mode='readonly')
try:
_extn = fileutil.findExtname(fdq,extname,extver=extver)
if _extn != None:
# Read in DQ array
dqarr = fdq[_extn].data
else:
dqarr = None
# For the case where there is no DQ array,
# create a mask image of all ones.
if dqarr == None:
# We need to get the dimensions of the output DQ array
# Since the DQ array is non-existent, look for the SCI extension
_sci_extn = fileutil.findExtname(fdq,'SCI',extver=extver)
if _sci_extn != None:
_shape = fdq[_sci_extn].data.shape
dqarr = np.zeros(_shape,dtype=np.uint16)
else:
raise Exception
# Build mask array from DQ array
maskarr = buildMask(dqarr,bitvalue)
#Write out the mask file as simple FITS file
fmask = fits.open(maskname, 'append')
maskhdu = fits.PrimaryHDU(data = maskarr)
fmask.append(maskhdu)
#Close files
fmask.close()
del fmask
fdq.close()
del fdq
except:
fdq.close()
del fdq
# Safeguard against leaving behind an incomplete file
if fileutil.findFile(maskname):
os.remove(maskname)
_errstr = "\nWarning: Problem creating MASK file for "+rootname+".\n"
#raise IOError, _errstr
print(_errstr)
return None
# Return the name of the mask image written out
return maskname
def writeFITS(self, template, sciarr, whtarr, ctxarr=None,
versions=None, overwrite=yes, blend=True, virtual=False):
"""
Generate PyFITS objects for each output extension
using the file given by 'template' for populating
headers.
The arrays will have the size specified by 'shape'.
"""
if not isinstance(template, list):
template = [template]
if fileutil.findFile(self.output):
if overwrite:
log.info('Deleting previous output product: %s' % self.output)
fileutil.removeFile(self.output)
else:
log.warning('Output file %s already exists and overwrite not '
'specified!' % self.output)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
# initialize output value for this method
outputFITS = {}
# Default value for NEXTEND when 'build'== True
nextend = 3
if not self.build:
nextend = 0
if self.outweight:
if overwrite:
if fileutil.findFile(self.outweight):
log.info('Deleting previous output WHT product: %s' %
self.outweight)
fileutil.removeFile(self.outweight)
else:
log.warning('Output file %s already exists and overwrite '
'not specified!' % self.outweight)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
if self.outcontext:
if overwrite:
if fileutil.findFile(self.outcontext):
log.info('Deleting previous output CTX product: %s' %
self.outcontext)
fileutil.removeFile(self.outcontext)
else:
log.warning('Output file %s already exists and overwrite '
'not specified!' % self.outcontext)
log.error('Quitting... Please remove before resuming '
'operations.')
raise IOError
# Get default headers from multi-extension FITS file
# If only writing out single drizzle product, blending needs to be
# forced off as there is only 1 input to report, no blending needed
if self.single:
blend=False
# If input data is not in MEF FITS format, it will return 'None'
# and those headers will have to be generated from drizzle output
# file FITS headers.
# NOTE: These are HEADER objects, not HDUs
#prihdr,scihdr,errhdr,dqhdr = getTemplates(template)
self.fullhdrs, intab = getTemplates(template, blend=False)
newhdrs, newtab = getTemplates(template,blend=blend)
if newtab is not None: nextend += 1 # account for new table extn
prihdr = newhdrs[0]
scihdr = newhdrs[1]
errhdr = newhdrs[2]
dqhdr = newhdrs[3]
# Setup primary header as an HDU ready for appending to output FITS file
prihdu = fits.PrimaryHDU(header=prihdr, data=None)
# Start by updating PRIMARY header keywords...
prihdu.header.set('EXTEND', value=True, after='NAXIS')
prihdu.header['NEXTEND'] = nextend
prihdu.header['FILENAME'] = self.output
prihdu.header['PROD_VER'] = 'DrizzlePac {}'.format(version.__version__)
# Update the ROOTNAME with the new value as well
_indx = self.output.find('_drz')
if _indx < 0:
rootname_val = self.output
else:
rootname_val = self.output[:_indx]
prihdu.header['ROOTNAME'] = rootname_val
# Get the total exposure time for the image
# If not calculated by PyDrizzle and passed through
# the pardict, then leave value from the template image.
if self.texptime:
prihdu.header['EXPTIME'] = self.texptime
prihdu.header.set('TEXPTIME', value=self.texptime, after='EXPTIME')
prihdu.header['EXPSTART'] = self.expstart
prihdu.header['EXPEND'] = self.expend
#Update ASN_MTYPE to reflect the fact that this is a product
# Currently hard-wired to always output 'PROD-DTH' as MTYPE
prihdu.header['ASN_MTYP'] = 'PROD-DTH'
# Update DITHCORR calibration keyword if present
# Remove when we can modify FITS headers in place...
if 'DRIZCORR' in prihdu.header:
prihdu.header['DRIZCORR'] = 'COMPLETE'
if 'DITHCORR' in prihdu.header:
prihdu.header['DITHCORR'] = 'COMPLETE'
prihdu.header['NDRIZIM'] =(len(self.parlist),
'Drizzle, No. images drizzled onto output')
# Only a subset of these keywords makes sense for the new WCS based
# transformations. They need to be reviewed to decide what to keep
# and what to leave out.
if not self.blot:
self.addDrizKeywords(prihdu.header,versions)
if scihdr:
try:
del scihdr['OBJECT']
except KeyError:
pass
if 'CCDCHIP' in scihdr: scihdr['CCDCHIP'] = '-999'
if 'NCOMBINE' in scihdr:
scihdr['NCOMBINE'] = self.parlist[0]['nimages']
# If BUNIT keyword was found and reset, then
bunit_last_kw = self.find_kwupdate_location(scihdr,'bunit')
if self.bunit is not None:
comment_str = "Units of science product"
if self.bunit.lower()[:5] == 'count':
comment_str = "counts * gain = electrons"
scihdr.set('BUNIT', value=self.bunit,
comment=comment_str,
after=bunit_last_kw)
else:
# check to see whether to update already present BUNIT comment
if 'bunit' in scihdr and scihdr['bunit'].lower()[:5] == 'count':
comment_str = "counts * gain = electrons"
scihdr.set('BUNIT', value=scihdr['bunit'],
comment=comment_str,
after=bunit_last_kw)
# Add WCS keywords to SCI header
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(scihdr,'CD1_1')
addWCSKeywords(self.wcs,scihdr,blot=self.blot,
single=self.single, after=pre_wcs_kw)
# Recompute this after removing distortion kws
pre_wcs_kw = self.find_kwupdate_location(scihdr,'CD1_1')
##########
# Now, build the output file
##########
if self.build:
print('-Generating multi-extension output file: ',self.output)
fo = fits.HDUList()
# Add primary header to output file...
fo.append(prihdu)
if self.single and self.compress:
hdu = fits.CompImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0])
else:
hdu = fits.ImageHDU(data=sciarr, header=scihdr, name=EXTLIST[0])
last_kw = self.find_kwupdate_location(scihdr,'EXTNAME')
hdu.header.set('EXTNAME', value='SCI', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
fo.append(hdu)
# Build WHT extension here, if requested...
if errhdr:
errhdr['CCDCHIP'] = '-999'
if self.single and self.compress:
hdu = fits.CompImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1])
else:
hdu = fits.ImageHDU(data=whtarr, header=errhdr, name=EXTLIST[1])
last_kw = self.find_kwupdate_location(errhdr,'EXTNAME')
hdu.header.set('EXTNAME', value='WHT', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,hdu.header,blot=self.blot,
single=self.single, after=pre_wcs_kw)
fo.append(hdu)
# Build CTX extension here
# If there is only 1 plane, write it out as a 2-D extension
if self.outcontext:
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
else:
_ctxarr = None
if self.single and self.compress:
hdu = fits.CompImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2])
else:
hdu = fits.ImageHDU(data=_ctxarr, header=dqhdr, name=EXTLIST[2])
last_kw = self.find_kwupdate_location(dqhdr,'EXTNAME')
hdu.header.set('EXTNAME', value='CTX', after=last_kw)
hdu.header.set('EXTVER', value=1, after='EXTNAME')
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,hdu.header,blot=self.blot,
single=self.single, after=pre_wcs_kw)
fo.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fo,[1])
# add table of combined header keyword values to FITS file
if newtab is not None:
fo.append(newtab)
if not virtual:
print('Writing out to disk:',self.output)
# write out file to disk
fo.writeto(self.output)
fo.close()
del fo, hdu
fo = None
# End 'if not virtual'
outputFITS[self.output]= fo
else:
print('-Generating simple FITS output: %s' % self.outdata)
fo = fits.HDUList()
hdu_header = prihdu.header.copy()
del hdu_header['nextend']
# Append remaining unique header keywords from template DQ
# header to Primary header...
if scihdr:
for _card in scihdr.cards:
if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu_header:
hdu_header.append(_card)
for kw in ['PCOUNT', 'GCOUNT']:
try:
del kw
except KeyError:
pass
hdu_header['filename'] = self.outdata
if self.compress:
hdu = fits.CompImageHDU(data=sciarr, header=hdu_header)
wcs_ext = [1]
else:
hdu = fits.ImageHDU(data=sciarr, header=hdu_header)
wcs_ext = [0]
# explicitly set EXTEND to FALSE for simple FITS files.
dim = len(sciarr.shape)
hdu.header.set('extend', value=False, after='NAXIS%s'%dim)
# Add primary header to output file...
fo.append(hdu)
# remove all alternate WCS solutions from headers of this product
logutil.logging.disable(logutil.logging.INFO)
wcs_functions.removeAllAltWCS(fo,wcs_ext)
logutil.logging.disable(logutil.logging.NOTSET)
# add table of combined header keyword values to FITS file
if newtab is not None:
fo.append(newtab)
if not virtual:
print('Writing out image to disk:',self.outdata)
# write out file to disk
fo.writeto(self.outdata)
del fo,hdu
fo = None
# End 'if not virtual'
outputFITS[self.outdata]= fo
if self.outweight and whtarr is not None:
# We need to build new PyFITS objects for each WHT array
fwht = fits.HDUList()
if errhdr:
errhdr['CCDCHIP'] = '-999'
if self.compress:
hdu = fits.CompImageHDU(data=whtarr, header=prihdu.header)
else:
hdu = fits.ImageHDU(data=whtarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if errhdr:
for _card in errhdr.cards:
if _card.keyword not in RESERVED_KEYS and _card.keyword not in hdu.header:
hdu.header.append(_card)
hdu.header['filename'] = self.outweight
hdu.header['CCDCHIP'] = '-999'
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,hdu.header, blot=self.blot,
single=self.single, after=pre_wcs_kw)
# Add primary header to output file...
fwht.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fwht,wcs_ext)
if not virtual:
print('Writing out image to disk:',self.outweight)
fwht.writeto(self.outweight)
del fwht,hdu
fwht = None
# End 'if not virtual'
outputFITS[self.outweight]= fwht
# If a context image was specified, build a PyFITS object
# for it as well...
if self.outcontext and ctxarr is not None:
fctx = fits.HDUList()
# If there is only 1 plane, write it out as a 2-D extension
if ctxarr.shape[0] == 1:
_ctxarr = ctxarr[0]
else:
_ctxarr = ctxarr
if self.compress:
hdu = fits.CompImageHDU(data=_ctxarr, header=prihdu.header)
else:
hdu = fits.ImageHDU(data=_ctxarr, header=prihdu.header)
# Append remaining unique header keywords from template DQ
# header to Primary header...
if dqhdr:
for _card in dqhdr.cards:
if ( (_card.keyword not in RESERVED_KEYS) and
_card.keyword not in hdu.header):
hdu.header.append(_card)
hdu.header['filename'] = self.outcontext
if self.wcs:
pre_wcs_kw = self.find_kwupdate_location(hdu.header,'CD1_1')
# Update WCS Keywords based on PyDrizzle product's value
# since 'drizzle' itself doesn't update that keyword.
addWCSKeywords(self.wcs,hdu.header, blot=self.blot,
single=self.single, after=pre_wcs_kw)
fctx.append(hdu)
# remove all alternate WCS solutions from headers of this product
wcs_functions.removeAllAltWCS(fctx,wcs_ext)
if not virtual:
print('Writing out image to disk:',self.outcontext)
fctx.writeto(self.outcontext)
del fctx,hdu
fctx = None
# End 'if not virtual'
outputFITS[self.outcontext]= fctx
return outputFITS
def __init__(self, image=None, inimage=None):
if image == None:
self.scale = 1.0
self.coeffs = None
self.lam = 555.0
self.xsh = 0.0
self.ysh = 0.0
self.rot = 0.0
self.shft_un = "input"
self.shft_fr = "input"
self.align = "center"
self.xgeoim = ""
self.ygeoim = ""
self.d2xscale = 0.0
self.d2yscale = 0.0
self.d2xsh = 0.0
self.d2ysh = 0.0
self.d2rot = 0.0
self.d2shft_fr = "output"
else:
# Read geometric parameters from a header using an image name as
# the key
found = FALSE
# First search for the entry for this image
i = 1
while i < MaxImages:
datkey = 'D%3iDATA' % i
datkey = datkey.replace(' ', '0')
iraf.keypar(image, datkey, silent='yes')
# If we can't read this no point considering
if iraf.keypar.value == '':
break
# If we have a match set flag and leave
if iraf.keypar.value == inimage:
found = TRUE
break
i += 1
if not found:
raise Exception(
"Failed to get keyword information from header.")
# Now we know that the selected image is present we can
# get all the other parameters - we don't check whether this
# succeeds, if it doesn't let it crash
stem = datkey[:4]
iraf.keypar(image, stem + "SCAL", silent='yes')
self.scale = float(iraf.keypar.value)
iraf.keypar(image, stem + "COEF", silent='yes')
self.coeffs = iraf.keypar.value
# Check for existence
if fileutil.findFile(self.coeffs) == FALSE:
try:
print(
'\n-Coeffs file not found. Trying to reproduce them using PyDrizzle...'
)
# Try to generate the coeffs file automatically
indx = inimage.find('[')
p = pydrizzle.PyDrizzle(inimage[:indx],
bits_single=None,
bits_final=None,
updatewcs=FALSE)
del p
except:
print("! Cannot access coefficients file. (", self.coeffs,
")")
raise Exception("File missing or inaccessible.")
iraf.keypar(image, stem + "LAM", silent='yes')
if iraf.keypar.value != '':
self.lam = float(iraf.keypar.value)
else:
self.lam = 555.0
iraf.keypar(image, stem + "XSH", silent='yes')
self.xsh = float(iraf.keypar.value)
iraf.keypar(image, stem + "YSH", silent='yes')
self.ysh = float(iraf.keypar.value)
iraf.keypar(image, stem + "ROT", silent='yes')
self.rot = float(iraf.keypar.value)
iraf.keypar(image, stem + "SFTU", silent='yes')
self.shft_un = iraf.keypar.value
iraf.keypar(image, stem + "SFTF", silent='yes')
self.shft_fr = iraf.keypar.value
iraf.keypar(image, stem + "XGIM", silent='yes')
self.xgeoim = iraf.keypar.value
indx = self.xgeoim.find('[')
# Check for existence
if fileutil.findFile(
self.xgeoim[:indx]) == FALSE and self.xgeoim != '':
print("! Warning, cannot access X distortion correction image")
print(" continuing without it. (", self.xgeoim, ")")
self.xgeoim = ''
iraf.keypar(image, stem + "YGIM", silent='yes')
self.ygeoim = iraf.keypar.value
indx = self.ygeoim.find('[')
# Check for existence
if fileutil.findFile(
self.ygeoim[:indx]) == FALSE and self.ygeoim != '':
print("! Warning, cannot access Y distortion correction image")
print(" continuing without it. (", self.ygeoim, ")")
self.ygeoim = ''
# The case of the "align" parameter is more tricky, we
# have to deduce it from INXC keyword
iraf.keypar(image, stem + "INXC", silent='yes')
inxc = float(iraf.keypar.value)
# Need the X and Y dimensions as well - both input and
# output
iraf.keypar(inimage, 'i_naxis1', silent='yes')
xdim = int(iraf.keypar.value)
iraf.keypar(inimage, 'i_naxis2', silent='yes')
ydim = int(iraf.keypar.value)
self.nxin = xdim
self.nyin = ydim
iraf.keypar(image, 'i_naxis1', silent='yes')
xdim = int(iraf.keypar.value)
iraf.keypar(image, 'i_naxis2', silent='yes')
ydim = int(iraf.keypar.value)
self.nxout = xdim
self.nyout = ydim
if abs(inxc - float(xdim / 2) - 0.5) < 1e-4:
self.align = 'corner'
else:
self.align = 'center'
# Check for the presence of secondary parameters
iraf.keypar(image, stem + "SECP", silent='yes')
if iraf.keypar.value == "yes":
raise Exception(
"Sorry, this version does NOT support secondary parameters."
)
else:
self.secp = FALSE
def writeSingleFITS(data,wcs,output,template,clobber=True,verbose=True):
""" Write out a simple FITS file given a numpy array and the name of another
FITS file to use as a template for the output image header.
"""
outname,outextn = fileutil.parseFilename(output)
outextname,outextver = fileutil.parseExtn(outextn)
if fileutil.findFile(outname):
if clobber:
log.info('Deleting previous output product: %s' % outname)
fileutil.removeFile(outname)
else:
log.warning('Output file %s already exists and overwrite not '
'specified!' % outname)
log.error('Quitting... Please remove before resuming operations.')
raise IOError
# Now update WCS keywords with values from provided WCS
if hasattr(wcs.sip,'a_order'):
siphdr = True
else:
siphdr = False
wcshdr = wcs.wcs2header(sip2hdr=siphdr)
if template is not None:
# Get default headers from multi-extension FITS file
# If input data is not in MEF FITS format, it will return 'None'
# NOTE: These are HEADER objects, not HDUs
(prihdr,scihdr,errhdr,dqhdr),newtab = getTemplates(template,EXTLIST)
if scihdr is None:
scihdr = fits.Header()
indx = 0
for c in prihdr.cards:
if c.keyword not in ['INHERIT','EXPNAME']: indx += 1
else: break
for i in range(indx,len(prihdr)):
scihdr.append(prihdr.cards[i])
for i in range(indx, len(prihdr)):
del prihdr[indx]
else:
scihdr = fits.Header()
prihdr = fits.Header()
# Start by updating PRIMARY header keywords...
prihdr.set('EXTEND', value=True, after='NAXIS')
prihdr['FILENAME'] = outname
if outextname == '':
outextname = 'sci'
if outextver == 0: outextver = 1
scihdr['EXTNAME'] = outextname.upper()
scihdr['EXTVER'] = outextver
for card in wcshdr.cards:
scihdr[card.keyword] = (card.value, card.comment)
# Create PyFITS HDUList for all extensions
outhdu = fits.HDUList()
# Setup primary header as an HDU ready for appending to output FITS file
prihdu = fits.PrimaryHDU(header=prihdr)
scihdu = fits.ImageHDU(header=scihdr,data=data)
outhdu.append(prihdu)
outhdu.append(scihdu)
outhdu.writeto(outname)
if verbose:
print('Created output image: %s' % outname)
def apply_d2im_correction(fname, d2imcorr):
"""
Logic to decide whether to apply the D2IM correction.
Parameters
----------
fname : `~astropy.io.fits.HDUList` or str
Input FITS science file object.
d2imcorr : bool
Flag indicating if D2IM is should be enabled if allowed.
Return
------
applyD2IMCorr : bool
Flag whether to apply the correction.
The D2IM correction is applied to a science file if it is in the
allowed corrections for the instrument. The name of the file
with the correction is saved in the ``D2IMFILE`` keyword in the
primary header. When the correction is applied the name of the
file is saved in the ``D2IMEXT`` keyword in the 1st extension header.
"""
fname, toclose = _toclose(fname)
applyD2IMCorr = True
if not d2imcorr:
logger.info("D2IM correction not requested - not applying it.")
return False
# get D2IMFILE kw from primary header
try:
fd2im0 = fname[0].header['D2IMFILE']
except KeyError:
logger.info(
"D2IMFILE keyword is missing - D2IM correction will not be applied."
)
return False
if fd2im0 == 'N/A':
utils.remove_distortion(fname, "D2IMFILE")
return False
fd2im0 = fileutil.osfn(fd2im0)
if not fileutil.findFile(fd2im0):
message = "D2IMFILE {0} not found.".format(fd2im0)
logger.critical(message)
raise IOError(message)
try:
# get D2IMEXT kw from first extension header
fd2imext = fname[1].header['D2IMEXT']
except KeyError:
# the case of D2IMFILE kw present in primary header but D2IMEXT missing
# in first extension header
return True
fd2imext = fileutil.osfn(fd2imext)
if fd2imext and fileutil.findFile(fd2imext):
if fd2im0 != fd2imext:
applyD2IMCorr = True
else:
applyD2IMCorr = False
else:
# D2IM file defined in first extension may not be found
# but if a valid kw exists in the primary header,
# detector to image correction should be applied.
applyD2IMCorr = True
if toclose:
fname.close()
return applyD2IMCorr
def buildShadowMaskImage(dqfile,
detnum,
extnum,
maskname,
bitvalue=None,
binned=1):
""" Builds mask image from WFPC2 shadow calibrations.
detnum - string value for 'DETECTOR' detector
"""
# insure detnum is a string
if type(detnum) != type(''):
detnum = repr(detnum)
_funcroot = '_func_Shadow_WF'
# build template shadow mask's filename
# If an old version of the maskfile was present, remove it and rebuild it.
if fileutil.findFile(maskname):
fileutil.removeFile(maskname)
_use_inmask = not fileutil.findFile(dqfile) or bitvalue is None
# Check for existance of input .c1h file for use in making inmask file
if _use_inmask:
#_mask = 'wfpc2_inmask'+detnum+'.fits'
_mask = maskname
# Check to see if file exists...
if not fileutil.findFile(_mask):
# If not, create the file.
# This takes a long time to run, so it should be done
# only when absolutely necessary...
try:
_funcx = _funcroot + detnum + 'x'
_funcy = _funcroot + detnum + 'y'
_xarr = np.clip(np.fromfunction(eval(_funcx), (800, 800)), 0.0,
1.0).astype(np.uint8)
_yarr = np.clip(np.fromfunction(eval(_funcy), (800, 800)), 0.0,
1.0).astype(np.uint8)
maskarr = _xarr * _yarr
if binned != 1:
bmaskarr = maskarr[::2, ::2]
bmaskarr *= maskarr[1::2, ::2]
bmaskarr *= maskarr[::2, 1::2]
bmaskarr *= maskarr[1::2, 1::2]
maskarr = bmaskarr.copy()
del bmaskarr
#Write out the mask file as simple FITS file
fmask = fits.open(_mask, mode='append', memmap=False)
maskhdu = fits.PrimaryHDU(data=maskarr)
fmask.append(maskhdu)
#Close files
fmask.close()
del fmask
except:
return None
else:
#
# Build full mask based on .c1h and shadow mask
#
fdq = fileutil.openImage(dqfile, mode='readonly', memmap=False)
try:
# Read in DQ array from .c1h and from shadow mask files
dqarr = fdq[int(extnum)].data
#maskarr = fsmask[0].data
# Build mask array from DQ array
dqmaskarr = buildMask(dqarr, bitvalue)
#Write out the mask file as simple FITS file
fdqmask = fits.open(maskname, mode='append', memmap=False)
maskhdu = fits.PrimaryHDU(data=dqmaskarr)
fdqmask.append(maskhdu)
#Close files
fdqmask.close()
del fdqmask
fdq.close()
del fdq
except:
fdq.close()
del fdq
# Safeguard against leaving behind an incomplete file
if fileutil.findFile(maskname):
os.remove(maskname)
_errstr = "\nWarning: Problem creating DQMASK file for " + rootname + ".\n"
#raise IOError, _errstr
print(_errstr)
return None
# Return the name of the mask image written out
return maskname
def writeSingleFITS(data,
wcs,
output,
template,
clobber=True,
verbose=True,
rules_file=None):
""" Write out a simple FITS file given a numpy array and the name of another
FITS file to use as a template for the output image header.
"""
outname, outextn = fileutil.parseFilename(output)
outextname, outextver = fileutil.parseExtn(outextn)
if fileutil.findFile(outname):
if clobber:
log.info('Deleting previous output product: %s' % outname)
fileutil.removeFile(outname)
else:
log.warning('Output file %s already exists and overwrite not '
'specified!' % outname)
log.error('Quitting... Please remove before resuming operations.')
raise IOError
# Now update WCS keywords with values from provided WCS
if hasattr(wcs.sip, 'a_order'):
siphdr = True
else:
siphdr = False
wcshdr = wcs.wcs2header(sip2hdr=siphdr)
if template is not None:
# Get default headers from multi-extension FITS file
# If input data is not in MEF FITS format, it will return 'None'
# NOTE: These are HEADER objects, not HDUs
(prihdr, scihdr, errhdr,
dqhdr), newtab = getTemplates(template,
EXTLIST,
rules_file=rules_file)
if scihdr is None:
scihdr = fits.Header()
indx = 0
for c in prihdr.cards:
if c.keyword not in ['INHERIT', 'EXPNAME']: indx += 1
else: break
for i in range(indx, len(prihdr)):
scihdr.append(prihdr.cards[i])
for i in range(indx, len(prihdr)):
del prihdr[indx]
else:
scihdr = fits.Header()
prihdr = fits.Header()
# Start by updating PRIMARY header keywords...
prihdr.set('EXTEND', value=True, after='NAXIS')
prihdr['FILENAME'] = outname
if outextname == '':
outextname = 'sci'
if outextver == 0: outextver = 1
scihdr['EXTNAME'] = outextname.upper()
scihdr['EXTVER'] = outextver
scihdr.update(wcshdr)
# Create PyFITS HDUList for all extensions
outhdu = fits.HDUList()
# Setup primary header as an HDU ready for appending to output FITS file
prihdu = fits.PrimaryHDU(header=prihdr)
scihdu = fits.ImageHDU(header=scihdr, data=data)
outhdu.append(prihdu)
outhdu.append(scihdu)
outhdu.writeto(outname)
if verbose:
print('Created output image: %s' % outname)
def buildMaskImage(rootname, bitvalue, output, extname='DQ', extver=1):
""" Builds mask image from rootname's DQ array
If there is no valid 'DQ' array in image, then return
an empty string.
"""
# If no bitvalue is set or rootname given, assume no mask is desired
# However, this name would be useful as the output mask from
# other processing, such as MultiDrizzle, so return it anyway.
#if bitvalue == None or rootname == None:
# return None
# build output name
maskname = output
# If an old version of the maskfile was present, remove it and rebuild it.
if fileutil.findFile(maskname):
fileutil.removeFile(maskname)
# Open input file with DQ array
fdq = fileutil.openImage(rootname, mode='readonly', memmap=False)
try:
_extn = fileutil.findExtname(fdq, extname, extver=extver)
if _extn is not None:
# Read in DQ array
dqarr = fdq[_extn].data
else:
dqarr = None
# For the case where there is no DQ array,
# create a mask image of all ones.
if dqarr is None:
# We need to get the dimensions of the output DQ array
# Since the DQ array is non-existent, look for the SCI extension
_sci_extn = fileutil.findExtname(fdq, 'SCI', extver=extver)
if _sci_extn is not None:
_shape = fdq[_sci_extn].data.shape
dqarr = np.zeros(_shape, dtype=np.uint16)
else:
raise Exception
# Build mask array from DQ array
maskarr = buildMask(dqarr, bitvalue)
#Write out the mask file as simple FITS file
fmask = fits.open(maskname, mode='append', memmap=False)
maskhdu = fits.PrimaryHDU(data=maskarr)
fmask.append(maskhdu)
#Close files
fmask.close()
del fmask
fdq.close()
del fdq
except:
fdq.close()
del fdq
# Safeguard against leaving behind an incomplete file
if fileutil.findFile(maskname):
os.remove(maskname)
_errstr = "\nWarning: Problem creating MASK file for " + rootname + ".\n"
#raise IOError, _errstr
print(_errstr)
return None
# Return the name of the mask image written out
return maskname
def run_driz(imageObjectList,
output_wcs,
paramDict,
single,
build,
wcsmap=None):
""" Perform drizzle operation on input to create output.
The input parameters originally was a list
of dictionaries, one for each input, that matches the
primary parameters for an ``IRAF`` `drizzle` task.
This method would then loop over all the entries in the
list and run `drizzle` for each entry.
Parameters required for input in paramDict:
build,single,units,wt_scl,pixfrac,kernel,fillval,
rot,scale,xsh,ysh,blotnx,blotny,outnx,outny,data
"""
# Insure that input imageObject is a list
if not isinstance(imageObjectList, list):
imageObjectList = [imageObjectList]
#
# Setup the versions info dictionary for output to PRIMARY header
# The keys will be used as the name reported in the header, as-is
#
_versions = {
'AstroDrizzle': __version__,
'PyFITS': util.__fits_version__,
'Numpy': util.__numpy_version__
}
# Set sub-sampling rate for drizzling
# stepsize = 2.0
log.info(' **Using sub-sampling value of %s for kernel %s' %
(paramDict['stepsize'], paramDict['kernel']))
maskval = interpret_maskval(paramDict)
outwcs = copy.deepcopy(output_wcs)
# Check for existance of output file.
if (not single and build
and fileutil.findFile(imageObjectList[0].outputNames['outFinal'])):
log.info('Removing previous output product...')
os.remove(imageObjectList[0].outputNames['outFinal'])
# print out parameters being used for drizzling
log.info("Running Drizzle to create output frame with WCS of: ")
output_wcs.printwcs()
# Will we be running in parallel?
pool_size = util.get_pool_size(paramDict.get('num_cores'),
len(imageObjectList))
run_parallel = single and pool_size > 1
if run_parallel:
log.info(f'Executing {pool_size:d} parallel workers')
else:
if single: # not yet an option for final drizzle, msg would confuse
log.info('Executing serially')
# Set parameters for each input and run drizzle on it here.
#
# Perform drizzling...
numctx = 0
for img in imageObjectList:
numctx += img._nmembers
_numctx = {'all': numctx}
# if single:
# Determine how many chips make up each single image
for img in imageObjectList:
for chip in img.returnAllChips(extname=img.scienceExt):
plsingle = chip.outputNames['outSingle']
if plsingle in _numctx: _numctx[plsingle] += 1
else: _numctx[plsingle] = 1
# Compute how many planes will be needed for the context image.
_nplanes = int((_numctx['all'] - 1) / 32) + 1
# For single drizzling or when context is turned off,
# minimize to 1 plane only...
if single or imageObjectList[0][1].outputNames['outContext'] in [
None, '', ' '
]:
_nplanes = 1
#
# An image buffer needs to be setup for converting the input
# arrays (sci and wht) from FITS format to native format
# with respect to byteorder and byteswapping.
# This buffer should be reused for each input if possible.
#
_outsci = _outwht = _outctx = _hdrlist = None
if (not single) or \
(single and (not run_parallel) and (not imageObjectList[0].inmemory)):
# Note there are four cases/combinations for single drizzle alone here:
# (not-inmem, serial), (not-inmem, parallel), (inmem, serial), (inmem, parallel)
_outsci = np.empty(output_wcs.array_shape, dtype=np.float32)
_outsci.fill(maskval)
_outwht = np.zeros(output_wcs.array_shape, dtype=np.float32)
# initialize context to 3-D array but only pass appropriate plane to drizzle as needed
_outctx = np.zeros((_nplanes, ) + output_wcs.array_shape,
dtype=np.int32)
_hdrlist = []
# Keep track of how many chips have been processed
# For single case, this will determine when to close
# one product and open the next.
_chipIdx = 0
# Remember the name of the 1st image that goes into this particular product
# Insure that the header reports the proper values for the start of the
# exposure time used to make this; in particular, TIME-OBS and DATE-OBS.
template = None
#
# Work on each image
#
subprocs = []
for img in imageObjectList:
chiplist = img.returnAllChips(extname=img.scienceExt)
# How many inputs should go into this product?
num_in_prod = _numctx['all']
if single:
num_in_prod = _numctx[chiplist[0].outputNames['outSingle']]
# The name of the 1st image
fnames = []
for chip in chiplist:
fnames.append(chip.outputNames['data'])
if _chipIdx == 0:
template = fnames
else:
template.extend(fnames)
# Work each image, possibly in parallel
if run_parallel:
# use multiprocessing.Manager only if in parallel and in memory
mp_ctx = multiprocessing.get_context('fork')
if img.inmemory:
manager = mp_ctx.Manager()
dproxy = manager.dict(
img.virtualOutputs) # copy & wrap it in proxy
img.virtualOutputs = dproxy
# parallelize run_driz_img (currently for separate drizzle only)
p = mp_ctx.Process(
target=run_driz_img,
name='adrizzle.run_driz_img()', # for err msgs
args=(img, chiplist, output_wcs, outwcs, template, paramDict,
single, num_in_prod, build, _versions, _numctx, _nplanes,
_chipIdx, None, None, None, None, wcsmap))
subprocs.append(p)
else:
# serial run_driz_img run (either separate drizzle or final drizzle)
run_driz_img(img, chiplist, output_wcs, outwcs, template,
paramDict, single, num_in_prod, build, _versions,
_numctx, _nplanes, _chipIdx, _outsci, _outwht,
_outctx, _hdrlist, wcsmap)
# Increment/reset master chip counter
_chipIdx += len(chiplist)
if _chipIdx == num_in_prod:
_chipIdx = 0
# do the join if we spawned tasks
if run_parallel:
mputil.launch_and_wait(subprocs, pool_size) # blocks till all done
del _outsci, _outwht, _outctx, _hdrlist
def run(input,quiet=yes,restore=no,prepend='O', tddcorr=True):
print("+ MAKEWCS Version %s" % __version__)
_prepend = prepend
files = parseinput.parseinput(input)[0]
newfiles = []
if files == []:
print("No valid input files found.\n")
raise IOError
for image in files:
#find out what the input is
imgfits,imgtype = fileutil.isFits(image)
# Check for existence of waiver FITS input, and quit if found.
if imgfits and imgtype == 'waiver':
"""
errormsg = '\n\nPyDrizzle does not support waiver fits format.\n'
errormsg += 'Convert the input files to GEIS or multiextension FITS.\n\n'
raise ValueError, errormsg
"""
newfilename = fileutil.buildNewRootname(image, extn='_c0h.fits')
# Convert GEIS image to MEF file
newimage = fileutil.openImage(image,writefits=True,fitsname=newfilename,clobber=True)
del newimage
# Work with new file
image = newfilename
newfiles.append(image)
# If a GEIS image is provided as input, create a new MEF file with
# a name generated using 'buildFITSName()' and update that new MEF file.
if not imgfits:
# Create standardized name for MEF file
newfilename = fileutil.buildFITSName(image)
# Convert GEIS image to MEF file
newimage = fileutil.openImage(image,writefits=True,fitsname=newfilename,clobber=True)
del newimage
# Work with new file
image = newfilename
newfiles.append(image)
if not quiet:
print("Input files: ",files)
# First get the name of the IDC table
#idctab = drutil.getIDCFile(_files[0][0],keyword='idctab')[0]
idctab = drutil.getIDCFile(image,keyword='idctab')[0]
_found = fileutil.findFile(idctab)
if idctab == None or idctab == '':
print('#\n No IDCTAB specified. No correction can be done for file %s.Quitting makewcs\n' %image)
#raise ValueError
continue
elif not _found:
print('#\n IDCTAB: ',idctab,' could not be found. \n')
print('WCS keywords for file %s will not be updated.\n' %image)
#raise IOError
continue
_phdu = image + '[0]'
_instrument = fileutil.getKeyword(_phdu,keyword='INSTRUME')
if _instrument == 'WFPC2':
Nrefchip, Nrefext = getNrefchip(image)
else:
Nrefchip = None
Nrefext = None
if _instrument not in NUM_PER_EXTN:
raise ValueError("Instrument %s not supported yet. Exiting..." \
%_instrument)
_detector = fileutil.getKeyword(_phdu, keyword='DETECTOR')
_nimsets = get_numsci(image)
for i in range(_nimsets):
if image.find('.fits') > 0:
_img = image+'[sci,'+repr(i+1)+']'
else:
_img = image+'['+repr(i+1)+']'
if not restore:
if not quiet:
print('Updating image: ', _img)
_update(_img,idctab, _nimsets, apply_tdd=False,
quiet=quiet,instrument=_instrument,prepend=_prepend,
nrchip=Nrefchip, nrext = Nrefext)
if _instrument == 'ACS' and _detector == 'WFC':
tddswitch = fileutil.getKeyword(_phdu,keyword='TDDCORR')
# This logic requires that TDDCORR be in the primary header
# and set to PERFORM in order to turn this on at all. It can
# be turned off by setting either tddcorr=False or setting
# the keyword to anything but PERFORM or by deleting the
# keyword altogether. PyDrizzle will rely simply on the
# values of alpha and beta as computed here to apply the
# correction to the coefficients.
if (tddcorr and tddswitch != 'OMIT'):
print('Applying time-dependent distortion corrections...')
_update(_img,idctab, _nimsets, apply_tdd=True, \
quiet=quiet,instrument=_instrument,prepend=_prepend, nrchip=Nrefchip, nrext = Nrefext)
else:
if not quiet:
print('Restoring original WCS values for',_img)
restoreCD(_img,_prepend)
#fimg = fileutil.openImage(image,mode='update')
#if 'TDDCORR' in fimg[0].header and fimg[0].header['TDDCORR'] == 'PERFORM':
# fimg[0].header['TDDCORR'] = 'COMPLETE'
#fimg.close()
if newfiles == []:
return files
else:
return newfiles
