def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration.
This is mostly needed for destriping tools.
"""
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'DQICORR': ['BPIXTAB', 'SNKCFILE'],
'ATODCORR': ['ATODTAB'],
'BLEVCORR': ['OSCNTAB'],
'SINKCORR': ['SNKCFILE'],
'BIASCORR': ['BIASFILE'],
'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'],
'FLSHCORR': ['FLSHFILE'],
'CRCORR': ['CRREJTAB'],
'SHADCORR': ['SHADFILE'],
'DARKCORR': ['DARKFILE', 'TDCTAB'],
'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'],
'PHOTCORR': ['IMPHTTAB'],
'LFLGCORR': ['MLINTAB'],
'GLINCORR': ['MLINTAB'],
'NLINCORR': ['NLINFILE'],
'ZSIGCORR': ['DARKFILE', 'NLINFILE'],
'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'],
'SGEOCORR': ['SDSTFILE'],
'X1DCORR': ['XTRACTAB', 'SDCTAB'],
'SC2DCORR': [
'CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB',
'SRWTAB'
],
'BACKCORR': ['XTRACTAB'],
'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']
}
hdr = fits.getheader(input_image, ext=0)
# Mandatory CRDS reference file.
# Destriping tries to ingest some *FILE regardless of *CORR.
ref_files = ref_from_image(input_image, ['CCDTAB', 'DARKFILE', 'PFLTFILE'])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
# Destriping also does reverse-calib.
if ((step not in hdr)
or (hdr[step].strip().upper() not in ('PERFORM', 'COMPLETE'))):
continue
ref_files += ref_from_image(input_image, corr_lookup[step])
return list(set(ref_files)) # Remove duplicates
def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration.
This is mostly needed for destriping tools.
"""
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'DQICORR': ['BPIXTAB', 'SNKCFILE'],
'ATODCORR': ['ATODTAB'],
'BLEVCORR': ['OSCNTAB'],
'SINKCORR': ['SNKCFILE'],
'BIASCORR': ['BIASFILE'],
'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'],
'FLSHCORR': ['FLSHFILE'],
'CRCORR': ['CRREJTAB'],
'SHADCORR': ['SHADFILE'],
'DARKCORR': ['DARKFILE', 'TDCTAB'],
'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'],
'PHOTCORR': ['IMPHTTAB'],
'LFLGCORR': ['MLINTAB'],
'GLINCORR': ['MLINTAB'],
'NLINCORR': ['NLINFILE'],
'ZSIGCORR': ['DARKFILE', 'NLINFILE'],
'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'],
'SGEOCORR': ['SDSTFILE'],
'X1DCORR': ['XTRACTAB', 'SDCTAB'],
'SC2DCORR': ['CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB',
'TELTAB', 'SRWTAB'],
'BACKCORR': ['XTRACTAB'],
'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']}
hdr = fits.getheader(input_image, ext=0)
# Mandatory CRDS reference file.
# Destriping tries to ingest some *FILE regardless of *CORR.
ref_files = ref_from_image(input_image, ['CCDTAB', 'DARKFILE', 'PFLTFILE'])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
# Destriping also does reverse-calib.
if ((step not in hdr) or
(hdr[step].strip().upper() not in ('PERFORM', 'COMPLETE'))):
continue
ref_files += ref_from_image(input_image, corr_lookup[step])
return list(set(ref_files)) # Remove duplicates
def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration; i.e.,
only those associated with ``*CORR`` set to ``PERFORM`` will be
considered.
"""
# NOTE: Add additional mapping as needed.
# Map mandatory CRDS reference file for instrument/detector combo.
# This is for file not tied to any particular *CORR or used throughout.
det_lookup = {
('COS', 'FUV'): ['PROFTAB', 'SPWCSTAB'],
('COS', 'NUV'): []}
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'BADTCORR': ['BADTTAB'],
'TEMPCORR': ['BRFTAB'],
'GEOCORR': ['GEOFILE'],
'DGEOCORR': ['DGEOFILE'],
'YWLKCORR': ['YWLKFILE'],
'XWLKCORR': ['XWLKFILE'],
'DEADCORR': ['DEADTAB'],
'PHACORR': ['PHATAB', 'PHAFILE'],
'FLATCORR': ['FLATFILE'],
'WAVECORR': ['LAMPTAB', 'DISPTAB', 'TWOZXTAB', 'XTRACTAB'],
'BRSTCORR': ['BRSTTAB'],
'TRCECORR': ['TRACETAB'],
'ALGNCORR': ['TWOZXTAB'],
'DQICORR': ['SPOTTAB', 'TRACETAB', 'BPIXTAB', 'GSAGTAB'],
'X1DCORR': ['WCPTAB', 'TWOZXTAB', 'XTRACTAB'],
'BACKCORR': ['TWOZXTAB', 'XTRACTAB'],
'FLUXCORR': ['FLUXTAB', 'TDSTAB', 'PHOTTAB'],
'WALKCORR': ['WALKTAB']}
hdr = fits.getheader(input_image, ext=0)
ref_files = ref_from_image(
input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
if (step not in hdr) or (hdr[step].strip().upper() != 'PERFORM'):
continue
ref_files += ref_from_image(input_image, corr_lookup[step])
return list(set(ref_files)) # Remove duplicates
def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration; i.e.,
only those associated with ``*CORR`` set to ``PERFORM`` will be
considered.
"""
# NOTE: Add additional mapping as needed.
# Map mandatory CRDS reference file for instrument/detector combo.
# This is for file not tied to any particular *CORR or used throughout.
det_lookup = {
('COS', 'FUV'): ['PROFTAB', 'SPWCSTAB'],
('COS', 'NUV'): []}
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'BADTCORR': ['BADTTAB'],
'TEMPCORR': ['BRFTAB'],
'GEOCORR': ['GEOFILE'],
'DGEOCORR': ['DGEOFILE'],
'YWLKCORR': ['YWLKFILE'],
'XWLKCORR': ['XWLKFILE'],
'DEADCORR': ['DEADTAB'],
'PHACORR': ['PHATAB', 'PHAFILE'],
'FLATCORR': ['FLATFILE'],
'WAVECORR': ['LAMPTAB', 'DISPTAB', 'TWOZXTAB', 'XTRACTAB'],
'BRSTCORR': ['BRSTTAB'],
'TRCECORR': ['TRACETAB'],
'ALGNCORR': ['TWOZXTAB'],
'DQICORR': ['SPOTTAB', 'TRACETAB', 'BPIXTAB', 'GSAGTAB'],
'X1DCORR': ['WCPTAB', 'TWOZXTAB', 'XTRACTAB'],
'BACKCORR': ['TWOZXTAB', 'XTRACTAB'],
'FLUXCORR': ['FLUXTAB', 'TDSTAB', 'PHOTTAB'],
'WALKCORR': ['WALKTAB']}
hdr = fits.getheader(input_image, ext=0)
ref_files = ref_from_image(
input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
if (step not in hdr) or (hdr[step].strip().upper() != 'PERFORM'):
continue
ref_files += ref_from_image(input_image, corr_lookup[step])
return list(set(ref_files)) # Remove duplicates
def get_input_file(self, *args, refsep='$'):
"""
Download or copy input file (e.g., RAW) into the working directory.
The associated CRDS reference files in ``refstr`` are also
downloaded, if necessary.
"""
filename = self.get_data(*args)
ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE'])
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
refname = self.get_data('customRef', ref_file)
else: # Download from FTP, if applicable
refname = os.path.join(ref_file)
if self.use_ftp_crds:
download_crds(refname, self.timeout)
return filename
def get_input_file(self, *args, refsep='$'):
"""
Download or copy input file (e.g., RAW) into the working directory.
The associated CRDS reference files in ``refstr`` are also
downloaded, if necessary.
"""
filename = self.get_data(*args)
ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE'])
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
refname = self.get_data('customRef', ref_file)
else: # Download from FTP, if applicable
refname = os.path.join(ref_file)
if self.use_ftp_crds:
download_crds(refname, self.timeout)
return filename
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
curdir = os.getcwd()
filenames = self.get_data(*args, docopy=docopy)
for filename in filenames:
ref_files = ref_from_image(filename,
reffile_lookup=self.reffile_lookup)
print("Looking for {} REF_FILES: {}".format(filename, ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else:
# Start by checking to see whether IRAF variable *ref/*tab
# has been added to os.environ
refdir, refname = ref_file.split(refsep)
refdir_parent = os.path.split(refdir)[0]
# Define refdir to point to current directory if:
# i. refdir is not defined in environment already
# ii. refdir in os.environ points to another test directory
# This logic should leave refdir unchanged if it already
# points to a globally defined directory.
if refdir not in os.environ or refdir_parent in curdir:
os.environ[refdir] = curdir + os.sep
# Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filenames
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
filename = self.get_data(*args, docopy=docopy)
ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup)
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else: # Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filename
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
curdir = os.getcwd()
filenames = self.get_data(*args, docopy=docopy)
for filename in filenames:
ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup)
print("Looking for {} REF_FILES: {}".format(filename, ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else:
# Start by checking to see whether IRAF variable *ref/*tab
# has been added to os.environ
refdir, refname = ref_file.split(refsep)
refdir_parent = os.path.split(refdir)[0]
# Define refdir to point to current directory if:
# i. refdir is not defined in environment already
# ii. refdir in os.environ points to another test directory
# This logic should leave refdir unchanged if it already
# points to a globally defined directory.
if refdir not in os.environ or refdir_parent in curdir:
os.environ[refdir] = curdir + os.sep
# Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filenames
def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration.
"""
# NOTE: Add additional mapping as needed.
# Map mandatory CRDS reference file for instrument/detector combo.
# This is for file not tied to any particular *CORR or used throughout.
det_lookup = {
('ACS', 'WFC'): ['CCDTAB'],
('ACS', 'HRC'): ['CCDTAB'],
('ACS', 'SBC'): ['CCDTAB'],
('WFC3', 'UVIS'): ['CCDTAB'],
('WFC3', 'IR'): ['CCDTAB'],
('STIS', 'CCD'): ['CCDTAB'],
('STIS', 'FUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB',
'SPTRCTAB'],
('STIS', 'NUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB',
'SPTRCTAB']}
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'DQICORR': ['BPIXTAB', 'SNKCFILE'],
'ATODCORR': ['ATODTAB'],
'BLEVCORR': ['OSCNTAB'],
'SINKCORR': ['SNKCFILE'],
'BIASCORR': ['BIASFILE'],
'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'],
'FLSHCORR': ['FLSHFILE'],
'CRCORR': ['CRREJTAB'],
'SHADCORR': ['SHADFILE'],
'DARKCORR': ['DARKFILE', 'TDCTAB'],
'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'],
'PHOTCORR': ['IMPHTTAB'],
'LFLGCORR': ['MLINTAB'],
'GLINCORR': ['MLINTAB'],
'NLINCORR': ['NLINFILE'],
'ZSIGCORR': ['DARKFILE', 'NLINFILE'],
'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'],
'SGEOCORR': ['SDSTFILE'],
'X1DCORR': ['XTRACTAB', 'SDCTAB'],
'SC2DCORR': ['CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB',
'TELTAB', 'SRWTAB'],
'BACKCORR': ['XTRACTAB'],
'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']}
hdr = fits.getheader(input_image, ext=0)
ref_files = ref_from_image(
input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
# Download ALL reference files associated with a calibration
# step present in the header
if step not in hdr:
continue
single_step_files = ref_from_image(input_image, corr_lookup[step])
if single_step_files:
ref_files += single_step_files
return ref_files
def compare_wcs_alignment(dataset, force=False):
"""Return results from aligning dataset using all available WCS solutions.
This code will ALWAYS make sure the ASTROMETRY_STEP_CONTROL variable
is set to "ON" when running and will reset to the original state when
completed. This insures that the code ALWAYS queries the astrometry
database to apply all avaialable a priori WCS solutions.
Parameters
-----------
dataset : str
Rootname of either a single (un-associated) exposure or an ASN
force : bool
Specify whether or not to overwrite dataset files found locally
with fresh copies retrieved from MAST.
Returns
-------
results : dict
A dictionary whose keys are the WCS's found and fit to GAIA.
Each WCS has entries for:
* imageName - filenames of input exposures included in the fit
* offset_x - offset in X (pixels)
* offset_y - offset in X (pixels)
* rotation - rotation in degrees
* scale - scale from fit
* rms_x - RMS in pixels
* rms_y - RMS in pixels
* fit_rms - RMS in arcseconds
* total_rms - RMS of entire fit in arcseconds
* status - flag indicating success/failure of fit
* fit_qual - flag indicating quality of fit (1-5)
* matched_sources - number of sources used in fit
ASSUMPTIONS
-----------
- All images in dataset have the same set of a priori solutions
- All images in dataset have the same setting for the IDCTAB file
"""
# Setup
# Remember what state the environment was in before this code runs
control = os.environ.get('ASTROMETRY_STEP_CONTROL')
# Insure that database will be queried for new WCS solutions
os.environ['ASTROMETRY_STEP_CONTROL'] = 'ON'
try:
# Step 1:
# Determine alignment for pipeline-defined WCS
align_table = align.perform_align([dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=force,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
if not results:
msg = "No valid exposures found for {}.".format(dataset)
msg += "\n Please check that input was either a valid ASN"
msg += "\n or a single un-associated exposure."
raise ValueError(msg)
imglist = results['imageName'].astype(str).tolist()
# Step 2:
# Create results output organized by WCSNAME
default_wcsname = fits.getval(imglist[0], 'wcsname', ext=1)
log.info("Default WCSNAME: {}".format(default_wcsname))
alignment = {default_wcsname: extract_results(results)}
# Download the calibration reference files to ensure availability
ref_files = ref_from_image(
imglist[0], ['IDCTAB', 'DGEOFILE', 'NPOLFILE', 'D2IMFILE'])
for file in ref_files:
download_crds(file, verbose=True)
# Step 3:
# Update inputs with latest distortion model and pull in solutions from dB
imglist = updatewcs.updatewcs(imglist)
img0 = imglist[0]
# Step 4:
# Loop over each WCS solution and perform alignment to GAIA
wcsnames = headerlet.get_headerlet_kw_names(img0, kw='WCSNAME')
if not wcsnames:
msg = "No a priori solutions found for {}".format(img0)
log.error(msg)
raise ValueError(msg)
for wcs in wcsnames:
log.info("Starting with {}".format(wcs))
if 'OPUS' in wcs or wcs == default_wcsname:
continue # skip default pipeline solutions, since we have already aligned it
# apply WCS from headerlet
for img in imglist:
wnames = headerlet.get_headerlet_kw_names(img, kw='WCSNAME')
hnames = headerlet.get_headerlet_kw_names(img)
# print("[testutils]WCSNAMES[{}]: {}".format(img, wnames))
if wcs in wnames:
hdrname = hnames[wnames.index(wcs)]
log.info("[testutils] Applying WCS {} to {}".format(
hdrname, img))
headerlet.restore_from_headerlet(img,
hdrname=hdrname,
archive=False,
force=True)
print("[testutils] Aligning: {} for WCSNAME: {}".format(
dataset, wcs))
align_table = align.perform_align(
[dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=False,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
alignment[wcs] = extract_results(results)
except Exception as err:
print(traceback.format_exc())
raise err
finally:
# Regardless of what happens, always reset the environment variable
# if it was modified in the first place.
# Restore user environment to original state
if control is None: # Need to be explicit here since T/F are actually valid
del os.environ['ASTROMETRY_STEP_CONTROL']
else:
os.environ['ASTROMETRY_STEP_CONTROL'] = control
return alignment
def calref_from_image(input_image):
"""
Return a list of reference filenames, as defined in the primary
header of the given input image, necessary for calibration.
"""
# NOTE: Add additional mapping as needed.
# Map mandatory CRDS reference file for instrument/detector combo.
# This is for file not tied to any particular *CORR or used throughout.
det_lookup = {
('ACS', 'WFC'): ['CCDTAB'],
('ACS', 'HRC'): ['CCDTAB'],
('ACS', 'SBC'): ['CCDTAB'],
('WFC3', 'UVIS'): ['CCDTAB'],
('WFC3', 'IR'): ['CCDTAB'],
('STIS', 'CCD'): ['CCDTAB'],
('STIS', 'FUV-MAMA'):
['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB'],
('STIS', 'NUV-MAMA'):
['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB']
}
# NOTE: Add additional mapping as needed.
# Map *CORR to associated CRDS reference file.
corr_lookup = {
'DQICORR': ['BPIXTAB', 'SNKCFILE'],
'ATODCORR': ['ATODTAB'],
'BLEVCORR': ['OSCNTAB'],
'SINKCORR': ['SNKCFILE'],
'BIASCORR': ['BIASFILE'],
'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'],
'FLSHCORR': ['FLSHFILE'],
'CRCORR': ['CRREJTAB'],
'SHADCORR': ['SHADFILE'],
'DARKCORR': ['DARKFILE', 'TDCTAB'],
'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'],
'PHOTCORR': ['IMPHTTAB'],
'LFLGCORR': ['MLINTAB'],
'GLINCORR': ['MLINTAB'],
'NLINCORR': ['NLINFILE'],
'ZSIGCORR': ['DARKFILE', 'NLINFILE'],
'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'],
'SGEOCORR': ['SDSTFILE'],
'X1DCORR': ['XTRACTAB', 'SDCTAB'],
'SC2DCORR': [
'CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB',
'SRWTAB'
],
'BACKCORR': ['XTRACTAB'],
'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']
}
hdr = fits.getheader(input_image, ext=0)
ref_files = ref_from_image(input_image,
det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])])
for step in corr_lookup:
# Not all images have the CORR step and it is not always on.
# Download ALL reference files associated with a calibration
# step present in the header
if step not in hdr:
continue
single_step_files = ref_from_image(input_image, corr_lookup[step])
if single_step_files:
ref_files += single_step_files
return ref_files