def test_raw_from_asn(_jail):
# Make a dummy input file (to avoid package data headache)
tab = Table()
tab['MEMNAME'] = ['J6LQ01NAQ', 'J6LQ01NDQ', 'J6LQ01011']
tab['MEMTYPE'] = ['EXP-CRJ', 'EXP-CRJ', 'PROD-CRJ']
tab['MEMPRSNT'] = [True, True, True]
datafile = 'dummy_asn.fits'
tab.write(datafile, format='fits', overwrite=True)
raw_files = raw_from_asn(datafile)
assert raw_files == ['j6lq01naq_raw.fits', 'j6lq01ndq_raw.fits']
# Make sure do not download existing file.
# This will fail if download is attemped.
download_crds(datafile)
def get_input_file(self, *args, refsep='$'):
"""
Download or copy input file (e.g., RAW) into the working directory.
The associated CRDS reference files in ``refstr`` are also
downloaded, if necessary.
"""
filename = self.get_data(*args)
ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE'])
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
refname = self.get_data('customRef', ref_file)
else: # Download from FTP, if applicable
refname = os.path.join(ref_file)
if self.use_ftp_crds:
download_crds(refname, self.timeout)
return filename
def get_input_file(self, *args, refsep='$'):
"""
Download or copy input file (e.g., RAW) into the working directory.
The associated CRDS reference files in ``refstr`` are also
downloaded, if necessary.
"""
filename = self.get_data(*args)
ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE'])
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
refname = self.get_data('customRef', ref_file)
else: # Download from FTP, if applicable
refname = os.path.join(ref_file)
if self.use_ftp_crds:
download_crds(refname, self.timeout)
return filename
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
curdir = os.getcwd()
filenames = self.get_data(*args, docopy=docopy)
for filename in filenames:
ref_files = ref_from_image(filename,
reffile_lookup=self.reffile_lookup)
print("Looking for {} REF_FILES: {}".format(filename, ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else:
# Start by checking to see whether IRAF variable *ref/*tab
# has been added to os.environ
refdir, refname = ref_file.split(refsep)
refdir_parent = os.path.split(refdir)[0]
# Define refdir to point to current directory if:
# i. refdir is not defined in environment already
# ii. refdir in os.environ points to another test directory
# This logic should leave refdir unchanged if it already
# points to a globally defined directory.
if refdir not in os.environ or refdir_parent in curdir:
os.environ[refdir] = curdir + os.sep
# Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filenames
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
filename = self.get_data(*args, docopy=docopy)
ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup)
print("Looking for REF_FILES: {}".format(ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else: # Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filename
def get_input_file(self, *args, refsep='$', **kwargs):
# If user has specified action for docopy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
# Download or copy input file (e.g., RAW) into the working directory.
# The associated CRDS reference files in ``refstr`` are also
# downloaded, if necessary.
curdir = os.getcwd()
filenames = self.get_data(*args, docopy=docopy)
for filename in filenames:
ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup)
print("Looking for {} REF_FILES: {}".format(filename, ref_files))
for ref_file in ref_files:
if ref_file.strip() == '':
continue
if refsep not in ref_file: # Local file
self.get_data('customRef', ref_file, docopy=docopy)
else:
# Start by checking to see whether IRAF variable *ref/*tab
# has been added to os.environ
refdir, refname = ref_file.split(refsep)
refdir_parent = os.path.split(refdir)[0]
# Define refdir to point to current directory if:
# i. refdir is not defined in environment already
# ii. refdir in os.environ points to another test directory
# This logic should leave refdir unchanged if it already
# points to a globally defined directory.
if refdir not in os.environ or refdir_parent in curdir:
os.environ[refdir] = curdir + os.sep
# Download from FTP, if applicable
if self.use_ftp_crds:
download_crds(ref_file, timeout=self.timeout)
return filenames
def get_input_file(self, filename):
"""
Copy input file (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table is also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for HSTCAL::
instrument/
detector/
input/
truth/
.. note::
If given filename has ".orig" suffix, the suffix is
also automatically removed.
Parameters
----------
filename : str
Filename of the ASN/RAW/etc to copy over, along with its
associated files.
"""
# Copy over main input file.
dest = get_bigdata(self.env, self.instrument, self.detector,
'input', filename)
# For historical reason, need to remove ".orig" suffix if it exists.
if filename.endswith('.orig'):
newfilename = filename.rstrip('.orig')
os.rename(filename, newfilename)
filename = newfilename
if filename.endswith('_asn.fits'):
all_raws = raw_from_asn(filename)
for raw in all_raws: # Download RAWs in ASN.
get_bigdata(self.env, self.instrument, self.detector,
'input', raw)
else:
all_raws = [filename]
first_pass = ('JENKINS_URL' in os.environ and
'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file and
os.path.basename(ref_file) == ref_file):
get_bigdata(self.env, self.instrument, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
def compare_wcs_alignment(dataset, force=False):
"""Return results from aligning dataset using all available WCS solutions.
This code will ALWAYS make sure the ASTROMETRY_STEP_CONTROL variable
is set to "ON" when running and will reset to the original state when
completed. This insures that the code ALWAYS queries the astrometry
database to apply all avaialable a priori WCS solutions.
Parameters
-----------
dataset : str
Rootname of either a single (un-associated) exposure or an ASN
force : bool
Specify whether or not to overwrite dataset files found locally
with fresh copies retrieved from MAST.
Returns
-------
results : dict
A dictionary whose keys are the WCS's found and fit to GAIA.
Each WCS has entries for:
* imageName - filenames of input exposures included in the fit
* offset_x - offset in X (pixels)
* offset_y - offset in X (pixels)
* rotation - rotation in degrees
* scale - scale from fit
* rms_x - RMS in pixels
* rms_y - RMS in pixels
* fit_rms - RMS in arcseconds
* total_rms - RMS of entire fit in arcseconds
* status - flag indicating success/failure of fit
* fit_qual - flag indicating quality of fit (1-5)
* matched_sources - number of sources used in fit
ASSUMPTIONS
-----------
- All images in dataset have the same set of a priori solutions
- All images in dataset have the same setting for the IDCTAB file
"""
# Setup
# Remember what state the environment was in before this code runs
control = os.environ.get('ASTROMETRY_STEP_CONTROL')
# Insure that database will be queried for new WCS solutions
os.environ['ASTROMETRY_STEP_CONTROL'] = 'ON'
try:
# Step 1:
# Determine alignment for pipeline-defined WCS
align_table = align.perform_align([dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=force,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
if not results:
msg = "No valid exposures found for {}.".format(dataset)
msg += "\n Please check that input was either a valid ASN"
msg += "\n or a single un-associated exposure."
raise ValueError(msg)
imglist = results['imageName'].astype(str).tolist()
# Step 2:
# Create results output organized by WCSNAME
default_wcsname = fits.getval(imglist[0], 'wcsname', ext=1)
log.info("Default WCSNAME: {}".format(default_wcsname))
alignment = {default_wcsname: extract_results(results)}
# Download the calibration reference files to ensure availability
ref_files = ref_from_image(
imglist[0], ['IDCTAB', 'DGEOFILE', 'NPOLFILE', 'D2IMFILE'])
for file in ref_files:
download_crds(file, verbose=True)
# Step 3:
# Update inputs with latest distortion model and pull in solutions from dB
imglist = updatewcs.updatewcs(imglist)
img0 = imglist[0]
# Step 4:
# Loop over each WCS solution and perform alignment to GAIA
wcsnames = headerlet.get_headerlet_kw_names(img0, kw='WCSNAME')
if not wcsnames:
msg = "No a priori solutions found for {}".format(img0)
log.error(msg)
raise ValueError(msg)
for wcs in wcsnames:
log.info("Starting with {}".format(wcs))
if 'OPUS' in wcs or wcs == default_wcsname:
continue # skip default pipeline solutions, since we have already aligned it
# apply WCS from headerlet
for img in imglist:
wnames = headerlet.get_headerlet_kw_names(img, kw='WCSNAME')
hnames = headerlet.get_headerlet_kw_names(img)
# print("[testutils]WCSNAMES[{}]: {}".format(img, wnames))
if wcs in wnames:
hdrname = hnames[wnames.index(wcs)]
log.info("[testutils] Applying WCS {} to {}".format(
hdrname, img))
headerlet.restore_from_headerlet(img,
hdrname=hdrname,
archive=False,
force=True)
print("[testutils] Aligning: {} for WCSNAME: {}".format(
dataset, wcs))
align_table = align.perform_align(
[dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
clobber=False,
debug=True,
product_type='pipeline')
results = align_table.filtered_table
alignment[wcs] = extract_results(results)
except Exception as err:
print(traceback.format_exc())
raise err
finally:
# Regardless of what happens, always reset the environment variable
# if it was modified in the first place.
# Restore user environment to original state
if control is None: # Need to be explicit here since T/F are actually valid
del os.environ['ASTROMETRY_STEP_CONTROL']
else:
os.environ['ASTROMETRY_STEP_CONTROL'] = control
return alignment
def get_input_file(self, filename):
"""
Copy input file (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table is also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for CALCOS::
detector/
input/
truth/
Parameters
----------
filename : str
Filename of the ASN/RAW/etc to copy over, along with its
associated files.
"""
# Copy over main input file.
dest = get_bigdata('scsb-calcos', self.env, self.detector, 'input',
filename)
# TODO: This logic was based on ONE FUV test. Please revise as needed.
if filename.endswith('_asn.fits'):
all_raws = raw_from_asn(filename, '_rawtag_a.fits')
all_raws += raw_from_asn(filename, '_rawtag_b.fits')
for raw in all_raws: # Download RAWs in ASN.
get_bigdata('scsb-calcos', self.env, self.detector, 'input',
raw)
for raw in raw_from_asn(filename, '_spt.fits'): # Input SPTs
get_bigdata('scsb-calcos', self.env, self.detector, 'input',
raw)
else:
all_raws = [filename]
first_pass = ('JENKINS_URL' in os.environ
and 'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file
and os.path.basename(ref_file) == ref_file):
get_bigdata('scsb-calcos', self.env, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
def get_input_file(self, filename):
"""
Copy input file (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table is also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for CALCOS::
detector/
input/
truth/
Parameters
----------
filename : str
Filename of the ASN/RAW/etc to copy over, along with its
associated files.
"""
# Copy over main input file.
dest = get_bigdata('scsb-calcos', self.env, self.detector, 'input',
filename)
# TODO: This logic was based on ONE FUV test. Please revise as needed.
if filename.endswith('_asn.fits'):
all_raws = raw_from_asn(filename, '_rawtag_a.fits')
all_raws += raw_from_asn(filename, '_rawtag_b.fits')
for raw in all_raws: # Download RAWs in ASN.
get_bigdata('scsb-calcos', self.env, self.detector, 'input',
raw)
for raw in raw_from_asn(filename, '_spt.fits'): # Input SPTs
get_bigdata('scsb-calcos', self.env, self.detector, 'input',
raw)
else:
all_raws = [filename]
first_pass = ('JENKINS_URL' in os.environ and
'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file and
os.path.basename(ref_file) == ref_file):
get_bigdata('scsb-calcos', self.env, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
def get_input_files(self, filenames):
"""
Copy input files (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table are also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for CALCOS::
detector/
input/
truth/
Parameters
----------
filename : list
List of filenames of the ASN/RAW/etc to copy over, along with their
associated files.
"""
all_raws = []
for file in filenames:
if file.endswith('_rawtag_a.fits') or file.endswith('_rawtag_b.fits'):
all_raws.append(file)
# List of filenames can include _rawtag, _asn and _spt files
dest = get_bigdata('scsb-calcos', self.env, self.detector, 'input',
file)
# If file is an association table, download raw files specified in the table
if file.endswith('_asn.fits'):
asn_raws = raw_from_asn(file, '_rawtag_a.fits')
asn_raws += raw_from_asn(file, '_rawtag_b.fits')
for raw in asn_raws: # Download RAWs in ASN.
get_bigdata('scsb-calcos', self.env, self.detector, 'input',
raw)
all_raws += asn_raws
first_pass = ('JENKINS_URL' in os.environ and
'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file and
os.path.basename(ref_file) == ref_file):
get_bigdata('scsb-calcos', self.env, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
def get_input_file(self, filename):
"""
Copy input file (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table is also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for HSTCAL::
instrument/
detector/
input/
truth/
.. note::
If given filename has ".orig" suffix, the suffix is
also automatically removed.
Parameters
----------
filename : str
Filename of the ASN/RAW/etc to copy over, along with its
associated files.
"""
# Copy over main input file.
dest = get_bigdata(self.env, self.instrument, self.detector, 'input',
filename)
# For historical reason, need to remove ".orig" suffix if it exists.
if filename.endswith('.orig'):
newfilename = filename.rstrip('.orig')
os.rename(filename, newfilename)
filename = newfilename
if filename.endswith('_asn.fits'):
all_raws = raw_from_asn(filename)
for raw in all_raws: # Download RAWs in ASN.
get_bigdata(self.env, self.instrument, self.detector, 'input',
raw)
else:
all_raws = [filename]
first_pass = ('JENKINS_URL' in os.environ
and 'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file
and os.path.basename(ref_file) == ref_file):
get_bigdata(self.env, self.instrument, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
