def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characteristics of this test:
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits',
'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_ngc188', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
totalRMS = 0.0
input_filenames = ['ib6v06c4q_flc.fits','ib6v06c7q_flc.fits',
'ib6v25aqq_flc.fits','ib6v25atq_flc.fits',
'jddh02gjq_flc.fits','jddh02glq_flc.fits',
'jddh02goq_flc.fits']
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','mosaic_47tuc',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits',
'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits',
'jddh02goq_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_47tuc', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characteristics of this test:
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
totalRMS = 0.0
input_filenames = ['iaal01hxq_flc.fits', 'iaala3btq_flc.fits',
'iaal01hyq_flc.fits', 'iaala3bsq_flc.fits',
'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits']
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','mosaic_ngc188',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_truth_missing(self, docopy):
get_bigdata(*self.inpath, 'j6lq01010_asn.fits', docopy=True)
with pytest.raises(AssertionError) as exc:
compare_outputs(
[('j6lq01010_asn.fits', 'doesnotexist.fits')],
input_path=self.inpath, docopy=docopy, verbose=False)
assert 'Cannot find doesnotexist.fits' in str(exc)
def test_all_okay(self, filename):
"""Same file has no difference."""
get_bigdata(*self.inpath, filename, docopy=True)
report = compare_outputs(
[(filename, filename)], input_path=self.inpath,
docopy=self.copy, verbose=False)
assert 'No differences found' in report
def test_ambiguous_extlist(self, outputs):
"""Too many ways to do the same thing."""
get_bigdata(*self.inpath, 'j6lq01010_asn.fits', docopy=True)
with pytest.raises(AssertionError) as exc:
compare_outputs(outputs, input_path=self.inpath, docopy=self.copy,
verbose=False)
assert 'Ambiguous extension requirements' in str(exc)
def test_mixed_bunch(self):
"""
Test different forms of acceptable ``outputs``.
.. note:: Some other crazy combos are theoretically possible given
the logic but they are not officially supported, hence
not tested here. Add new combo as its support is added.
"""
for filename in ('j6lq01010_asn.fits', 'j6lq01010_asn.txt'):
get_bigdata(*self.inpath, filename, docopy=True)
outputs = [('j6lq01010_asn.fits', 'j6lq01010_asn.fits'),
('j6lq01010_asn.fits[asn]', 'j6lq01010_asn.fits[ASN]'),
{'files': ('j6lq01010_asn.fits[image]',
'j6lq01010_asn_mod.fits[IMAGE]'),
'pars': {'rtol': 1e-7, 'atol': 0.05}},
{'files': ('j6lq01010_asn.fits',
'j6lq01010_asn_mod.fits',
['image']),
'pars': {'rtol': 1e-7, 'atol': 0.05}},
{'files': ('j6lq01010_asn.txt', 'j6lq01010_asn.txt')},
('j6lq01010_asn.fits', 'j6lq01010_asn_mod.fits',
['primary', 'IMAGE']),
('j6lq01010_asn.txt', 'j6lq01010_asn.txt')]
report = compare_outputs(
outputs, input_path=self.inpath, docopy=self.copy,
verbose=False, raise_error=False)
# There are 7 comparisons, and only 1 should show a difference
assert report.count("No differences found") == 6
assert report.count("different pixels found") == 1
def test_nirspec_ifu_wcs(envopt, _jail, test_id, input_file, truth_file):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
del test_id
input_file = get_bigdata('jwst-pipeline', envopt, 'nirspec', 'test_wcs',
'nrs1-ifu', input_file)
truth_file = get_bigdata('jwst-pipeline', envopt, 'nirspec', 'test_wcs',
'nrs1-ifu', 'truth', truth_file)
result = AssignWcsStep.call(input_file,
save_results=True,
suffix='assign_wcs')
result.close()
im = ImageModel(result.meta.filename)
imref = ImageModel(truth_file)
w = nirspec.nrs_wcs_set_input(im, 0)
grid = grid_from_bounding_box(w.bounding_box)
ra, dec, lam = w(*grid)
wref = nirspec.nrs_wcs_set_input(imref, 0)
raref, decref, lamref = wref(*grid)
# equal_nan is used here as many of the entries are nan.
# The domain is defined but it is only a few entries in there that are valid
# as it is a curved narrow slit.
assert_allclose(ra, raref, equal_nan=True)
assert_allclose(dec, decref, equal_nan=True)
assert_allclose(lam, lamref, equal_nan=True)
def test_nirspec_ifu_wcs(envopt, _jail, test_id, input_file, truth_file):
"""
Regression test of creating a WCS object and doing pixel to sky transformation.
"""
del test_id
input_file = get_bigdata('jwst-pipeline', envopt,
'nirspec', 'test_wcs', 'nrs1-ifu', input_file)
truth_file = get_bigdata('jwst-pipeline', envopt,
'nirspec', 'test_wcs', 'nrs1-ifu', 'truth', truth_file)
result = AssignWcsStep.call(input_file, save_results=True, suffix='assign_wcs')
result.close()
im = ImageModel(result.meta.filename)
imref = ImageModel(truth_file)
w = nirspec.nrs_wcs_set_input(im, 0)
grid = grid_from_bounding_box(w.bounding_box)
ra, dec, lam = w(*grid)
wref = nirspec.nrs_wcs_set_input(imref, 0)
raref, decref, lamref = wref(*grid)
# equal_nan is used here as many of the entries are nan.
# The domain is defined but it is only a few entries in there that are valid
# as it is a curved narrow slit.
assert_allclose(ra, raref, equal_nan=True)
assert_allclose(dec, decref, equal_nan=True)
assert_allclose(lam, lamref, equal_nan=True)
def _destripe_one(self,
inputfile,
outsuffix,
outputfile,
truthfile,
masks=None):
# Prepare input file.
# acs_destripe needs access to some reference files.
self.get_input_file(inputfile)
# De-stripe
if masks is None:
acs_destripe.clean(inputfile, outsuffix)
else:
# Get extra input masks
for mfile in masks:
get_bigdata('scsb-acstools', self.env, self.detector, 'input',
mfile)
acs_destripe.clean(inputfile,
outsuffix,
mask1=masks[0],
mask2=masks[1])
# Compare results
self.compare_outputs([(outputfile, truthfile)], rtol=1e-6)
def test_skip_bigdata(pytestconfig):
if not pytestconfig.getoption('bigdata'):
pytest.fail('@pytest.mark.bigdata was not skipped')
# User use bigdata option and decorator but has no big data access.
else:
with pytest.raises(BigdataError):
get_bigdata('foo', 'bar')
def test_raise_error_fits(self):
"""Test mismatched extensions from the same file."""
get_bigdata(*self.inpath, 'j6lq01010_asn.fits', docopy=True)
outputs = [('j6lq01010_asn.fits[PRIMARY]', 'j6lq01010_asn.fits[asn]')]
with pytest.raises(AssertionError) as exc:
compare_outputs(outputs, input_path=self.inpath,
docopy=self.copy, verbose=False)
assert 'Headers contain differences' in str(exc)
def test_align_fail_single_visit(self):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry. This test is known to fail due to "RuntimeError: Number
of output coordinates exceeded allocation (475)a". It will exercise the code using both
catalogs for each of the three fitting algorithms at this time. Nans will be present
in the output table.
The following datasets are used in these tests:
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame
WFC3/UVIS images
"""
total_rms = 0.0
input_filenames = [
'ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits', 'ibjt01aiq_flt.fits',
'ibjt01amq_flt.fits', 'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits',
'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits', 'ibjt01zwq_flc.fits',
'ibjt01a4q_flc.fits', 'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits',
'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits', 'ibjt01avq_flt.fits',
'ibjt01yuq_flc.fits', 'ibjt01ztq_flc.fits'
],
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(
input_filenames,
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
product_type='pipeline',
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def get_asn(self, path=None, docopy=True, get_members=True):
"""Copy association and association members from Artifactory remote
resource to the CWD/truth.
Updates self.input and self.input_remote upon completion
Parameters
----------
path: str
The remote path
docopy : bool
Switch to control whether or not to copy a file
into the test output directory when running the test.
If you wish to open the file directly from remote
location or just to set path to source, set this to `False`.
Default: `True`
get_members: bool
If an association is the input, retrieve the members.
Otherwise, do not.
"""
if path is None:
path = self.input_remote
else:
self.input_remote = path
if docopy is None:
docopy = self.docopy
# Get the association JSON file
self.input = get_bigdata(self._inputs_root,
self._env,
path,
docopy=docopy)
with open(self.input) as fp:
asn = load_asn(fp)
self.asn = asn
# Get each member in the association as well
if get_members:
for product in asn['products']:
for member in product['members']:
fullpath = os.path.join(os.path.dirname(self.input_remote),
member['expname'])
get_bigdata(self._inputs_root,
self._env,
fullpath,
docopy=self.docopy)
def test_align_fail_single_visit(self):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include exposures from a number of single visit datasets to explore what impact differing
observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry.
This test is known to fail due to "RuntimeError: Number of output coordinates exceeded allocation (475)a".
It will exercise the code using both catalogs for each of the three fitting algorithms at this time. Nans
will be present in the output table.
The following datasets are used in these tests:
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame WFC3/UVIS images
"""
totalRMS = 0.0
input_filenames = ['ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits',
'ibjt01aiq_flt.fits', 'ibjt01amq_flt.fits',
'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits',
'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits',
'ibjt01zwq_flc.fits', 'ibjt01a4q_flc.fits',
'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits',
'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits',
'ibjt01avq_flt.fits', 'ibjt01yuq_flc.fits',
'ibjt01ztq_flc.fits'],
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','base_tests',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
assert (0.0 < totalRMS <= RMS_LIMIT)
def get_data(self, *args, **kwargs):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()` or
`astroquery_utils.retrieve_observation`. Use of `astroquery_utils`
will allow getting data directly from MAST via astroquery.
Returns
--------
local_files : list
This will return a list of all the files downloaded with
the full path to the local copy of the file.
"""
if len(args[0]) == 9: # Only a rootname provided
local_files = retrieve_observation(args[0])
else:
# If user has specified action for no_copy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
local_files = get_bigdata(*self.get_input_path(),
*args,
docopy=docopy)
local_files = [local_files]
return local_files
def get_data(self, *args, **kwargs):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()` or
`astroquery_utils.retrieve_observation`. Use of `astroquery_utils`
will allow getting data directly from MAST via astroquery.
Returns
--------
local_files : list
This will return a list of all the files downloaded with
the full path to the local copy of the file.
"""
if len(args[0]) == 9: # Only a rootname provided
local_files = retrieve_observation(args[0])
else:
# If user has specified action for no_copy, apply it with
# default behavior being whatever was defined in the base class.
docopy = kwargs.get('docopy', self.docopy)
local_files = get_bigdata(*self.get_input_path(),
*args,
docopy=docopy)
local_files = [local_files]
return local_files
def get_truth(self, path=None, docopy=None):
"""Copy truth data from Artifactory remote resource to the CWD/truth
Updates self.truth and self.truth_remote on completion
"""
if path is None:
path = self.truth_remote
else:
self.truth_remote = path
if docopy is None:
docopy = self.docopy
os.makedirs('truth', exist_ok=True)
os.chdir('truth')
try:
self.truth = get_bigdata(self._inputs_root,
self._env,
path,
docopy=docopy)
self.truth_remote = os.path.join(self._inputs_root, self._env,
path)
except BigdataError:
os.chdir('..')
raise
os.chdir('..')
return self.truth
def test_lev1(self):
"""This was ``calstis_lev1_FUVspec``."""
raw_file = 'o5cl02040_raw.fits'
wav_file = 'o5cl02040_wav.fits'
# Prepare input files.
self.get_input_file(raw_file)
get_bigdata('scsb-hstcal', self.env, 'stis', 'fuv-mama', 'input',
wav_file)
# Run CALSTIS (equivalent to stistools.calstis.calstis)
subprocess.call(['cs0.e', raw_file, '-v'])
# Compare results
outputs = [('o5cl02040_flt.fits', 'calstis_lev1_FUVspec_flt.fits'),
('o5cl02040_x1d.fits', 'calstis_lev1_FUVspec_x1d.fits')]
self.compare_outputs(outputs)
def get_data(self, *pathargs, docopy=True):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()`.
This will then return the full path to the local copy of the file.
"""
local_file = get_bigdata(*self.repo_path, *pathargs, docopy=docopy)
return local_file
def get_data(self, *pathargs, docopy=True):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()`.
This will then return the full path to the local copy of the file.
"""
local_file = get_bigdata(*self.repo_path, *pathargs, docopy=docopy)
return local_file
def test_get_data(self, _jail):
"""
This tests download when TEST_BIGDATA is pointing to Artifactory.
And tests copy when it is pointing to local path.
"""
args = ('ci-watson', 'dev', 'input', 'j6lq01010_asn.fits')
dest = get_bigdata(*args)
assert dest == os.path.abspath(os.path.join(os.curdir, args[-1]))
def test_nocopy(self, _jail, pytestconfig):
args = (pytestconfig.getini('inputs_root')[0],
'dev',
'input',
'j6lq01010_asn.fits')
dest = get_bigdata(*args, docopy=False)
assert dest == os.path.abspath(os.path.join(self.root, *args))
assert len(os.listdir()) == 0
def test_align_single_visits(self, input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W,
1x F606W, 1x F625W, 2x F658N, 1x F775W, 1x F814W, 1x F850LP, and
2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
total_rms = 0.0
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(
input_filenames,
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
product_type='pipeline',
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def get_data(self, *args):
"""
Download `filename` into working directory using
`get_bigdata`. This will then return the full path to
the local copy of the file.
"""
local_file = get_bigdata(self.inputs_root, self.tree, self.input_loc, *args)
return local_file
def get_data(self, *args, docopy=True):
"""
Download `filename` into working directory using
`get_bigdata`. This will then return the full path to
the local copy of the file.
"""
local_file = get_bigdata(*args, docopy=docopy)
return local_file
def get_data(self, *args):
"""
Download `filename` into working directory using
`get_bigdata`. This will then return the full path to
the local copy of the file.
"""
local_file = get_bigdata(self.inputs_root, self.tree, self.input_loc, *args)
return local_file
def test_generate_catalog(self,input_filenames, truth_file):
""" Verify whether sources from astrometric catalogs can be extracted from images.
Success Criteria
-----------------
* Initially, source catalog matches >80% of 'truth' catalog sources
"""
self.input_loc = 'catalog_tests'
self.curdir = os.getcwd()
truth_path = [self.input_repo, self.tree, self.input_loc, *self.ref_loc]
if not isinstance(input_filenames, list):
input_filenames = [input_filenames]
try:
# Make local copies of input files
local_files = []
for infile in input_filenames:
downloaded_files = self.get_input_file(infile, docopy=True)
local_files.extend(downloaded_files)
test_image = local_files[0]
print("Testing with {}".format(test_image))
imghdu = fits.open(test_image)
instrume = imghdu[0].header['instrume'].lower()
detector = imghdu[0].header['detector'].lower()
instr_pars = detector_specific_params[instrume][detector]
reference_wcs = amutils.build_reference_wcs(local_files)
imcat = amutils.generate_sky_catalog(imghdu, reference_wcs, **instr_pars)
imcat.rename_column('xcentroid', 'x')
imcat.rename_column('ycentroid', 'y')
# create FITS WCS corrector object
wcs_corrector = tweakwcs.FITSWCS(reference_wcs)
# get reference catalog as 'truth' files
reference_catalog = get_bigdata(*truth_path, truth_file, docopy=True)
if os.path.basename(reference_catalog).endswith('ecsv'):
tab_format = 'ascii.ecsv'
else:
tab_format = 'ascii.fast_commented_header'
reference_table = Table.read(reference_catalog, format=tab_format)
num_expected = len(reference_table)
# Perform matching
match = tweakwcs.TPMatch(searchrad=200, separation=0.1, tolerance=5, use2dhist=True)
ridx, iidx = match(reference_table, imcat, wcs_corrector)
nmatches = len(ridx)
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
sys.exit()
assert (nmatches > 0.8*num_expected)
def test_difference_ascii(self):
"""
Test ASCII with differences but suppress error to inspect
returned report.
"""
get_bigdata(*self.inpath, 'j6lq01010_asn_mod.txt', docopy=True)
report = compare_outputs(
[('j6lq01010_asn_mod.txt', 'j6lq01010_asn.txt')],
input_path=self.inpath, docopy=self.copy, verbose=False,
raise_error=False)
s = report.split(os.linesep)
assert s[2:] == ['@@ -1,4 +1,4 @@',
' # MEMNAME MEMTYPE MEMPRSNT',
'-J6LQ01NAQ EXP-CRJ 2',
'+J6LQ01NAQ EXP-CRJ 1',
' J6LQ01NDQ EXP-CRJ 1',
'-J6LQ01013 PROD-RPT 1',
'+J6LQ01011 PROD-CRJ 1',
'']
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
ignore_keywords_overwrite=None):
"""
Compare ACSTOOLS output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata('scsb-acstools', self.env, self.detector,
'truth', desired)
fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical and all_okay:
all_okay = False
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def compare_outputs(self, outputs, atol=0, rtol=1e-7, raise_error=True,
ignore_keywords_overwrite=None):
"""
Compare ACSTOOLS output with "truth" using ``fitsdiff``.
Parameters
----------
outputs : list of tuple
A list of tuples, each containing filename (without path)
of CALXXX output and truth, in that order. Example::
[('output1.fits', 'truth1.fits'),
('output2.fits', 'truth2.fits'),
...]
atol, rtol : float
Absolute and relative tolerance for data comparison.
raise_error : bool
Raise ``AssertionError`` if difference is found.
ignore_keywords_overwrite : list of str or `None`
If not `None`, these will overwrite
``self.ignore_keywords`` for the calling test.
Returns
-------
report : str
Report from ``fitsdiff``.
This is part of error message if ``raise_error=True``.
"""
all_okay = True
creature_report = ''
if ignore_keywords_overwrite is None:
ignore_keywords = self.ignore_keywords
else:
ignore_keywords = ignore_keywords_overwrite
for actual, desired in outputs:
desired = get_bigdata('scsb-acstools', self.env, self.detector,
'truth', desired)
fdiff = FITSDiff(actual, desired, rtol=rtol, atol=atol,
ignore_keywords=ignore_keywords)
creature_report += fdiff.report()
if not fdiff.identical and all_okay:
all_okay = False
if not all_okay and raise_error:
raise AssertionError(os.linesep + creature_report)
return creature_report
def get_data(self, *pathargs, docopy=True):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()`.
This will then return the full path to the local copy of the file.
"""
# If user has specified action for no_copy, apply it with
# default behavior being whatever was defined in the base class.
local_file = get_bigdata(*self.repo_path, *pathargs, docopy=self.docopy)
return local_file
def get_data(self, *pathargs, docopy=True):
"""
Download `filename` into working directory using
`artifactory_helpers/get_bigdata()`.
This will then return the full path to the local copy of the file.
"""
# If user has specified action for no_copy, apply it with
# default behavior being whatever was defined in the base class.
local_file = get_bigdata(*self.repo_path,
*pathargs,
docopy=self.docopy)
return local_file
def test_nrs2_nodata_api(envopt, _jail):
"""
Regression test of handling NRS2 detector that has no data.\
"""
# Only need to ensure that assing_wcs is run.
# This still will fail and should cause the pipeline to halt.
step = Spec2Pipeline()
step.assign_wcs.skip = False
with pytest.raises(NoDataOnDetectorError):
step.run(
get_bigdata('jwst-pipeline', envopt, 'nirspec', 'test_assignwcs',
'jw84700006001_02101_00001_nrs2_rate.fits'))
def test_nrs2_nodata_strun(envopt, _jail):
"""Ensure that the appropriate exit status is returned from strun"""
data_file = get_bigdata('jwst-pipeline', envopt, 'nirspec',
'test_assignwcs',
'jw84700006001_02101_00001_nrs2_rate.fits')
cmd = [
'strun', 'jwst.pipeline.Spec2Pipeline', data_file,
'--steps.assign_wcs.skip=false'
]
status = subprocess.run(cmd)
assert status.returncode == 64
def test_align_single_visits(self,input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what impact differing
observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W, 1x F606W, 1x F625W,
2x F658N, 1x F775W, 1x F814W, 1x F850LP, and 2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
totalRMS = 0.0
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','base_tests',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_nrs2_nodata_api(envopt, _jail):
"""
Regression test of handling NRS2 detector that has no data.\
"""
# Only need to ensure that assing_wcs is run.
# This still will fail and should cause the pipeline to halt.
step = Spec2Pipeline()
step.assign_wcs.skip = False
with pytest.raises(NoDataOnDetectorError):
step.run(get_bigdata('jwst-pipeline', envopt,
'nirspec', 'test_assignwcs',
'jw84700006001_02101_00001_nrs2_rate.fits'
))
def test_trail_mask(self):
"""Mask satellite trail on WFC EXT 6."""
rootname = 'jc8m10syq'
inputfile = rootname + '_flc.fits' # This is modified in-place
truthfile = rootname + '_flc_ref.fits'
# Prepare input file.
get_bigdata('scsb-acstools', self.env, self.detector, 'input',
inputfile)
# Satellite trail masking.
sciext = 4
dqext = 6
trail = ((1199, 1357), (2841, 1023))
mask = satdet.make_mask(inputfile,
sciext,
trail,
plot=False,
verbose=False)
satdet.update_dq(inputfile, dqext, mask, verbose=False)
# Compare results.
self.compare_outputs([(inputfile, truthfile)])
def test_miri_setpointing(self):
"""
Regression test of the set_telescope_pointing script on a level-1b MIRI file.
"""
# Copy original version of file to test file, which will get overwritten by test
input_file = self.get_data(self.test_dir,
'jw80600010001_02101_00001_mirimage_uncal_orig.fits')
# Get SIAF PRD database file
siaf_prd_loc = ['jwst-pipeline', self.env, 'common', 'prd.db']
siaf_path = get_bigdata(*siaf_prd_loc)
add_wcs(input_file, allow_default=True, siaf_path=siaf_path)
outputs = [(input_file,
'jw80600010001_02101_00001_mirimage_uncal_ref.fits')]
self.compare_outputs(outputs)
def test_nrs2_nodata_strun(envopt, _jail):
"""Ensure that the appropriate exit status is returned from strun"""
data_file = get_bigdata('jwst-pipeline', envopt,
'nirspec', 'test_assignwcs',
'jw84700006001_02101_00001_nrs2_rate.fits'
)
cmd = [
'strun',
'jwst.pipeline.Spec2Pipeline',
data_file,
'--steps.assign_wcs.skip=false'
]
status = subprocess.run(cmd)
assert status.returncode == 64
def get_data(self, path=None, docopy=None):
"""Copy data from Artifactory remote resource to the CWD
Updates self.input and self.input_remote upon completion
"""
if path is None:
path = self.input_remote
else:
self.input_remote = path
if docopy is None:
docopy = self.docopy
self.input = get_bigdata(self._inputs_root,
self._env,
path,
docopy=docopy)
return self.input
def test_miri_setpointing(self):
"""
Regression test of the set_telescope_pointing script on a level-1b MIRI file.
"""
# Copy original version of file to test file, which will get overwritten by test
input_file = self.get_data(
self.test_dir,
'jw80600010001_02101_00001_mirimage_uncal_orig.fits')
# Get SIAF PRD database file
siaf_prd_loc = ['jwst-pipeline', self.env, 'common', 'prd.db']
siaf_path = get_bigdata(*siaf_prd_loc)
add_wcs(input_file, allow_default=True, siaf_path=siaf_path)
outputs = [(input_file,
'jw80600010001_02101_00001_mirimage_uncal_ref.fits')]
self.compare_outputs(outputs)
def get_input_file(self, filename):
"""
Copy input file (ASN, RAW, etc) into the working directory.
If ASN is given, RAW files in the ASN table is also copied.
The associated CRDS reference files are also copied or
downloaded, if necessary.
Data directory layout for CALCOS::
detector/
input/
truth/
Parameters
----------
filename : str
Filename of the ASN/RAW/etc to copy over, along with its
associated files.
"""
# Copy over main input file.
dest = get_bigdata('scsb-acstools', self.env, self.detector, 'input',
filename)
# For historical reason, need to remove ".orig" suffix if it exists.
if filename.endswith('.orig'):
newfilename = filename.rstrip('.orig')
os.rename(filename, newfilename)
filename = newfilename
if filename.endswith('_asn.fits'):
all_raws = raw_from_asn(filename)
for raw in all_raws: # Download RAWs in ASN.
get_bigdata('scsb-acstools', self.env, self.detector, 'input',
raw)
else:
all_raws = [filename]
first_pass = ('JENKINS_URL' in os.environ and
'ssbjenkins' in os.environ['JENKINS_URL'])
for raw in all_raws:
ref_files = calref_from_image(raw)
for ref_file in ref_files:
# Special reference files that live with inputs.
if ('$' not in ref_file and
os.path.basename(ref_file) == ref_file):
get_bigdata('scsb-acstools', self.env, self.detector,
'input', ref_file)
continue
# Jenkins cannot see Central Storage on push event,
# and somehow setting, say, jref to "." does not work anymore.
# So, we need this hack.
if '$' in ref_file and first_pass:
first_pass = False
if not os.path.isdir('/grp/hst/cdbs'):
ref_path = os.path.dirname(dest) + os.sep
var = ref_file.split('$')[0]
os.environ[var] = ref_path # hacky hack hack
# Download reference files, if needed only.
download_crds(ref_file, timeout=self.timeout)
