def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits',
'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits',
'jddh02goq_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_47tuc', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characteristics of this test:
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits',
'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_ngc188', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_fail_single_visit(self):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry. This test is known to fail due to "RuntimeError: Number
of output coordinates exceeded allocation (475)a". It will exercise the code using both
catalogs for each of the three fitting algorithms at this time. Nans will be present
in the output table.
The following datasets are used in these tests:
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame
WFC3/UVIS images
"""
total_rms = 0.0
input_filenames = [
'ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits', 'ibjt01aiq_flt.fits',
'ibjt01amq_flt.fits', 'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits',
'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits', 'ibjt01zwq_flc.fits',
'ibjt01a4q_flc.fits', 'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits',
'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits', 'ibjt01avq_flt.fits',
'ibjt01yuq_flc.fits', 'ibjt01ztq_flc.fits'
],
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(
input_filenames,
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
product_type='pipeline',
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_astroquery(self):
"""Verify that new astroquery interface will work"""
total_rms = 0.0
dataset_table = alignimages.perform_align(['IB6V06060'], archive=False, clobber=True,
debug=False, update_hdr_wcs=False,
print_fit_parameters=True, print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_single_visits(self, input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W,
1x F606W, 1x F625W, 2x F658N, 1x F775W, 1x F814W, 1x F850LP, and
2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
total_rms = 0.0
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(
input_filenames,
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
product_type='pipeline',
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_astroquery(self):
"""Verify that new astroquery interface will work"""
total_rms = 0.01
dataset_table = alignimages.perform_align(
['IB6V06060'],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=True,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
product_type='pipeline',
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table.filtered_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_randomlist(tmpdir, dataset, request):
""" Tests which validate whether mosaics can be aligned to an astrometric standard.
Characteristics of these tests:
* A reference WCS is generated based upon all the input images for
the field.
* A source astrometric catalog is created using the Photutils
package to detect explicitly sources in the images.
* An astrometric catalog is created to extract astrometric positions
for the found sources in the input images' field-of-view using
GAIADR2 (preferred) or GAIADR1.
* Cross matching/fitting is done between found sources and catalog
coordinates with the Tweakwcs package.
* The quality of the fit is evaluated against a minimum threshold and
potentially another fit algorithm is invoked or an alternative
catalog is used in an effort to obtain a better quality fit.
* If the option is set, the WCS information is updated for the
input exposures. The default is False.
* No mosaic is generated.
* An output table containing characterizations of the process and
associated fit is generated.
Success Criteria:
* Success criterion hard-coded for this test represents whether a
statistical sample (70%) of ACS and WFC3 datasets were able to be
aligned to within 10mas RMS.
* RMS values are extracted from the table output from `perform_align`
* This criterion will need to be determined by the user after the test has been
run based on how many datasets were run and skipped.
The input master_list CSV file is
output from a database and lists associations and singletons for ACS
and WFC3 instruments randomly sorted. The actual data files are
downloaded from MAST via astroquery.
This test file can be executed in the following manner:
$ pytest -n # -s --basetemp=/internal/hladata/yourUniqueDirectoryHere --bigdata --slow
--master_list ACSWFC3ListDefault50.csv --start_row 0 --num_rows 50 test_randomlist.py >&
test_random_output.txt &
$ tail -f test_random_output.txt
* The `-n #` option can be used to run tests in parallel if `pytest-xdist` has
been installed where `#` is the number of cpus to use.
* Note: When running this test, the `--basetemp` directory should be set to a unique
existing directory to avoid deleting previous test output.
* The default master list exists in the tests/hap directory and contains 50 datasets. The
full master list of thousands of datasets resides in Artifactory as ACSWFC3List.csv
(https://bytesalad.stsci.edu/artifactory/hst-hla-pipeline/dev/master_lists).
"""
print("TEST_RANDOM. Dataset: ", dataset)
output_name = dataset + '.ecsv'
current_dt = datetime.datetime.now()
print(str(current_dt))
subdir = ""
prevdir = os.getcwd()
# create working directory specified for the test
if not tmpdir.ensure(subdir, dir=True):
curdir = tmpdir.mkdir(subdir).strpath
else:
curdir = tmpdir.join(subdir).strpath
os.chdir(curdir)
try:
align_table = alignimages.perform_align([dataset],
catalog_list=['GAIADR2', 'GAIADR1'],
num_sources=250,
archive=False,
clobber=True, debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False,
product_type='pipeline')
# The align_table could be None if there were not enough sources
# available for alignment. This is not an error.
if align_table is None:
mark = pytest.mark.xfail(reason="TEST_RANDOM. There were not enough sources for processing for dataset.")
request.node.add_marker(mark)
assert 0
else:
dataset_table = align_table.filtered_table
# Filtered datasets
if dataset_table['doProcess'].sum() == 0:
pytest.skip("TEST_RANDOM. Filtered Dataset: {}.".format(dataset))
# Datasets to process
elif dataset_table['doProcess'].sum() > 0:
# Determine images in dataset to be processed and the number of images
# This is in case an image was filtered out (e.g., expotime = 0)
index = np.where(dataset_table['doProcess'] == 1)[0]
fit_qual = dataset_table['fit_qual'][index[0]]
# Update the table with the dataset_key which is really just a counter
dataset_table['completed'][:] = True
dataset_table.write(output_name, format='ascii.ecsv')
if fit_qual > 4:
mark = pytest.mark.xfail(reason="TEST_RANDOM. Unsuccessful Dataset (fit_qual = 5).")
request.node.add_marker(mark)
assert 0
else:
assert 0 < fit_qual <= 4
# Catch anything that happens as this dataset will be considered a failure, but
# the processing of datasets should continue. This is meant to catch
# unexpected errors and generate sufficient output exception
# information so algorithmic problems can be addressed.
except Exception as except_details:
print(except_details)
pytest.fail("TEST_RANDOM. Exception Dataset: {}\n", dataset)
finally:
# Perform some clean up
if os.path.isfile('ref_cat.ecsv'):
os.remove('ref_cat.ecsv')
if os.path.isfile('refcatalog.cat'):
os.remove('refcatalog.cat')
for filename in os.listdir():
if filename.endswith('flt.fits') or filename.endswith('flc.fits'):
os.remove(filename)
# Return to original directory
os.chdir(prevdir)