def test_astrometric_singleton(self, dataset_names):
""" Tests pipeline-style processing of a singleton exposure using runastrodriz.
"""
# Get sample data through astroquery
flcfile = aqutils.retrieve_observation(dataset_names,
suffix=['FLC'])[0]
fltfile = aqutils.retrieve_observation(dataset_names,
suffix=['FLT'])[0]
rawfile = aqutils.retrieve_observation(dataset_names,
suffix=['RAW'])[0]
# Retrieve reference files for these as well
self.get_input_file('', fltfile, docopy=False)
# Insure environment variables are set for full processing
os.environ['ASTROMETRY_STEP_CONTROL'] = 'on'
os.environ['ASTROMETRY_COMPUTE_APOSTERIORI'] = 'on'
os.environ['ASTROMETRY_APPLY_APRIORI'] = 'on'
# Run pipeline processing using
runastrodriz.process(rawfile, force=True, inmemory=True)
# compare WCSNAMEs from flt and flc files
flc_wcsname = fits.getval(flcfile, 'wcsname', ext=1)
flt_wcsname = fits.getval(fltfile, 'wcsname', ext=1)
# Perform comparisons:
# - WCSNAME values should contain '-' from either a priori or a posteriori solution
# - WCSNAME value should be the same for FLT and FLC images
assert ('-' in flc_wcsname)
assert ('-' in flt_wcsname)
assert (flc_wcsname == flt_wcsname)
def test_alignpipe_randomlist(tmpdir, dataset):
""" Tests which validate whether mosaics can be aligned to an astrometric standard.
Characteristics of these tests:
* A reference WCS is generated based upon all the input images for
the field.
* A source astrometric catalog is created using the Photutils
package to detect explicitly sources in the images.
* An astrometric catalog is created to extract astrometric positions
for the found sources in the input images' field-of-view using
GAIADR2 (preferred) or GAIADR1.
* Cross matching/fitting is done between found sources and catalog
coordinates with the Tweakwcs package.
* The quality of the fit is evaluated against a minimum threshold and
potentially another fit algorithm is invoked or an alternative
catalog is used in an effort to obtain a better quality fit.
* If the option is set, the WCS information is updated for the
input exposures. The default is False.
* No mosaic is generated.
* An output table containing characterizations of the process and
associated fit is generated.
Success Criteria:
* Success criterion hard-coded for this test represents whether a
statistical sample (70%) of ACS and WFC3 datasets were able to be
aligned to within 10mas RMS.
* RMS values are extracted from the table output from `perform_align`
* This criterion will need to be determined by the user after the test has been
run based on how many datasets were run and skipped.
The input master_list CSV file is
output from a database and lists associations and singletons for ACS
and WFC3 instruments randomly sorted. The actual data files are
downloaded from MAST via astroquery.
This test file can be executed in the following manner:
$ pytest -n # -s --basetemp=/internal/hladata/yourUniqueDirectoryHere --bigdata --slow
--master_list ACSWFC3ListDefault50.csv --start_row 0 --num_rows 50 test_pipe_randomlist.py >&
test_pipe_random_output.txt &
$ tail -f test_pipe_random_output.txt
* The `-n #` option can be used to run tests in parallel if `pytest-xdist` has
been installed where `#` is the number of cpus to use.
* Note: When running this test, the `--basetemp` directory should be set to a unique
existing directory to avoid deleting previous test output.
* The default master list exists in the tests/hla directory and contains 50 datasets. The
full master list of thousands of datasets resides in Artifactory as ACSWFC3List.csv
(https://bytesalad.stsci.edu/artifactory/hst-hla-pipeline/dev/master_lists).
"""
# Start by resetting the logging for all the modules used
rl = logging.getLogger('stwcs.wcsutil.headerlet')
if len(rl.handlers) > 1: del rl.handlers[-1]
print("TEST_RANDOM_ALIGN. Dataset: ", dataset)
output_name = dataset + '.ecsv'
current_dt = datetime.datetime.now()
print(str(current_dt))
subdir = ""
prevdir = os.getcwd()
# create working directory specified for the test
if not tmpdir.ensure(subdir, dir=True):
curdir = tmpdir.mkdir(subdir).strpath
else:
curdir = tmpdir.join(subdir).strpath
os.chdir(curdir)
return_value = 1
try:
# The dataset name is an ipppssoot only.
dataset = dataset.lower()
# Download the data necessary for processing - The dataset list contains only the ipppssoot of a
# singleton or an ASN. Need to get the FLT/FLC files as runastrodriz expects them to be present.
input_parameter = "RAW"
filename_for_runastrodriz = dataset.lower() + "_raw.fits"
if dataset.endswith("0"):
input_parameter = "ASN"
filename_for_runastrodriz = dataset.lower() + "_asn.fits"
log.info("Input parameter: {}".format(input_parameter))
files_on_disk = check_disk_get_data([dataset], suffix=input_parameter, archive=False, clobber=False)
log.info("Dataset: {}".format(dataset))
if os.path.exists('mastDownload'):
shutil.rmtree('mastDownload')
log.info("\nFiles: {}".format(files_on_disk))
# Insure environment variables are set for full processing
os.environ['ASTROMETRY_STEP_CONTROL'] = 'on'
os.environ['ASTROMETRY_COMPUTE_APOSTERIORI'] = 'on'
os.environ['ASTROMETRY_APPLY_APRIORI'] = 'on'
flts = sorted(glob.glob('*fl?.fits'))
jref_dir = 'jref.old/' if '16r12191j_mdz' in fits.getval(flts[0], 'mdriztab') else 'jref/'
os.environ['jref'] = os.path.join(os.environ['crrefer'], jref_dir)
log.info("JREF: {}".format(os.environ['jref']))
# Run pipeline processing using
# runastrodriz accepts *_raw.fits or *_asn.fits, but it assumes the *_fl[t|c].fits files
# are also present
runastrodriz.process(filename_for_runastrodriz, force=True)
return_value = 0
# Catch anything that happens as this dataset will be considered a failure, but
# the processing of datasets should continue. This is meant to catch
# unexpected errors and generate sufficient output exception
# information so algorithmic problems can be addressed.
except Exception as except_details:
traceback.print_exc()
pytest.fail("TEST_RANDOM. Exception Dataset: {}\n", dataset)
return_value = 1
"""
finally:
# Perform some clean up
if os.path.isfile('ref_cat.ecsv'):
os.remove('ref_cat.ecsv')
if os.path.isfile('refcatalog.cat'):
os.remove('refcatalog.cat')
for filename in os.listdir():
if filename.endswith('flt.fits') or filename.endswith('flc.fits'):
os.remove(filename)
"""
assert return_value == 0
# Return to original directory
os.chdir(prevdir)