def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characteristics of this test:
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
totalRMS = 0.0
input_filenames = ['iaal01hxq_flc.fits', 'iaala3btq_flc.fits',
'iaal01hyq_flc.fits', 'iaala3bsq_flc.fits',
'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits']
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','mosaic_ngc188',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
totalRMS = 0.0
input_filenames = ['ib6v06c4q_flc.fits','ib6v06c7q_flc.fits',
'ib6v25aqq_flc.fits','ib6v25atq_flc.fits',
'jddh02gjq_flc.fits','jddh02glq_flc.fits',
'jddh02goq_flc.fits']
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','mosaic_47tuc',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits',
'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits',
'jddh02goq_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_47tuc', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def align_to_gaia(self):
"""Extract the flt/flc filenames from the exposure product list, as
well as the corresponding headerlet filenames to use legacy alignment
routine.
"""
exposure_filenames = []
headerlet_filenames = {}
align_table = None
try:
if self.edp_list:
for edp in self.edp_list:
exposure_filenames.append(edp.full_filename)
headerlet_filenames[edp.full_filename] = edp.headerlet_filename
align_table = alignimages.perform_align(exposure_filenames,
debug=False,
runfile="alignimages.log",
update_hdr_wcs=True,
headerlet_filenames=headerlet_filenames)
except Exception:
# TODO: Fix up the logging
# Report a problem with the alignment
# log.info("EXCEPTION encountered in alignimages.\n")
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
# log.info("No correction to absolute astrometric frame applied.\n")
align_table = None
# Return a table which contains data regarding the alignment, as well as the
# list of the flt/flc exposures which were part of the alignment process
# TODO: This does not account for individual exposures which might have been
# excluded from alignment.
return align_table, exposure_filenames
def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characteristics of this test:
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
total_rms = 0.0
input_filenames = [
'iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits',
'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits'
]
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_ngc188', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_fail_single_visit(self):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include exposures from a number of single visit datasets to explore what impact differing
observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry.
This test is known to fail due to "RuntimeError: Number of output coordinates exceeded allocation (475)a".
It will exercise the code using both catalogs for each of the three fitting algorithms at this time. Nans
will be present in the output table.
The following datasets are used in these tests:
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame WFC3/UVIS images
"""
totalRMS = 0.0
input_filenames = ['ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits',
'ibjt01aiq_flt.fits', 'ibjt01amq_flt.fits',
'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits',
'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits',
'ibjt01zwq_flc.fits', 'ibjt01a4q_flc.fits',
'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits',
'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits',
'ibjt01avq_flt.fits', 'ibjt01yuq_flc.fits',
'ibjt01ztq_flc.fits'],
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','base_tests',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_fail_single_visit(self):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of this test:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry. This test is known to fail due to "RuntimeError: Number
of output coordinates exceeded allocation (475)a". It will exercise the code using both
catalogs for each of the three fitting algorithms at this time. Nans will be present
in the output table.
The following datasets are used in these tests:
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame
WFC3/UVIS images
"""
total_rms = 0.0
input_filenames = [
'ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits', 'ibjt01aiq_flt.fits',
'ibjt01amq_flt.fits', 'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits',
'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits', 'ibjt01zwq_flc.fits',
'ibjt01a4q_flc.fits', 'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits',
'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits', 'ibjt01avq_flt.fits',
'ibjt01yuq_flc.fits', 'ibjt01ztq_flc.fits'
],
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_astroquery(self):
"""Verify that new astroquery interface will work"""
totalRMS = 0.0
try:
datasetTable = alignimages.perform_align(['IB6V06060'],archive=False,clobber=True,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
sys.exit()
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_astroquery(self):
"""Verify that new astroquery interface will work"""
total_rms = 0.0
dataset_table = alignimages.perform_align(['IB6V06060'],
archive=False,
clobber=True,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_single_visits(self, input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what
impact differing observing modes (differing instruments, detectors, filters, subarray
size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W,
1x F606W, 1x F625W, 2x F658N, 1x F775W, 1x F814W, 1x F850LP, and
2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
total_rms = 0.0
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file)
dataset_table = alignimages.perform_align(input_filenames,
archive=False,
clobber=False,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised
# information
if dataset_table:
total_rms = dataset_table['total_rms'][0]
assert 0.0 < total_rms <= RMS_LIMIT
def test_align_single_visits(self,input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what impact differing
observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W, 1x F606W, 1x F625W,
2x F658N, 1x F775W, 1x F814W, 1x F850LP, and 2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
totalRMS = 0.0
# Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from
# MAST, get the data now using ci_watson.
for input_file in input_filenames:
get_bigdata('hst-hla-pipeline','dev','base_tests',input_file)
try:
datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Examine the output table to extract the RMS for the entire fit and the compromised information
if datasetTable:
totalRMS = datasetTable['total_rms'][0]
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
assert (0.0 < totalRMS <= RMS_LIMIT)
def test_randomlist(tmpdir, dataset):
""" Tests which validate whether mosaics can be aligned to an astrometric standard.
Characteristics of these tests:
* A reference WCS is generated based upon all the input images for
the field.
* A source astrometric catalog is created using the Photutils
package to detect explicitly sources in the images.
* An astrometric catalog is created to extract astrometric positions
for the found sources in the input images' field-of-view using
GAIADR2 (preferred) or GAIADR1.
* Cross matching/fitting is done between found sources and catalog
coordinates with the Tweakwcs package.
* The quality of the fit is evaluated against a minimum threshold and
potentially another fit algorithm is invoked or an alternative
catalog is used in an effort to obtain a better quality fit.
* If the option is set, the WCS information is updated for the
input exposures. The default is False.
* No mosaic is generated.
* An output table containing characterizations of the process and
associated fit is generated.
Success Criteria:
* Success criterion hard-coded for this test represents whether a
statistical sample (70%) of ACS and WFC3 datasets were able to be
aligned to within 10mas RMS.
* RMS values are extracted from the table output from `perform_align`
* This criterion will need to be determined by the user after the test has been
run based on how many datasets were run and skipped.
The input master_list CSV file is
output from a database and lists associations and singletons for ACS
and WFC3 instruments randomly sorted. The actual data files are
downloaded from MAST via astroquery.
This test file can be executed in the following manner:
$ pytest -n # -s --basetemp=/internal/hladata/yourUniqueDirectoryHere --bigdata --slow
--master_list ACSWFC3ListDefault50.csv --start_row 0 --num_rows 50 test_randomlist.py >&
test_random_output.txt &
$ tail -f test_random_output.txt
* The `-n #` option can be used to run tests in parallel if `pytest-xdist` has
been installed where `#` is the number of cpus to use.
* Note: When running this test, the `--basetemp` directory should be set to a unique
existing directory to avoid deleting previous test output.
* The default master list exists in the tests/hla directory and contains 50 datasets. The
full master list of thousands of datasets resides in Artifactory as ACSWFC3List.csv
(https://bytesalad.stsci.edu/artifactory/hst-hla-pipeline/dev/master_lists).
"""
print("TEST_RANDOM. Dataset: ", dataset)
output_name = dataset + '.ecsv'
current_dt = datetime.datetime.now()
print(str(current_dt))
subdir = ""
prevdir = os.getcwd()
# create working directory specified for the test
if not tmpdir.ensure(subdir, dir=True):
curdir = tmpdir.mkdir(subdir).strpath
else:
curdir = tmpdir.join(subdir).strpath
os.chdir(curdir)
try:
dataset_table = alignimages.perform_align([dataset],
archive=False,
clobber=True,
debug=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False,
output=False)
# Filtered datasets
if dataset_table['doProcess'].sum() == 0:
pytest.skip("TEST_RANDOM. Filtered Dataset: {}.".format(dataset))
# Datasets to process
elif dataset_table['doProcess'].sum() > 0:
# Determine images in dataset to be processed and the number of images
# This is in case an image was filtered out (e.g., expotime = 0)
index = np.where(dataset_table['doProcess'] == 1)[0]
fit_qual = dataset_table['fit_qual'][index[0]]
# Update the table with the dataset_key which is really just a counter
dataset_table['completed'][:] = True
dataset_table.write(output_name, format='ascii.ecsv')
if fit_qual > 4:
pytest.fail(
"TEST_RANDOM. Unsuccessful Dataset (fit_qual = 5): {}.".
format(dataset))
else:
assert 0 < fit_qual <= 4
# Catch anything that happens as this dataset will be considered a failure, but
# the processing of datasets should continue. This is meant to catch
# unexpected errors and generate sufficient output exception
# information so algorithmic problems can be addressed.
except Exception as except_details:
print(except_details)
pytest.fail("TEST_RANDOM. Exception Dataset: {}\n", dataset)
finally:
# Perform some clean up
if os.path.isfile('ref_cat.ecsv'):
os.remove('ref_cat.ecsv')
if os.path.isfile('refcatalog.cat'):
os.remove('refcatalog.cat')
for filename in os.listdir():
if filename.endswith('flt.fits') or filename.endswith('flc.fits'):
os.remove(filename)
# Return to original directory
os.chdir(prevdir)
def align_randomFields(self, randomTable):
""" Process randomly selected fields (aka datasets) stored in an Astropy table.
Each field is used as input to determine if it can be aligned to an
astrometric standard. The success or fail status for each test is retained
as the overall success or fail statistic is the necessary output from
this test.
"""
numSuccess = 0
numUnsuccessful = 0
numException = 0
numProcessedDatasets = 0
# Read the table and extract a list of each dataset name in IPPSSOOT format
# which is either an association ID or an individual filename
dataset_list = get_dataset_list(randomTable)
numProcessedDatasets = len(dataset_list)
numStartTests = numProcessedDatasets
# Process the dataset names in the list
#
# If the dataset name represents an association ID, the multiplicity
# of images within the association need to be processed. Otherwise,
# the dataset is a single image.
#
# If the "alignment" of a field/dataset fails for any reason, trap
# the exception and keep going.
allDatasetTable = Table()
datasetKey = -1
print("TEST_RANDOM. Dataset List: ", dataset_list)
for dataset in dataset_list:
datasetKey += 1
outputName = dataset + '.ecsv'
print("TEST_RANDOM. Dataset: ", dataset)
currentDT = datetime.datetime.now()
print(str(currentDT))
try:
datasetTable = alignimages.perform_align([dataset],archive=False,clobber=True,debug=False,
update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False)
# Filtered datasets
if datasetTable['doProcess'].sum() == 0:
print("TEST_RANDOM. Filtered Dataset: ", dataset, "\n")
numProcessedDatasets -= 1;
# Datasets to process
elif datasetTable['doProcess'].sum() > 0:
# Determine images in dataset to be processed and the number of images
# This is in case an image was filtered out (e.g., expotime = 0)
index = np.where(datasetTable['doProcess']==1)[0]
sumOfStatus = datasetTable['status'][index].sum()
# Update the table with the datasetKey which is really just a counter
datasetTable['datasetKey'][:] = datasetKey
datasetTable['completed'][:] = True
datasetTable.write(outputName, format='ascii.ecsv')
#datasetTable.pprint(max_width=-1)
# Successful datasets
if (sumOfStatus == 0):
print("TEST_RANDOM. Successful Dataset: ", dataset, "\n")
numSuccess += 1
# Unsuccessful datasets
else:
print("TEST_RANDOM. Unsuccessful Dataset: ", dataset, "\n")
numUnsuccessful += 1
# Append the latest dataset table to the summary table
allDatasetTable = vstack([allDatasetTable, datasetTable])
# Catch anything that happens as this dataset will be considered a failure, but
# the processing of datasets should continue. Generate sufficient output exception
# information so problems can be addressed.
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
print("TEST_RANDOM. Exception Dataset: ", dataset, "\n")
numException += 1
continue
# Perform some clean up
if os.path.isfile('ref_cat.ecsv'):
os.remove('ref_cat.ecsv')
if os.path.isfile('refcatalog.cat'):
os.remove('refcatalog.cat')
for filename in os.listdir():
if filename.endswith('flt.fits') or filename.endswith('flc.fits'):
os.unlink(filename)
# Write out the table
allDatasetTable.write('resultsBigTest.ecsv', format='ascii.ecsv')
# Determine the percent success over all datasets processed
percentSuccess = numSuccess/numProcessedDatasets
print('TEST_RANDOM. Number of tests started: ', numStartTests)
print('TEST_RANDOM. Number of tests (excluding filtered): ', numProcessedDatasets)
print('TEST_RANDOM. Number of successful tests: ', numSuccess)
print('TEST_RANDOM. Number of unsuccessful tests: ', numUnsuccessful)
print('TEST_RANDOM. Number of exception tests: ', numException)
print('TEST_RANDOM. Percentage success/numberOfTests: ', numSuccess/numProcessedDatasets*100.0)
return percentSuccess
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None,
headerlets=True, align_to_gaia=True):
""" Run astrodrizzle on input file/ASN table
using default values for astrodrizzle parameters.
"""
# We only need to import this package if a user run the task
import drizzlepac
from drizzlepac import processInput # used for creating new ASNs for _flc inputs
from stwcs import updatewcs
from drizzlepac import alignimages
# interpret envvar variable, if specified
if envvar_compute_name in os.environ:
val = os.environ[envvar_compute_name].lower()
if val not in envvar_bool_dict:
msg = "ERROR: invalid value for {}.".format(envvar_compute_name)
msg += " \n Valid Values: on, off, yes, no, true, false"
raise ValueError(msg)
align_to_gaia = envvar_bool_dict[val]
if envvar_new_apriori_name in os.environ:
# Reset ASTROMETRY_STEP_CONTROL based on this variable
# This provides backward-compatibility until ASTROMETRY_STEP_CONTROL
# gets removed entirely.
val = os.environ[envvar_new_apriori_name].lower()
if val not in envvar_dict:
msg = "ERROR: invalid value for {}.".format(envvar_new_apriori_name)
msg += " \n Valid Values: on, off, yes, no, true, false"
raise ValueError(msg)
os.environ[envvar_old_apriori_name] = envvar_dict[val]
if headerlets or align_to_gaia:
from stwcs.wcsutil import headerlet
# Open the input file
try:
# Make sure given filename is complete and exists...
inFilename = fileutil.buildRootname(inFile,ext=['.fits'])
if not os.path.exists(inFilename):
print("ERROR: Input file - %s - does not exist." % inFilename)
return
except TypeError:
print("ERROR: Inappropriate input file.")
return
#If newpath was specified, move all files to that directory for processing
if newpath:
orig_processing_dir = os.getcwd()
new_processing_dir = _createWorkingDir(newpath,inFilename)
_copyToNewWorkingDir(new_processing_dir,inFilename)
os.chdir(new_processing_dir)
# Initialize for later use...
_mname = None
_new_asn = None
_calfiles = []
# Identify WFPC2 inputs to account for differences in WFPC2 inputs
wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2'
cal_ext = None
# Check input file to see if [DRIZ/DITH]CORR is set to PERFORM
if '_asn' in inFilename:
# We are working with an ASN table.
# Use asnutil code to extract filename
inFilename = _lowerAsn(inFilename)
_new_asn = [inFilename]
_asndict = asnutil.readASNTable(inFilename,None,prodonly=False)
_cal_prodname = _asndict['output'].lower()
#_fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits'])
# Retrieve the first member's rootname for possible use later
_fimg = fits.open(inFilename, memmap=False)
for name in _fimg[1].data.field('MEMNAME'):
if name[-1] != '*':
_mname = name.split('\0', 1)[0].lower()
break
_fimg.close()
del _fimg
else:
# Check to see if input is a _RAW file
# If it is, strip off the _raw.fits extension...
_indx = inFilename.find('_raw')
if _indx < 0: _indx = len(inFilename)
# ... and build the CALXXX product rootname.
if wfpc2_input:
# force code to define _c0m file as calibrated product to be used
cal_ext = ['_c0m.fits']
_mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext)
_cal_prodname = inFilename[:_indx]
# Reset inFilename to correspond to appropriate input for
# drizzle: calibrated product name.
inFilename = _mname
if _mname is None:
errorMsg = 'Could not find calibrated product!'
raise Exception(errorMsg)
# Create trailer filenames based on ASN output filename or
# on input name for single exposures
if '_raw' in inFile:
# Output trailer file to RAW file's trailer
_trlroot = inFile[:inFile.find('_raw')]
elif '_asn' in inFile:
# Output trailer file to ASN file's trailer, not product's trailer
_trlroot = inFile[:inFile.find('_asn')]
else:
# Default: trim off last suffix of input filename
# and replacing with .tra
_indx = inFile.rfind('_')
if _indx > 0:
_trlroot = inFile[:_indx]
else:
_trlroot = inFile
_trlfile = _trlroot + '.tra'
# Open product and read keyword value
# Check to see if product already exists...
dkey = 'DRIZCORR'
# ...if product does NOT exist, interrogate input file
# to find out whether 'dcorr' has been set to PERFORM
# Check if user wants to process again regardless of DRIZCORR keyword value
if force:
dcorr = 'PERFORM'
else:
if _mname :
_fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False)
_phdr = _fimg['PRIMARY'].header
if dkey in _phdr:
dcorr = _phdr[dkey]
else:
dcorr = None
_fimg.close()
del _fimg
else:
dcorr = None
time_str = _getTime()
_tmptrl = _trlroot + '_tmp.tra'
_drizfile = _trlroot + '_pydriz'
_drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle
_alignlog = _trlroot + '_align.log'
if dcorr == 'PERFORM':
if '_asn.fits' not in inFilename:
# Working with a singleton
# However, we always want to make sure we always use
# a calibrated product as input, if available.
_infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext)
_infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits'])
_cal_prodname = _infile
_inlist = _calfiles = [_infile]
# Add CTE corrected filename as additional input if present
if os.path.exists(_infile_flc) and _infile_flc != _infile:
_inlist.append(_infile_flc)
else:
# Working with an ASN table...
_infile = inFilename
flist,duplist = processInput.checkForDuplicateInputs(_asndict['order'])
_calfiles = flist
if len(duplist) > 0:
origasn = processInput.changeSuffixinASN(inFilename,'flt')
dupasn = processInput.changeSuffixinASN(inFilename,'flc')
_inlist = [origasn,dupasn]
else:
_inlist = [_infile]
# We want to keep the original specification of the calibration
# product name, though, not a lower-case version...
_cal_prodname = inFilename
_new_asn.extend(_inlist) # kept so we can delete it when finished
# check to see whether FLC files are also present, and need to be updated
# generate list of FLC files
align_files = None
_calfiles_flc = [f.replace('_flt.fits','_flc.fits') for f in _calfiles]
# insure these files exist, if not, blank them out
# Also pick out what files will be used for additional alignment to GAIA
if not os.path.exists(_calfiles_flc[0]):
_calfiles_flc = None
align_files = _calfiles
align_update_files = None
else:
align_files = _calfiles_flc
align_update_files = _calfiles
# Run updatewcs on each list of images
updatewcs.updatewcs(_calfiles)
if _calfiles_flc:
updatewcs.updatewcs(_calfiles_flc)
if align_to_gaia:
# Perform additional alignment on the FLC files, if present
###############
#
# call hlapipeline code here on align_files list of files
#
###############
# Create trailer marker message for start of align_to_GAIA processing
_trlmsg = _timestamp("Align_to_GAIA started ")
print(_trlmsg)
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_trlmsg = ""
# Create an empty astropy table so it can be used as input/output for the perform_align function
#align_table = Table()
try:
align_table = alignimages.perform_align(align_files,update_hdr_wcs=True, runfile=_alignlog)
for row in align_table:
if row['status'] == 0:
trlstr = "Successfully aligned {} to {} astrometric frame\n"
_trlmsg += trlstr.format(row['imageName'], row['catalog'])
else:
trlstr = "Could not align {} to absolute astrometric frame\n"
_trlmsg += trlstr.format(row['imageName'])
except Exception:
# Something went wrong with alignment to GAIA, so report this in
# trailer file
_trlmsg = "EXCEPTION encountered in alignimages...\n"
_trlmsg += " No correction to absolute astrometric frame applied!\n"
# Write the perform_align log to the trailer file...(this will delete the _alignlog)
_appendTrlFile(_trlfile,_alignlog)
# Append messages from this calling routine post-perform_align
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_trlmsg = ""
#Check to see whether there are any additional input files that need to
# be aligned (namely, FLT images)
if align_update_files and align_table:
# Apply headerlets from alignment to FLT version of the files
for fltfile, flcfile in zip(align_update_files, align_files):
row = align_table[align_table['imageName']==flcfile]
headerletFile = row['headerletFile'][0]
if headerletFile != "None":
headerlet.apply_headerlet_as_primary(fltfile, headerletFile,
attach=True, archive=True)
# append log file contents to _trlmsg for inclusion in trailer file
_trlstr = "Applying headerlet {} as Primary WCS to {}\n"
_trlmsg += _trlstr.format(headerletFile, fltfile)
else:
_trlmsg += "No absolute astrometric headerlet applied to {}\n".format(fltfile)
# Finally, append any further messages associated with alignement from this calling routine
_trlmsg += _timestamp('Align_to_GAIA completed ')
print(_trlmsg)
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# Run astrodrizzle and send its processing statements to _trlfile
_pyver = drizzlepac.astrodrizzle.__version__
for _infile in _inlist: # Run astrodrizzle for all inputs
# Create trailer marker message for start of astrodrizzle processing
_trlmsg = _timestamp('astrodrizzle started ')
_trlmsg += __trlmarker__
_trlmsg += '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver)
print(_trlmsg)
# Write out trailer comments to trailer file...
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
_pyd_err = _trlroot+'_pydriz.stderr'
try:
b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile,
configobj='defaults',in_memory=inmemory,
num_cores=num_cores, **pipeline_pars)
except Exception as errorobj:
_appendTrlFile(_trlfile,_drizlog)
_appendTrlFile(_trlfile,_pyd_err)
_ftrl = open(_trlfile,'a')
_ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile)
_ftrl.write(str(sys.exc_info()[0])+': ')
_ftrl.writelines(str(errorobj))
_ftrl.write('\n')
_ftrl.close()
print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile)
raise Exception(str(errorobj))
# Now, append comments created by PyDrizzle to CALXXX trailer file
print('Updating trailer file %s with astrodrizzle comments.' % _trlfile)
_appendTrlFile(_trlfile,_drizlog)
# Save this for when astropy.io.fits can modify a file 'in-place'
# Update calibration switch
_fimg = fits.open(_cal_prodname, mode='update', memmap=False)
_fimg['PRIMARY'].header[dkey] = 'COMPLETE'
_fimg.close()
del _fimg
# Enforce pipeline convention of all lower-case product
# names
_prodlist = glob.glob('*drz.fits')
for _prodname in _prodlist:
_plower = _prodname.lower()
if _prodname != _plower: os.rename(_prodname,_plower)
else:
# Create default trailer file messages when astrodrizzle is not
# run on a file. This will typically apply only to BIAS,DARK
# and other reference images.
# Start by building up the message...
_trlmsg = _timestamp('astrodrizzle skipped ')
_trlmsg = _trlmsg + __trlmarker__
_trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename)
_trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n'
print(_trlmsg)
# Write message out to temp file and append it to full trailer file
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# Append final timestamp to trailer file...
_final_msg = '%s: Finished processing %s \n' % (time_str,inFilename)
_final_msg += _timestamp('astrodrizzle completed ')
_trlmsg += _final_msg
ftmp = open(_tmptrl,'w')
ftmp.writelines(_trlmsg)
ftmp.close()
_appendTrlFile(_trlfile,_tmptrl)
# If we created a new ASN table, we need to remove it
if _new_asn is not None:
for _name in _new_asn: fileutil.removeFile(_name)
# Clean up any generated OrIg_files directory
if os.path.exists("OrIg_files"):
# check to see whether this directory is empty
flist = glob.glob('OrIg_files/*.fits')
if len(flist) == 0:
os.rmdir("OrIg_files")
else:
print('OrIg_files directory NOT removed as it still contained images...')
# If headerlets have already been written out by alignment code,
# do NOT write out this version of the headerlets
if headerlets:
# Generate headerlets for each updated FLT image
hlet_msg = _timestamp("Writing Headerlets started")
for fname in _calfiles:
frootname = fileutil.buildNewRootname(fname)
hname = "%s_flt_hlet.fits"%frootname
# Write out headerlet file used by astrodrizzle, however,
# do not overwrite any that was already written out by alignimages
if not os.path.exists(hname):
hlet_msg += "Created Headerlet file %s \n"%hname
try:
headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY',
author="OPUS",descrip="Default WCS from Pipeline Calibration",
attach=False,clobber=True,logging=False)
except ValueError:
hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname)
# update trailer file to log creation of headerlet files
hlet_msg += _timestamp("Writing Headerlets completed")
ftrl = open(_trlfile,'a')
ftrl.write(hlet_msg)
ftrl.close()
# If processing was done in a temp working dir, restore results to original
# processing directory, return to original working dir and remove temp dir
if newpath:
_restoreResults(new_processing_dir,orig_processing_dir)
os.chdir(orig_processing_dir)
_removeWorkingDir(new_processing_dir)
# Provide feedback to user
print(_final_msg)