def test_generate_catalog(self,input_filenames, truth_file):
""" Verify whether sources from astrometric catalogs can be extracted from images.
Success Criteria
-----------------
* Initially, source catalog matches >80% of 'truth' catalog sources
"""
self.input_loc = 'catalog_tests'
self.curdir = os.getcwd()
truth_path = [self.input_repo, self.tree, self.input_loc, *self.ref_loc]
if not isinstance(input_filenames, list):
input_filenames = [input_filenames]
try:
# Make local copies of input files
local_files = []
for infile in input_filenames:
downloaded_files = self.get_input_file(infile, docopy=True)
local_files.extend(downloaded_files)
test_image = local_files[0]
print("Testing with {}".format(test_image))
imghdu = fits.open(test_image)
instrume = imghdu[0].header['instrume'].lower()
detector = imghdu[0].header['detector'].lower()
instr_pars = detector_specific_params[instrume][detector]
reference_wcs = amutils.build_reference_wcs(local_files)
imcat = amutils.generate_sky_catalog(imghdu, reference_wcs, **instr_pars)
imcat.rename_column('xcentroid', 'x')
imcat.rename_column('ycentroid', 'y')
# create FITS WCS corrector object
wcs_corrector = tweakwcs.FITSWCS(reference_wcs)
# get reference catalog as 'truth' files
reference_catalog = get_bigdata(*truth_path, truth_file, docopy=True)
if os.path.basename(reference_catalog).endswith('ecsv'):
tab_format = 'ascii.ecsv'
else:
tab_format = 'ascii.fast_commented_header'
reference_table = Table.read(reference_catalog, format=tab_format)
num_expected = len(reference_table)
# Perform matching
match = tweakwcs.TPMatch(searchrad=200, separation=0.1, tolerance=5, use2dhist=True)
ridx, iidx = match(reference_table, imcat, wcs_corrector)
nmatches = len(ridx)
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout)
sys.exit()
assert (nmatches > 0.8*num_expected)
def test_astroquery(self):
"""Verify that new astroquery interface will work"""
self.curdir = os.getcwd()
self.input_loc = ''
shift_file, local_files = self.run_align('ib6v06060')
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
reference_wcs = amutils.build_reference_wcs(local_files)
test_limit = self.fit_limit / reference_wcs.pscale
assert (rms_x <= test_limit and rms_y <= test_limit)
def test_align_47tuc(self):
""" Verify whether 47Tuc exposures can be aligned to an astrometric standard.
Characeteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of 47Tuc suitable for creating a combined mosaic using both instruments.
"""
self.input_loc = 'mosaic_47tuc'
input_filenames = [
'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits',
'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits',
'jddh02goq_flc.fits'
]
self.output_shift_file = 'test_mosaic_47tuc_shifts.txt'
shift_file, local_files = self.run_align(input_filenames)
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
reference_wcs = amutils.build_reference_wcs(local_files)
test_limit = self.fit_limit / reference_wcs.pscale
assert (rms_x <= test_limit and rms_y <= test_limit)
def test_align_ngc188(self):
""" Verify whether NGC188 exposures can be aligned to an astrometric standard.
Characeteristics of this test:
* Input exposures include both ACS and WFC3 images of the same general field-of-view
of NGC188 suitable for creating a combined mosaic using both instruments.
"""
self.input_loc = 'mosaic_ngc188'
input_filenames = [
'iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits',
'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits',
'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits'
]
self.output_shift_file = 'test_mosaic_ngc188_shifts.txt'
shift_file, local_files = self.run_align(input_filenames)
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
reference_wcs = amutils.build_reference_wcs(local_files)
test_limit = self.fit_limit / reference_wcs.pscale
assert (rms_x <= test_limit and rms_y <= test_limit)
def test_align_single_visits(self, input_filenames):
""" Verify whether single-visit exposures can be aligned to an astrometric standard.
Characteristics of these tests:
* Input exposures include exposures from a number of single visit datasets to explore what impact differing
observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry.
The following datasets are used in these tests:
* ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W, 1x F606W, 1x F625W,
2x F658N, 1x F775W, 1x F814W, 1x F850LP, and 2x F892N ACS/HRC images
* ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images
* ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images
* WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images
* WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame WFC3/UVIS images
* WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images
"""
self.input_loc = 'base_tests'
self.curdir = os.getcwd()
try:
shift_file, local_files = self.run_align(input_filenames)
x_shift = numpy.alltrue(numpy.isnan(shift_file['col2']))
y_shift = numpy.alltrue(numpy.isnan(shift_file['col3']))
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_tb,
file=sys.stdout)
sys.exit()
reference_wcs = amutils.build_reference_wcs(local_files)
test_limit = self.fit_limit / reference_wcs.pscale
assert (x_shift == False and y_shift == False and rms_x <= test_limit
and rms_y <= test_limit)
def perform_align(input_list,
archive=False,
clobber=False,
update_hdr_wcs=False):
"""Main calling function.
Parameters
----------
input_list : list
List of one or more IPPSSOOTs (rootnames) to align.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories?
clobber : Boolean
Download and overwrite existing local copies of input files?
update_hdr_wcs : Boolean
Write newly computed WCS information to image image headers?
Returns
-------
int value 0 if successful, int value 1 if unsuccessful
"""
# Define astrometric catalog list in priority order
catalogList = ['GAIADR2', 'GSC241']
numCatalogs = len(catalogList)
# 0: print git info
print(
"-------------------- STEP 0: Display Git revision info --------------------"
)
full_path = os.path.dirname(__file__) + "/utils"
repo_path = None
if "hlapipeline/hlapipeline" in full_path:
repo_path = full_path.split(
"hlapipeline/hlapipeline")[0] + "hlapipeline"
elif "hlapipeline" in full_path:
repo_path = full_path.split("hlapipeline")[0] + "hlapipeline"
else:
pass
if not os.path.exists(repo_path):
repo_path = None # protect against non-existent paths
if repo_path:
get_git_rev_info.print_rev_id(
repo_path) # Display git repository information
else:
print(
"WARNING: Unable to display Git repository revision information.")
# 1: Interpret input data and optional parameters
print("-------------------- STEP 1: Get data --------------------")
imglist = check_and_get_data(input_list, archive=archive, clobber=clobber)
print("\nSUCCESS")
# 2: Apply filter to input observations to insure that they meet minimum criteria for being able to be aligned
print("-------------------- STEP 2: Filter data --------------------")
filteredTable = filter.analyze_data(imglist)
# Check the table to determine if there is any viable data to be aligned. The
# 'doProcess' column (bool) indicates the image/file should or should not be used
# for alignment purposes.
if filteredTable['doProcess'].sum() == 0:
print("No viable images in filtered table - no processing done.\n")
return (1)
# Get the list of all "good" files to use for the alignment
processList = filteredTable['imageName'][np.where(
filteredTable['doProcess'])]
processList = list(
processList
) #Convert processList from numpy list to regular python list
print("\nSUCCESS")
# 3: Build WCS for full set of input observations
print("-------------------- STEP 3: Build WCS --------------------")
refwcs = amutils.build_reference_wcs(processList)
print("\nSUCCESS")
# 4: Retrieve list of astrometric sources from database
# While loop to accommodate using multiple catalogs
doneFitting = False
catalogIndex = 0
extracted_sources = None
while not doneFitting:
skip_all_other_steps = False
retry_fit = False
print(
"-------------------- STEP 4: Detect astrometric sources --------------------"
)
print("Astrometric Catalog: ", catalogList[catalogIndex])
reference_catalog = generate_astrometric_catalog(
processList, catalog=catalogList[catalogIndex])
# The table must have at least MIN_CATALOG_THRESHOLD entries to be useful
if len(reference_catalog) >= MIN_CATALOG_THRESHOLD:
print("\nSUCCESS")
else:
if catalogIndex < numCatalogs - 1:
print("Not enough sources found in catalog " +
catalogList[catalogIndex])
print("Try again with the next catalog")
catalogIndex += 1
retry_fit = True
skip_all_other_steps = True
else:
print(
"Not enough sources found in any catalog - no processing done."
)
return (1)
if not skip_all_other_steps:
# 5: Extract catalog of observable sources from each input image
print(
"-------------------- STEP 5: Source finding --------------------"
)
if not extracted_sources:
extracted_sources = generate_source_catalogs(processList)
for imgname in extracted_sources.keys():
table = extracted_sources[imgname]["catalog_table"]
# The catalog of observable sources must have at least MIN_OBSERVABLE_THRESHOLD entries to be useful
total_num_sources = 0
for chipnum in table.keys():
total_num_sources += len(table[chipnum])
if total_num_sources < MIN_OBSERVABLE_THRESHOLD:
print(
"Not enough sources ({}) found in image {}".format(
total_num_sources, imgname))
return (1)
# Convert input images to tweakwcs-compatible NDData objects and
# attach source catalogs to them.
imglist = []
for group_id, image in enumerate(processList):
imglist.extend(
amutils.build_nddata(
image, group_id,
extracted_sources[image]['catalog_table']))
print("\nSUCCESS")
# 6: Cross-match source catalog with astrometric reference source catalog, Perform fit between source catalog and reference catalog
print(
"-------------------- STEP 6: Cross matching and fitting --------------------"
)
# Specify matching algorithm to use
match = tweakwcs.TPMatch(searchrad=250,
separation=0.1,
tolerance=100,
use2dhist=False)
# Align images and correct WCS
tweakwcs.tweak_image_wcs(imglist, reference_catalog, match=match)
# Interpret RMS values from tweakwcs
interpret_fit_rms(imglist, reference_catalog)
tweakwcs_info_keys = OrderedDict(
imglist[0].meta['tweakwcs_info']).keys()
imgctr = 0
for item in imglist:
retry_fit = False
#Handle fitting failures (no matches found)
if item.meta['tweakwcs_info']['status'].startswith(
"FAILED") == True:
if catalogIndex < numCatalogs - 1:
print(
"No cross matches found between astrometric catalog and sources found in images"
)
print("Try again with the next catalog")
catalogIndex += 1
retry_fit = True
break
else:
print(
"No cross matches found in any catalog - no processing done."
)
return (1)
max_rms_val = item.meta['tweakwcs_info']['TOTAL_RMS']
num_xmatches = item.meta['tweakwcs_info']['nmatches']
# print fit params to screen
print(
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FIT PARAMETERS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
)
if item.meta['chip'] == 1:
image_name = processList[imgctr]
imgctr += 1
print("image: {}".format(image_name))
print("chip: {}".format(item.meta['chip']))
print("group_id: {}".format(item.meta['group_id']))
for tweakwcs_info_key in tweakwcs_info_keys:
if not tweakwcs_info_key.startswith("matched"):
print("{} : {}".format(
tweakwcs_info_key,
item.meta['tweakwcs_info'][tweakwcs_info_key]))
# print("Radial shift: {}".format(math.sqrt(item.meta['tweakwcs_info']['shift'][0]**2+item.meta['tweakwcs_info']['shift'][1]**2)))
print(
"~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
)
if num_xmatches < MIN_CROSS_MATCHES:
if catalogIndex < numCatalogs - 1:
print(
"Not enough cross matches found between astrometric catalog and sources found in images"
)
print("Try again with the next catalog")
catalogIndex += 1
retry_fit = True
break
else:
print(
"Not enough cross matches found in any catalog - no processing done."
)
return (1)
elif max_rms_val > MAX_FIT_RMS:
if catalogIndex < numCatalogs - 1:
print(
"Fit RMS value = {}mas greater than the maximum threshold value {}."
.format(
item.meta['tweakwcs_info']['FIT_RMS'].value,
MAX_FIT_RMS))
print("Try again with the next catalog")
catalogIndex += 1
retry_fit = True
break
else:
print(
"Fit RMS values too large using any catalog - no processing done."
)
return (1)
else:
print("Fit calculations successful.")
if not retry_fit:
print("\nSUCCESS")
# 7: Write new fit solution to input image headers
print(
"-------------------- STEP 7: Update image headers with new WCS information --------------------"
)
if update_hdr_wcs:
update_image_wcs_info(imglist, processList)
print("\nSUCCESS")
else:
print("\n STEP SKIPPED")
return (0)
def perform_align(input_list,
archive=False,
clobber=False,
update_hdr_wcs=False,
print_fit_parameters=True,
print_git_info=False):
"""Main calling function.
Parameters
----------
input_list : list
List of one or more IPPSSOOTs (rootnames) to align.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories?
clobber : Boolean
Download and overwrite existing local copies of input files?
update_hdr_wcs : Boolean
Write newly computed WCS information to image image headers?
print_fit_parameters : Boolean
Specify whether or not to print out FIT results for each chip.
print_git_info : Boolean
Display git repository information?
Returns
-------
int value 0 if successful, int value 1 if unsuccessful
"""
# Define astrometric catalog list in priority order
catalogList = ['GAIADR2', 'GSC241']
# 0: print git info
if print_git_info:
print(
"-------------------- STEP 0: Display Git revision info --------------------"
)
full_path = os.path.dirname(__file__) + "/utils"
repo_path = None
if "hlapipeline/hlapipeline" in full_path:
repo_path = full_path.split(
"hlapipeline/hlapipeline")[0] + "hlapipeline"
elif "hlapipeline" in full_path:
repo_path = full_path.split("hlapipeline")[0] + "hlapipeline"
else:
pass
if not os.path.exists(repo_path):
repo_path = None # protect against non-existent paths
if repo_path:
get_git_rev_info.print_rev_id(
repo_path) # Display git repository information
else:
print(
"WARNING: Unable to display Git repository revision information."
)
# 1: Interpret input data and optional parameters
print("-------------------- STEP 1: Get data --------------------")
zeroDT = startingDT = datetime.datetime.now()
print(str(startingDT))
imglist = check_and_get_data(input_list, archive=archive, clobber=clobber)
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 1]: {} sec'.format(deltaDT))
startingDT = currentDT
# 2: Apply filter to input observations to insure that they meet minimum criteria for being able to be aligned
print("-------------------- STEP 2: Filter data --------------------")
filteredTable = filter.analyze_data(imglist)
# Check the table to determine if there is any viable data to be aligned. The
# 'doProcess' column (bool) indicates the image/file should or should not be used
# for alignment purposes. For filtered data, 'doProcess=0' and 'status=9999' in the table
# (the status value by default), so there is no need to update the filteredTable here.
if filteredTable['doProcess'].sum() == 0:
print("No viable images in filtered table - no processing done.\n")
return (filteredTable)
# Get the list of all "good" files to use for the alignment
processList = filteredTable['imageName'][np.where(
filteredTable['doProcess'])]
processList = list(
processList
) #Convert processList from numpy list to regular python list
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 2]: {} sec'.format(deltaDT))
startingDT = currentDT
# 3: Build WCS for full set of input observations
print("-------------------- STEP 3: Build WCS --------------------")
refwcs = amutils.build_reference_wcs(processList)
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 3]: {} sec'.format(deltaDT))
startingDT = currentDT
# 4: Extract catalog of observable sources from each input image
print("-------------------- STEP 4: Source finding --------------------")
extracted_sources = generate_source_catalogs(processList,
centering_mode='starfind',
nlargest=MAX_SOURCES_PER_CHIP)
for imgname in extracted_sources.keys():
table = extracted_sources[imgname]["catalog_table"]
# Get the location of the current image in the filtered table
index = np.where(filteredTable['imageName'] == imgname)[0][0]
# First ensure sources were found
if table[1] == None:
print("No sources found in image {}".format(imgname))
filteredTable[index]['status'] = 1
return (filteredTable)
# The catalog of observable sources must have at least MIN_OBSERVABLE_THRESHOLD entries to be useful
total_num_sources = 0
for chipnum in table.keys():
total_num_sources += len(table[chipnum])
# Update filtered table with number of found sources
filteredTable[index]['foundSources'] = total_num_sources
if total_num_sources < MIN_OBSERVABLE_THRESHOLD:
print("Not enough sources ({}) found in image {}".format(
total_num_sources, imgname))
filteredTable[index]['status'] = 1
return (filteredTable)
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 4]: {} sec'.format(deltaDT))
startingDT = currentDT
# 5: Retrieve list of astrometric sources from database
# Convert input images to tweakwcs-compatible NDData objects and
# attach source catalogs to them.
imglist = []
for group_id, image in enumerate(processList):
img = amutils.build_nddata(image, group_id,
extracted_sources[image]['catalog_table'])
# add the name of the image to the imglist object
for im in img:
im.meta['name'] = image
imglist.extend(img)
best_fit_rms = -99999.0
fit_algorithm_list = [match_2dhist_fit, match_default_fit]
for catalogIndex in range(
0, len(catalogList)): #loop over astrometric catalog
print(
"-------------------- STEP 5: Detect astrometric sources --------------------"
)
print("Astrometric Catalog: ", catalogList[catalogIndex])
reference_catalog = generate_astrometric_catalog(
processList, catalog=catalogList[catalogIndex])
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 5]: {} sec'.format(deltaDT))
startingDT = currentDT
if len(reference_catalog) < MIN_CATALOG_THRESHOLD:
print("Not enough sources found in catalog " +
catalogList[catalogIndex])
if catalogIndex < len(catalogList) - 1:
print("Try again with other catalog")
else:
print(
"ERROR! No astrometric sources found in any catalog. Exiting..."
) #bail out if not enough sources can be found any of the astrometric catalogs
filteredTable['status'][:] = 1
return (filteredTable)
else:
print(
"-------------------- STEP 5b: Cross matching and fitting --------------------"
)
for algorithm_name in fit_algorithm_list: #loop over fit algorithm type
print(
"------------------ Catalog {} matched using {} ------------------ "
.format(catalogList[catalogIndex],
algorithm_name.__name__))
#execute the correct fitting/matching algorithm
try:
fit_rms, fit_num = algorithm_name(
imglist,
reference_catalog,
print_fit_parameters=print_fit_parameters)
# Potentially update the previously set best fit with improved values
if best_fit_rms >= 0.:
if fit_rms < best_fit_rms:
best_fit_rms = fit_rms
best_fit_num = fit_num
for item in imglist:
item.best_meta = item.meta.copy()
# If a reasonable fit has been found, this is the initial setting of the best_fit_xxx variables
else:
if fit_rms < MAX_FIT_LIMIT:
best_fit_rms = fit_rms
best_fit_num = fit_num
for item in imglist:
item.best_meta = item.meta.copy()
except Exception:
print(
"WARNING: Catastrophic fitting failure with catalog {} and matching algorithm {}."
.format(catalogList[catalogIndex],
algorithm_name.__name__))
filteredTable['status'][:] = 1
# It may be there are additional catalogs and algorithms to try, so keep going
continue
# If this is true, we are done so break out of the catalog loop (inner for loop)
# THIS AND THE NEXT BREAK ARE FIXES TO THE CURRENT LOGIC. IT IS ASSUMED THESE
# FIXES WILL BECOME OBSOLETE ONCE THE ENHANCED LOGIC FOR CHOOSING THE BEST
# SOLUTION IS IMPLEMENTED.
if best_fit_rms > 0 and best_fit_rms < MAX_FIT_RMS:
break
# If this is true, we are done so break out of the catalog loop (outer for loop)
if best_fit_rms > 0 and best_fit_rms < MAX_FIT_RMS:
break
#imglist_temp = imglist.copy() # preserve best fit solution so that it can be inserted into a reinitialized imglist next time through.
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 5b]: {} sec'.format(deltaDT))
startingDT = currentDT
# 6: Populate the filteredTable
print(
"-------------------- STEP 6: Collect up information and populate the filtered table --------------------"
)
if best_fit_rms > 0 and best_fit_rms < MAX_FIT_RMS:
print(
"The fitting process was successful with a best fit total rms of {} mas"
.format(best_fit_rms))
else:
print(
"The fitting process was unsuccessful with a best fit total rms of {} mas"
.format(best_fit_rms))
if best_fit_rms > 0 and best_fit_rms < MAX_FIT_LIMIT:
# update to the meta information with the lowest rms if it is reasonable
for item in imglist:
item.meta = item.best_meta.copy()
filteredTable['status'][:] = 0
# Protect the writing of the table within the best_fit_rms
info_keys = OrderedDict(imglist[0].meta['tweakwcs_info']).keys()
# Update filtered table with number of matched sources and other information
for item in imglist:
imgname = item.meta['name']
index = np.where(filteredTable['imageName'] == imgname)[0][0]
if item.meta['tweakwcs_info']['status'].startswith(
"FAILED") != True:
for tweakwcs_info_key in info_keys:
if not tweakwcs_info_key.startswith("matched"):
if tweakwcs_info_key.lower() == 'rms':
filteredTable[index]['rms_x'] = item.meta[
'tweakwcs_info'][tweakwcs_info_key][0]
filteredTable[index]['rms_y'] = item.meta[
'tweakwcs_info'][tweakwcs_info_key][1]
filteredTable[index]['catalog'] = item.meta['tweakwcs_info'][
'catalog']
filteredTable[index]['catalogSources'] = len(reference_catalog)
filteredTable[index]['matchSources'] = item.meta[
'tweakwcs_info']['nmatches']
filteredTable[index]['rms_ra'] = item.meta['tweakwcs_info'][
'RMS_RA'].value
filteredTable[index]['rms_dec'] = item.meta['tweakwcs_info'][
'RMS_DEC'].value
filteredTable[index]['fit_rms'] = item.meta['tweakwcs_info'][
'FIT_RMS']
filteredTable[index]['total_rms'] = item.meta['tweakwcs_info'][
'TOTAL_RMS']
#filteredTable.pprint(max_width=-1)
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 6]: {} sec'.format(deltaDT))
startingDT = currentDT
# 7: Write new fit solution to input image headers
print(
"-------------------- STEP 7: Update image headers with new WCS information --------------------"
)
if best_fit_rms > 0 and update_hdr_wcs:
headerlet_dict = update_image_wcs_info(imglist)
for tableIndex in range(0, len(filteredTable)):
filteredTable[tableIndex]['headerletFile'] = headerlet_dict[
filteredTable[tableIndex]['imageName']]
print("\nSUCCESS")
else:
print("\n STEP SKIPPED")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 7]: {} sec'.format(deltaDT))
print('TOTAL Processing time of {} sec'.format(
(currentDT - zeroDT).total_seconds()))
return (filteredTable)
def align_randomFields(self, randomTable):
""" Process randomly selected fields (aka datasets) stored in an Astropy table.
Each field is used as input to determine if it can be aligned to an
astrometric standard. The success or fail status for each test is retained
as the overall success or fail statistic is the necessary output from
this test.
"""
numSuccess = 0
numAllDatasets = 0
# Read the table and extract a list of each dataset name in IPPSSOOT format
# which is either an association ID or an individual filename
dataset_list = get_dataset_list(randomTable)
numAllDatasets = len(dataset_list)
# Process the dataset names in the list
#
# If the dataset name represents an association ID, the multiplicity
# of images within the association need to be processed. Otherwise,
# the dataset is a single image.
#
# If the "alignment" of a field/dataset fails for any reason, trap
# the exception and keep going.
for dataset in dataset_list:
print("TEST_ALIGN. Dataset: ", dataset)
currentDT = datetime.datetime.now()
print(str(currentDT))
try:
shift_file, local_files = self.run_align([dataset])
x_shift = numpy.alltrue(numpy.isnan(shift_file['col2']))
rms_x = max(shift_file['col6'])
rms_y = max(shift_file['col7'])
reference_wcs = amutils.build_reference_wcs(local_files)
test_limit = self.fit_limit / reference_wcs.pscale
if not x_shift and ((rms_x <= test_limit) and
(rms_y <= test_limit)):
numSuccess += 1
print("TEST_ALIGN. Successful Dataset: ", dataset, "\n")
else:
print("TEST_ALIGN. Unsuccessful Dataset: ", dataset, "\n")
# Catch anything that happens as this dataset will be considered a failure, but
# the processing of datasets should continue. Generate sufficient output exception
# information so problems can be addressed.
except Exception:
exc_type, exc_value, exc_tb = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_tb,
file=sys.stdout)
print("TEST_ALIGN. Exception Dataset: ", dataset, "\n")
continue
# Determine the percent success over all datasets processed
percentSuccess = numSuccess / numAllDatasets
print('TEST_ALIGN. Number of successful tests: ', numSuccess,
' Total number of tests: ', numAllDatasets, ' Percent success: ',
percentSuccess * 100.0)
return percentSuccess
def perform_align(input_list, archive=False, clobber=False, update_hdr_wcs=False, print_fit_parameters=True,
print_git_info=False):
"""Main calling function.
Parameters
----------
input_list : list
List of one or more IPPSSOOTs (rootnames) to align.
archive : Boolean
Retain copies of the downloaded files in the astroquery created sub-directories?
clobber : Boolean
Download and overwrite existing local copies of input files?
update_hdr_wcs : Boolean
Write newly computed WCS information to image image headers?
Returns
-------
int value 0 if successful, int value 1 if unsuccessful
"""
# Define astrometric catalog list in priority order
catalogList = ['GAIADR2', 'GSC241']
# 0: print git info
if print_git_info:
print("-------------------- STEP 0: Display Git revision info --------------------")
full_path = os.path.dirname(__file__)+"/utils"
repo_path=None
if "hlapipeline/hlapipeline" in full_path:
repo_path = full_path.split("hlapipeline/hlapipeline")[0]+"hlapipeline"
elif "hlapipeline" in full_path:
repo_path = full_path.split("hlapipeline")[0]+"hlapipeline"
else:
pass
if not os.path.exists(repo_path): repo_path = None # protect against non-existent paths
if repo_path:
get_git_rev_info.print_rev_id(repo_path) # Display git repository information
else:
print("WARNING: Unable to display Git repository revision information.")
# 1: Interpret input data and optional parameters
print("-------------------- STEP 1: Get data --------------------")
startingDT = datetime.datetime.now()
print(str(startingDT))
imglist = check_and_get_data(input_list, archive=archive, clobber=clobber)
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 1]: {} sec'.format(deltaDT))
startingDT = currentDT
# 2: Apply filter to input observations to insure that they meet minimum criteria for being able to be aligned
print("-------------------- STEP 2: Filter data --------------------")
filteredTable = filter.analyze_data(imglist)
# Check the table to determine if there is any viable data to be aligned. The
# 'doProcess' column (bool) indicates the image/file should or should not be used
# for alignment purposes. For filtered data, 'doProcess=0' and 'status=9999' in the table.
if filteredTable['doProcess'].sum() == 0:
print("No viable images in filtered table - no processing done.\n")
return(filteredTable)
# Get the list of all "good" files to use for the alignment
processList = filteredTable['imageName'][np.where(filteredTable['doProcess'])]
processList = list(processList) #Convert processList from numpy list to regular python list
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 2]: {} sec'.format(deltaDT))
startingDT = currentDT
# 3: Build WCS for full set of input observations
print("-------------------- STEP 3: Build WCS --------------------")
refwcs = amutils.build_reference_wcs(processList)
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 3]: {} sec'.format(deltaDT))
startingDT = currentDT
# 4: Extract catalog of observable sources from each input image
print("-------------------- STEP 4: Source finding --------------------")
extracted_sources = generate_source_catalogs(processList,
centering_mode='starfind',
nlargest=MAX_SOURCES_PER_CHIP)
for imgname in extracted_sources.keys():
table=extracted_sources[imgname]["catalog_table"]
# The catalog of observable sources must have at least MIN_OBSERVABLE_THRESHOLD entries to be useful
total_num_sources = 0
for chipnum in table.keys():
total_num_sources += len(table[chipnum])
# Update filtered table with number of found sources
index = np.where(filteredTable['imageName']==imgname)[0][0]
filteredTable[index]['foundSources'] = total_num_sources
if total_num_sources < MIN_OBSERVABLE_THRESHOLD:
print("Not enough sources ({}) found in image {}".format(total_num_sources,imgname))
filteredTable[index]['status'] = 1
return(filteredTable)
# Convert input images to tweakwcs-compatible NDData objects and
# attach source catalogs to them.
imglist = []
for group_id, image in enumerate(processList):
img = amutils.build_nddata(image, group_id,
extracted_sources[image]['catalog_table'])
for im in img:
im.meta['name'] = image
imglist.extend(img)
# add the name of the image to the imglist object
print("\nSUCCESS")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 4]: {} sec'.format(deltaDT))
startingDT = currentDT
# 5: Retrieve list of astrometric sources from database
catalogIndex = 0
best_fit = MAX_FIT_LIMIT
print("-------------------- STEP 5: Detect Gaia astrometric sources --------------------")
print("Astrometric Catalog: ",catalogList[catalogIndex])
reference_catalog = generate_astrometric_catalog(processList, catalog=catalogList[catalogIndex])
best_fit_rms = MAX_FIT_RMS
if len(reference_catalog) < MIN_CATALOG_THRESHOLD:
print("Not enough sources found in Gaia catalog " + catalogList[catalogIndex])
print("Try again with other catalog")
catalogIndex += 1
retry_fit = True
skip_all_other_steps = True
else:
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 5]: {} sec'.format(deltaDT))
startingDT = currentDT
print("-------------------- STEP 5b: Cross matching and fitting --------------------")
best_fit_rms, best_fit_num = match_2dhist_fit(imglist, reference_catalog,
print_fit_parameters=print_fit_parameters)
info_keys = OrderedDict(imglist[0].meta['tweakwcs_info']).keys()
# Update filtered table with number of matched sources and other information
for item in imglist:
imgname = item.meta['name']
index = np.where(filteredTable['imageName']==imgname)[0][0]
if item.meta['tweakwcs_info']['status'].startswith("FAILED") != True:
for tweakwcs_info_key in info_keys:
if not tweakwcs_info_key.startswith("matched"):
if tweakwcs_info_key.lower() == 'rms':
filteredTable[index]['rms_x'] = item.meta['tweakwcs_info'][tweakwcs_info_key][0]
filteredTable[index]['rms_y'] = item.meta['tweakwcs_info'][tweakwcs_info_key][1]
filteredTable[index]['catalog'] = item.meta['tweakwcs_info']['catalog']
filteredTable[index]['catalogSources'] = len(reference_catalog)
filteredTable[index]['matchSources'] = item.meta['tweakwcs_info']['nmatches']
filteredTable[index]['rms_ra'] = item.meta['tweakwcs_info']['RMS_RA'].value
filteredTable[index]['rms_dec'] = item.meta['tweakwcs_info']['RMS_DEC'].value
filteredTable[index]['fit_rms'] = item.meta['tweakwcs_info']['FIT_RMS']
filteredTable[index]['total_rms'] = item.meta['tweakwcs_info']['TOTAL_RMS']
#filteredTable.pprint(max_width=-1)
# 6b: If available, the logic tree for fitting with different algorithms
# would be here. These would only be invoked if the above step failed.
# At this time, only one algorithm is being used and there are not
# currently other cases to run on the images and so this is empty. This
# section might be a good area to create as a function if this will be repeated
# with other catalogs.
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 5b]: {} sec'.format(deltaDT))
startingDT = currentDT
# 8: If available, try the fitting with different catalogs
if best_fit_rms >= MAX_FIT_RMS:
for catalogIndex in range(1, len(catalogList)):
print("-------------------- STEP 6: Detect catalog astrometric sources --------------------")
print("Astrometric Catalog: ",catalogList[catalogIndex])
reference_catalog = generate_astrometric_catalog(processList, catalog=catalogList[catalogIndex])
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 6]: {} sec'.format(deltaDT))
startingDT = currentDT
if len(reference_catalog) < MIN_CATALOG_THRESHOLD:
print("Not enough sources found in catalog " + catalogList[catalogIndex])
print("Try again with other catalog")
catalogIndex += 1
retry_fit = True
skip_all_other_steps = True
else:
print("-------------------- STEP 6b: Cross matching and fitting --------------------")
fit_rms, fit_num = match_default_fit(imglist, reference_catalog,
print_fit_parameters=print_fit_parameters)
# update the best fit
if fit_rms < best_fit_rms:
best_fit_rms = fit_rms
best_fit_num = fit_num
for item in imglist:
item.best_meta = item.meta.copy()
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 6b]: {} sec'.format(deltaDT))
startingDT = currentDT
if best_fit_rms < MAX_FIT_RMS:
print("The fitting process was successful with a best fit total rms of {} mas".format(best_fit_rms))
else:
print("The fitting process was unsuccessful with a best fit total rms of {} mas".format(best_fit_rms))
if best_fit_rms < MAX_FIT_LIMIT:
# update to the meta information with the lowest rms if it is reasonable
for item in imglist:
item.meta = item.best_meta
# 7: Write new fit solution to input image headers
print("-------------------- STEP 7: Update image headers with new WCS information --------------------")
if update_hdr_wcs:
update_image_wcs_info(imglist, processList)
print("\nSUCCESS")
else:
print("\n STEP SKIPPED")
currentDT = datetime.datetime.now()
deltaDT = (currentDT - startingDT).total_seconds()
print('Processing time of [STEP 7]: {} sec'.format(deltaDT))
startingDT = currentDT
filteredTable['status'][:] = 0
return (filteredTable)