def test_align_ngc188(self): """ Verify whether NGC188 exposures can be aligned to an astrometric standard. Characteristics of this test: of NGC188 suitable for creating a combined mosaic using both instruments. """ totalRMS = 0.0 input_filenames = ['iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits', 'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits', 'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits'] # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline','dev','mosaic_ngc188',input_file) try: datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_47tuc(self): """ Verify whether 47Tuc exposures can be aligned to an astrometric standard. Characteristics of this test: * Input exposures include both ACS and WFC3 images of the same general field-of-view of 47Tuc suitable for creating a combined mosaic using both instruments. """ totalRMS = 0.0 input_filenames = ['ib6v06c4q_flc.fits','ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits','ib6v25atq_flc.fits', 'jddh02gjq_flc.fits','jddh02glq_flc.fits', 'jddh02goq_flc.fits'] # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline','dev','mosaic_47tuc',input_file) try: datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_47tuc(self): """ Verify whether 47Tuc exposures can be aligned to an astrometric standard. Characteristics of this test: * Input exposures include both ACS and WFC3 images of the same general field-of-view of 47Tuc suitable for creating a combined mosaic using both instruments. """ total_rms = 0.0 input_filenames = [ 'ib6v06c4q_flc.fits', 'ib6v06c7q_flc.fits', 'ib6v25aqq_flc.fits', 'ib6v25atq_flc.fits', 'jddh02gjq_flc.fits', 'jddh02glq_flc.fits', 'jddh02goq_flc.fits' ] # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_47tuc', input_file) dataset_table = alignimages.perform_align(input_filenames, archive=False, clobber=False, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Examine the output table to extract the RMS for the entire fit and the compromised # information if dataset_table: total_rms = dataset_table['total_rms'][0] assert 0.0 < total_rms <= RMS_LIMIT
def align_to_gaia(self): """Extract the flt/flc filenames from the exposure product list, as well as the corresponding headerlet filenames to use legacy alignment routine. """ exposure_filenames = [] headerlet_filenames = {} align_table = None try: if self.edp_list: for edp in self.edp_list: exposure_filenames.append(edp.full_filename) headerlet_filenames[edp.full_filename] = edp.headerlet_filename align_table = alignimages.perform_align(exposure_filenames, debug=False, runfile="alignimages.log", update_hdr_wcs=True, headerlet_filenames=headerlet_filenames) except Exception: # TODO: Fix up the logging # Report a problem with the alignment # log.info("EXCEPTION encountered in alignimages.\n") exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) # log.info("No correction to absolute astrometric frame applied.\n") align_table = None # Return a table which contains data regarding the alignment, as well as the # list of the flt/flc exposures which were part of the alignment process # TODO: This does not account for individual exposures which might have been # excluded from alignment. return align_table, exposure_filenames
def test_align_ngc188(self): """ Verify whether NGC188 exposures can be aligned to an astrometric standard. Characteristics of this test: of NGC188 suitable for creating a combined mosaic using both instruments. """ total_rms = 0.0 input_filenames = [ 'iaal01hxq_flc.fits', 'iaala3btq_flc.fits', 'iaal01hyq_flc.fits', 'iaala3bsq_flc.fits', 'j8boa1m8q_flc.fits', 'j8boa1m4q_flc.fits', 'j8boa1maq_flc.fits', 'j8boa1m6q_flc.fits' ] # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline', 'dev', 'mosaic_ngc188', input_file) dataset_table = alignimages.perform_align(input_filenames, archive=False, clobber=False, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Examine the output table to extract the RMS for the entire fit and the compromised # information if dataset_table: total_rms = dataset_table['total_rms'][0] assert 0.0 < total_rms <= RMS_LIMIT
def test_align_fail_single_visit(self): """ Verify whether single-visit exposures can be aligned to an astrometric standard. Characteristics of this test: * Input exposures include exposures from a number of single visit datasets to explore what impact differing observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry. This test is known to fail due to "RuntimeError: Number of output coordinates exceeded allocation (475)a". It will exercise the code using both catalogs for each of the three fitting algorithms at this time. Nans will be present in the output table. The following datasets are used in these tests: * WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame WFC3/UVIS images """ totalRMS = 0.0 input_filenames = ['ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits', 'ibjt01aiq_flt.fits', 'ibjt01amq_flt.fits', 'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits', 'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits', 'ibjt01zwq_flc.fits', 'ibjt01a4q_flc.fits', 'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits', 'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits', 'ibjt01avq_flt.fits', 'ibjt01yuq_flc.fits', 'ibjt01ztq_flc.fits'], # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline','dev','base_tests',input_file) try: datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] assert (0.0 < totalRMS <= RMS_LIMIT)
def test_align_fail_single_visit(self): """ Verify whether single-visit exposures can be aligned to an astrometric standard. Characteristics of this test: * Input exposures include exposures from a number of single visit datasets to explore what impact differing observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry. This test is known to fail due to "RuntimeError: Number of output coordinates exceeded allocation (475)a". It will exercise the code using both catalogs for each of the three fitting algorithms at this time. Nans will be present in the output table. The following datasets are used in these tests: * WFC3 dataset 12219_01: 8x F160W full-frame WFC3/IR images, 9x F336W full-frame WFC3/UVIS images """ total_rms = 0.0 input_filenames = [ 'ibjt01a1q_flc.fits', 'ibjt01a8q_flc.fits', 'ibjt01aiq_flt.fits', 'ibjt01amq_flt.fits', 'ibjt01aqq_flt.fits', 'ibjt01auq_flt.fits', 'ibjt01yqq_flc.fits', 'ibjt01z0q_flc.fits', 'ibjt01zwq_flc.fits', 'ibjt01a4q_flc.fits', 'ibjt01acq_flc.fits', 'ibjt01akq_flt.fits', 'ibjt01aoq_flt.fits', 'ibjt01asq_flt.fits', 'ibjt01avq_flt.fits', 'ibjt01yuq_flc.fits', 'ibjt01ztq_flc.fits' ], # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file) dataset_table = alignimages.perform_align(input_filenames, archive=False, clobber=False, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Examine the output table to extract the RMS for the entire fit and the compromised # information if dataset_table: total_rms = dataset_table['total_rms'][0] assert 0.0 < total_rms <= RMS_LIMIT
def test_astroquery(self): """Verify that new astroquery interface will work""" totalRMS = 0.0 try: datasetTable = alignimages.perform_align(['IB6V06060'],archive=False,clobber=True,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) sys.exit() assert (0.0 < totalRMS <= RMS_LIMIT)
def test_astroquery(self): """Verify that new astroquery interface will work""" total_rms = 0.0 dataset_table = alignimages.perform_align(['IB6V06060'], archive=False, clobber=True, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Examine the output table to extract the RMS for the entire fit and the compromised # information if dataset_table: total_rms = dataset_table['total_rms'][0] assert 0.0 < total_rms <= RMS_LIMIT
def test_align_single_visits(self, input_filenames): """ Verify whether single-visit exposures can be aligned to an astrometric standard. Characteristics of these tests: * Input exposures include exposures from a number of single visit datasets to explore what impact differing observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry. The following datasets are used in these tests: * ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W, 1x F606W, 1x F625W, 2x F658N, 1x F775W, 1x F814W, 1x F850LP, and 2x F892N ACS/HRC images * ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images * ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images * WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images * WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images """ total_rms = 0.0 # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline', 'dev', 'base_tests', input_file) dataset_table = alignimages.perform_align(input_filenames, archive=False, clobber=False, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Examine the output table to extract the RMS for the entire fit and the compromised # information if dataset_table: total_rms = dataset_table['total_rms'][0] assert 0.0 < total_rms <= RMS_LIMIT
def test_align_single_visits(self,input_filenames): """ Verify whether single-visit exposures can be aligned to an astrometric standard. Characteristics of these tests: * Input exposures include exposures from a number of single visit datasets to explore what impact differing observing modes (differing instruments, detectors, filters, subarray size, etc.) have on astrometry. The following datasets are used in these tests: * ACS dataset 10048_a1: 2x F344N, 1x F435W, 1x F475W, 2x F502N, 2x F550M, 1x F555W, 1x F606W, 1x F625W, 2x F658N, 1x F775W, 1x F814W, 1x F850LP, and 2x F892N ACS/HRC images * ACS dataset 10265_01: 4x F606W full-frame ACS/WFC images * ACS dataset 12580_02: 5x F475W & 6x F814W ACS/WFC images * WFC3 dataset 11663_12: 4x F160W full-frame WFC3/IR images * WFC3 dataset 12379_02: 4X F606W, 4x F502N full-frame WFC3/UVIS images """ totalRMS = 0.0 # Since these are full file names (*_flc.fits) which cannot be obtained via astroquery from # MAST, get the data now using ci_watson. for input_file in input_filenames: get_bigdata('hst-hla-pipeline','dev','base_tests',input_file) try: datasetTable = alignimages.perform_align(input_filenames,archive=False,clobber=False,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Examine the output table to extract the RMS for the entire fit and the compromised information if datasetTable: totalRMS = datasetTable['total_rms'][0] except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) assert (0.0 < totalRMS <= RMS_LIMIT)
def test_randomlist(tmpdir, dataset): """ Tests which validate whether mosaics can be aligned to an astrometric standard. Characteristics of these tests: * A reference WCS is generated based upon all the input images for the field. * A source astrometric catalog is created using the Photutils package to detect explicitly sources in the images. * An astrometric catalog is created to extract astrometric positions for the found sources in the input images' field-of-view using GAIADR2 (preferred) or GAIADR1. * Cross matching/fitting is done between found sources and catalog coordinates with the Tweakwcs package. * The quality of the fit is evaluated against a minimum threshold and potentially another fit algorithm is invoked or an alternative catalog is used in an effort to obtain a better quality fit. * If the option is set, the WCS information is updated for the input exposures. The default is False. * No mosaic is generated. * An output table containing characterizations of the process and associated fit is generated. Success Criteria: * Success criterion hard-coded for this test represents whether a statistical sample (70%) of ACS and WFC3 datasets were able to be aligned to within 10mas RMS. * RMS values are extracted from the table output from `perform_align` * This criterion will need to be determined by the user after the test has been run based on how many datasets were run and skipped. The input master_list CSV file is output from a database and lists associations and singletons for ACS and WFC3 instruments randomly sorted. The actual data files are downloaded from MAST via astroquery. This test file can be executed in the following manner: $ pytest -n # -s --basetemp=/internal/hladata/yourUniqueDirectoryHere --bigdata --slow --master_list ACSWFC3ListDefault50.csv --start_row 0 --num_rows 50 test_randomlist.py >& test_random_output.txt & $ tail -f test_random_output.txt * The `-n #` option can be used to run tests in parallel if `pytest-xdist` has been installed where `#` is the number of cpus to use. * Note: When running this test, the `--basetemp` directory should be set to a unique existing directory to avoid deleting previous test output. * The default master list exists in the tests/hla directory and contains 50 datasets. The full master list of thousands of datasets resides in Artifactory as ACSWFC3List.csv (https://bytesalad.stsci.edu/artifactory/hst-hla-pipeline/dev/master_lists). """ print("TEST_RANDOM. Dataset: ", dataset) output_name = dataset + '.ecsv' current_dt = datetime.datetime.now() print(str(current_dt)) subdir = "" prevdir = os.getcwd() # create working directory specified for the test if not tmpdir.ensure(subdir, dir=True): curdir = tmpdir.mkdir(subdir).strpath else: curdir = tmpdir.join(subdir).strpath os.chdir(curdir) try: dataset_table = alignimages.perform_align([dataset], archive=False, clobber=True, debug=False, update_hdr_wcs=False, print_fit_parameters=True, print_git_info=False, output=False) # Filtered datasets if dataset_table['doProcess'].sum() == 0: pytest.skip("TEST_RANDOM. Filtered Dataset: {}.".format(dataset)) # Datasets to process elif dataset_table['doProcess'].sum() > 0: # Determine images in dataset to be processed and the number of images # This is in case an image was filtered out (e.g., expotime = 0) index = np.where(dataset_table['doProcess'] == 1)[0] fit_qual = dataset_table['fit_qual'][index[0]] # Update the table with the dataset_key which is really just a counter dataset_table['completed'][:] = True dataset_table.write(output_name, format='ascii.ecsv') if fit_qual > 4: pytest.fail( "TEST_RANDOM. Unsuccessful Dataset (fit_qual = 5): {}.". format(dataset)) else: assert 0 < fit_qual <= 4 # Catch anything that happens as this dataset will be considered a failure, but # the processing of datasets should continue. This is meant to catch # unexpected errors and generate sufficient output exception # information so algorithmic problems can be addressed. except Exception as except_details: print(except_details) pytest.fail("TEST_RANDOM. Exception Dataset: {}\n", dataset) finally: # Perform some clean up if os.path.isfile('ref_cat.ecsv'): os.remove('ref_cat.ecsv') if os.path.isfile('refcatalog.cat'): os.remove('refcatalog.cat') for filename in os.listdir(): if filename.endswith('flt.fits') or filename.endswith('flc.fits'): os.remove(filename) # Return to original directory os.chdir(prevdir)
def align_randomFields(self, randomTable): """ Process randomly selected fields (aka datasets) stored in an Astropy table. Each field is used as input to determine if it can be aligned to an astrometric standard. The success or fail status for each test is retained as the overall success or fail statistic is the necessary output from this test. """ numSuccess = 0 numUnsuccessful = 0 numException = 0 numProcessedDatasets = 0 # Read the table and extract a list of each dataset name in IPPSSOOT format # which is either an association ID or an individual filename dataset_list = get_dataset_list(randomTable) numProcessedDatasets = len(dataset_list) numStartTests = numProcessedDatasets # Process the dataset names in the list # # If the dataset name represents an association ID, the multiplicity # of images within the association need to be processed. Otherwise, # the dataset is a single image. # # If the "alignment" of a field/dataset fails for any reason, trap # the exception and keep going. allDatasetTable = Table() datasetKey = -1 print("TEST_RANDOM. Dataset List: ", dataset_list) for dataset in dataset_list: datasetKey += 1 outputName = dataset + '.ecsv' print("TEST_RANDOM. Dataset: ", dataset) currentDT = datetime.datetime.now() print(str(currentDT)) try: datasetTable = alignimages.perform_align([dataset],archive=False,clobber=True,debug=False, update_hdr_wcs=False,print_fit_parameters=True,print_git_info=False,output=False) # Filtered datasets if datasetTable['doProcess'].sum() == 0: print("TEST_RANDOM. Filtered Dataset: ", dataset, "\n") numProcessedDatasets -= 1; # Datasets to process elif datasetTable['doProcess'].sum() > 0: # Determine images in dataset to be processed and the number of images # This is in case an image was filtered out (e.g., expotime = 0) index = np.where(datasetTable['doProcess']==1)[0] sumOfStatus = datasetTable['status'][index].sum() # Update the table with the datasetKey which is really just a counter datasetTable['datasetKey'][:] = datasetKey datasetTable['completed'][:] = True datasetTable.write(outputName, format='ascii.ecsv') #datasetTable.pprint(max_width=-1) # Successful datasets if (sumOfStatus == 0): print("TEST_RANDOM. Successful Dataset: ", dataset, "\n") numSuccess += 1 # Unsuccessful datasets else: print("TEST_RANDOM. Unsuccessful Dataset: ", dataset, "\n") numUnsuccessful += 1 # Append the latest dataset table to the summary table allDatasetTable = vstack([allDatasetTable, datasetTable]) # Catch anything that happens as this dataset will be considered a failure, but # the processing of datasets should continue. Generate sufficient output exception # information so problems can be addressed. except Exception: exc_type, exc_value, exc_tb = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_tb, file=sys.stdout) print("TEST_RANDOM. Exception Dataset: ", dataset, "\n") numException += 1 continue # Perform some clean up if os.path.isfile('ref_cat.ecsv'): os.remove('ref_cat.ecsv') if os.path.isfile('refcatalog.cat'): os.remove('refcatalog.cat') for filename in os.listdir(): if filename.endswith('flt.fits') or filename.endswith('flc.fits'): os.unlink(filename) # Write out the table allDatasetTable.write('resultsBigTest.ecsv', format='ascii.ecsv') # Determine the percent success over all datasets processed percentSuccess = numSuccess/numProcessedDatasets print('TEST_RANDOM. Number of tests started: ', numStartTests) print('TEST_RANDOM. Number of tests (excluding filtered): ', numProcessedDatasets) print('TEST_RANDOM. Number of successful tests: ', numSuccess) print('TEST_RANDOM. Number of unsuccessful tests: ', numUnsuccessful) print('TEST_RANDOM. Number of exception tests: ', numException) print('TEST_RANDOM. Percentage success/numberOfTests: ', numSuccess/numProcessedDatasets*100.0) return percentSuccess
def process(inFile,force=False,newpath=None, inmemory=False, num_cores=None, headerlets=True, align_to_gaia=True): """ Run astrodrizzle on input file/ASN table using default values for astrodrizzle parameters. """ # We only need to import this package if a user run the task import drizzlepac from drizzlepac import processInput # used for creating new ASNs for _flc inputs from stwcs import updatewcs from drizzlepac import alignimages # interpret envvar variable, if specified if envvar_compute_name in os.environ: val = os.environ[envvar_compute_name].lower() if val not in envvar_bool_dict: msg = "ERROR: invalid value for {}.".format(envvar_compute_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) align_to_gaia = envvar_bool_dict[val] if envvar_new_apriori_name in os.environ: # Reset ASTROMETRY_STEP_CONTROL based on this variable # This provides backward-compatibility until ASTROMETRY_STEP_CONTROL # gets removed entirely. val = os.environ[envvar_new_apriori_name].lower() if val not in envvar_dict: msg = "ERROR: invalid value for {}.".format(envvar_new_apriori_name) msg += " \n Valid Values: on, off, yes, no, true, false" raise ValueError(msg) os.environ[envvar_old_apriori_name] = envvar_dict[val] if headerlets or align_to_gaia: from stwcs.wcsutil import headerlet # Open the input file try: # Make sure given filename is complete and exists... inFilename = fileutil.buildRootname(inFile,ext=['.fits']) if not os.path.exists(inFilename): print("ERROR: Input file - %s - does not exist." % inFilename) return except TypeError: print("ERROR: Inappropriate input file.") return #If newpath was specified, move all files to that directory for processing if newpath: orig_processing_dir = os.getcwd() new_processing_dir = _createWorkingDir(newpath,inFilename) _copyToNewWorkingDir(new_processing_dir,inFilename) os.chdir(new_processing_dir) # Initialize for later use... _mname = None _new_asn = None _calfiles = [] # Identify WFPC2 inputs to account for differences in WFPC2 inputs wfpc2_input = fits.getval(inFilename, 'instrume') == 'WFPC2' cal_ext = None # Check input file to see if [DRIZ/DITH]CORR is set to PERFORM if '_asn' in inFilename: # We are working with an ASN table. # Use asnutil code to extract filename inFilename = _lowerAsn(inFilename) _new_asn = [inFilename] _asndict = asnutil.readASNTable(inFilename,None,prodonly=False) _cal_prodname = _asndict['output'].lower() #_fname = fileutil.buildRootname(_cal_prodname,ext=['_drz.fits']) # Retrieve the first member's rootname for possible use later _fimg = fits.open(inFilename, memmap=False) for name in _fimg[1].data.field('MEMNAME'): if name[-1] != '*': _mname = name.split('\0', 1)[0].lower() break _fimg.close() del _fimg else: # Check to see if input is a _RAW file # If it is, strip off the _raw.fits extension... _indx = inFilename.find('_raw') if _indx < 0: _indx = len(inFilename) # ... and build the CALXXX product rootname. if wfpc2_input: # force code to define _c0m file as calibrated product to be used cal_ext = ['_c0m.fits'] _mname = fileutil.buildRootname(inFilename[:_indx], ext=cal_ext) _cal_prodname = inFilename[:_indx] # Reset inFilename to correspond to appropriate input for # drizzle: calibrated product name. inFilename = _mname if _mname is None: errorMsg = 'Could not find calibrated product!' raise Exception(errorMsg) # Create trailer filenames based on ASN output filename or # on input name for single exposures if '_raw' in inFile: # Output trailer file to RAW file's trailer _trlroot = inFile[:inFile.find('_raw')] elif '_asn' in inFile: # Output trailer file to ASN file's trailer, not product's trailer _trlroot = inFile[:inFile.find('_asn')] else: # Default: trim off last suffix of input filename # and replacing with .tra _indx = inFile.rfind('_') if _indx > 0: _trlroot = inFile[:_indx] else: _trlroot = inFile _trlfile = _trlroot + '.tra' # Open product and read keyword value # Check to see if product already exists... dkey = 'DRIZCORR' # ...if product does NOT exist, interrogate input file # to find out whether 'dcorr' has been set to PERFORM # Check if user wants to process again regardless of DRIZCORR keyword value if force: dcorr = 'PERFORM' else: if _mname : _fimg = fits.open(fileutil.buildRootname(_mname,ext=['_raw.fits']), memmap=False) _phdr = _fimg['PRIMARY'].header if dkey in _phdr: dcorr = _phdr[dkey] else: dcorr = None _fimg.close() del _fimg else: dcorr = None time_str = _getTime() _tmptrl = _trlroot + '_tmp.tra' _drizfile = _trlroot + '_pydriz' _drizlog = _drizfile + ".log" # the '.log' gets added automatically by astrodrizzle _alignlog = _trlroot + '_align.log' if dcorr == 'PERFORM': if '_asn.fits' not in inFilename: # Working with a singleton # However, we always want to make sure we always use # a calibrated product as input, if available. _infile = fileutil.buildRootname(_cal_prodname, ext=cal_ext) _infile_flc = fileutil.buildRootname(_cal_prodname,ext=['_flc.fits']) _cal_prodname = _infile _inlist = _calfiles = [_infile] # Add CTE corrected filename as additional input if present if os.path.exists(_infile_flc) and _infile_flc != _infile: _inlist.append(_infile_flc) else: # Working with an ASN table... _infile = inFilename flist,duplist = processInput.checkForDuplicateInputs(_asndict['order']) _calfiles = flist if len(duplist) > 0: origasn = processInput.changeSuffixinASN(inFilename,'flt') dupasn = processInput.changeSuffixinASN(inFilename,'flc') _inlist = [origasn,dupasn] else: _inlist = [_infile] # We want to keep the original specification of the calibration # product name, though, not a lower-case version... _cal_prodname = inFilename _new_asn.extend(_inlist) # kept so we can delete it when finished # check to see whether FLC files are also present, and need to be updated # generate list of FLC files align_files = None _calfiles_flc = [f.replace('_flt.fits','_flc.fits') for f in _calfiles] # insure these files exist, if not, blank them out # Also pick out what files will be used for additional alignment to GAIA if not os.path.exists(_calfiles_flc[0]): _calfiles_flc = None align_files = _calfiles align_update_files = None else: align_files = _calfiles_flc align_update_files = _calfiles # Run updatewcs on each list of images updatewcs.updatewcs(_calfiles) if _calfiles_flc: updatewcs.updatewcs(_calfiles_flc) if align_to_gaia: # Perform additional alignment on the FLC files, if present ############### # # call hlapipeline code here on align_files list of files # ############### # Create trailer marker message for start of align_to_GAIA processing _trlmsg = _timestamp("Align_to_GAIA started ") print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" # Create an empty astropy table so it can be used as input/output for the perform_align function #align_table = Table() try: align_table = alignimages.perform_align(align_files,update_hdr_wcs=True, runfile=_alignlog) for row in align_table: if row['status'] == 0: trlstr = "Successfully aligned {} to {} astrometric frame\n" _trlmsg += trlstr.format(row['imageName'], row['catalog']) else: trlstr = "Could not align {} to absolute astrometric frame\n" _trlmsg += trlstr.format(row['imageName']) except Exception: # Something went wrong with alignment to GAIA, so report this in # trailer file _trlmsg = "EXCEPTION encountered in alignimages...\n" _trlmsg += " No correction to absolute astrometric frame applied!\n" # Write the perform_align log to the trailer file...(this will delete the _alignlog) _appendTrlFile(_trlfile,_alignlog) # Append messages from this calling routine post-perform_align ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _trlmsg = "" #Check to see whether there are any additional input files that need to # be aligned (namely, FLT images) if align_update_files and align_table: # Apply headerlets from alignment to FLT version of the files for fltfile, flcfile in zip(align_update_files, align_files): row = align_table[align_table['imageName']==flcfile] headerletFile = row['headerletFile'][0] if headerletFile != "None": headerlet.apply_headerlet_as_primary(fltfile, headerletFile, attach=True, archive=True) # append log file contents to _trlmsg for inclusion in trailer file _trlstr = "Applying headerlet {} as Primary WCS to {}\n" _trlmsg += _trlstr.format(headerletFile, fltfile) else: _trlmsg += "No absolute astrometric headerlet applied to {}\n".format(fltfile) # Finally, append any further messages associated with alignement from this calling routine _trlmsg += _timestamp('Align_to_GAIA completed ') print(_trlmsg) ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Run astrodrizzle and send its processing statements to _trlfile _pyver = drizzlepac.astrodrizzle.__version__ for _infile in _inlist: # Run astrodrizzle for all inputs # Create trailer marker message for start of astrodrizzle processing _trlmsg = _timestamp('astrodrizzle started ') _trlmsg += __trlmarker__ _trlmsg += '%s: Processing %s with astrodrizzle Version %s\n' % (time_str,_infile,_pyver) print(_trlmsg) # Write out trailer comments to trailer file... ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) _pyd_err = _trlroot+'_pydriz.stderr' try: b = drizzlepac.astrodrizzle.AstroDrizzle(input=_infile,runfile=_drizfile, configobj='defaults',in_memory=inmemory, num_cores=num_cores, **pipeline_pars) except Exception as errorobj: _appendTrlFile(_trlfile,_drizlog) _appendTrlFile(_trlfile,_pyd_err) _ftrl = open(_trlfile,'a') _ftrl.write('ERROR: Could not complete astrodrizzle processing of %s.\n' % _infile) _ftrl.write(str(sys.exc_info()[0])+': ') _ftrl.writelines(str(errorobj)) _ftrl.write('\n') _ftrl.close() print('ERROR: Could not complete astrodrizzle processing of %s.' % _infile) raise Exception(str(errorobj)) # Now, append comments created by PyDrizzle to CALXXX trailer file print('Updating trailer file %s with astrodrizzle comments.' % _trlfile) _appendTrlFile(_trlfile,_drizlog) # Save this for when astropy.io.fits can modify a file 'in-place' # Update calibration switch _fimg = fits.open(_cal_prodname, mode='update', memmap=False) _fimg['PRIMARY'].header[dkey] = 'COMPLETE' _fimg.close() del _fimg # Enforce pipeline convention of all lower-case product # names _prodlist = glob.glob('*drz.fits') for _prodname in _prodlist: _plower = _prodname.lower() if _prodname != _plower: os.rename(_prodname,_plower) else: # Create default trailer file messages when astrodrizzle is not # run on a file. This will typically apply only to BIAS,DARK # and other reference images. # Start by building up the message... _trlmsg = _timestamp('astrodrizzle skipped ') _trlmsg = _trlmsg + __trlmarker__ _trlmsg = _trlmsg + '%s: astrodrizzle processing not requested for %s.\n' % (time_str,inFilename) _trlmsg = _trlmsg + ' astrodrizzle will not be run at this time.\n' print(_trlmsg) # Write message out to temp file and append it to full trailer file ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # Append final timestamp to trailer file... _final_msg = '%s: Finished processing %s \n' % (time_str,inFilename) _final_msg += _timestamp('astrodrizzle completed ') _trlmsg += _final_msg ftmp = open(_tmptrl,'w') ftmp.writelines(_trlmsg) ftmp.close() _appendTrlFile(_trlfile,_tmptrl) # If we created a new ASN table, we need to remove it if _new_asn is not None: for _name in _new_asn: fileutil.removeFile(_name) # Clean up any generated OrIg_files directory if os.path.exists("OrIg_files"): # check to see whether this directory is empty flist = glob.glob('OrIg_files/*.fits') if len(flist) == 0: os.rmdir("OrIg_files") else: print('OrIg_files directory NOT removed as it still contained images...') # If headerlets have already been written out by alignment code, # do NOT write out this version of the headerlets if headerlets: # Generate headerlets for each updated FLT image hlet_msg = _timestamp("Writing Headerlets started") for fname in _calfiles: frootname = fileutil.buildNewRootname(fname) hname = "%s_flt_hlet.fits"%frootname # Write out headerlet file used by astrodrizzle, however, # do not overwrite any that was already written out by alignimages if not os.path.exists(hname): hlet_msg += "Created Headerlet file %s \n"%hname try: headerlet.write_headerlet(fname,'OPUS',output='flt', wcskey='PRIMARY', author="OPUS",descrip="Default WCS from Pipeline Calibration", attach=False,clobber=True,logging=False) except ValueError: hlet_msg += _timestamp("SKIPPED: Headerlet not created for %s \n"%fname) # update trailer file to log creation of headerlet files hlet_msg += _timestamp("Writing Headerlets completed") ftrl = open(_trlfile,'a') ftrl.write(hlet_msg) ftrl.close() # If processing was done in a temp working dir, restore results to original # processing directory, return to original working dir and remove temp dir if newpath: _restoreResults(new_processing_dir,orig_processing_dir) os.chdir(orig_processing_dir) _removeWorkingDir(new_processing_dir) # Provide feedback to user print(_final_msg)