def calref_from_image(input_image): """ Return a list of reference filenames, as defined in the primary header of the given input image, necessary for calibration. This is mostly needed for destriping tools. """ # NOTE: Add additional mapping as needed. # Map *CORR to associated CRDS reference file. corr_lookup = { 'DQICORR': ['BPIXTAB', 'SNKCFILE'], 'ATODCORR': ['ATODTAB'], 'BLEVCORR': ['OSCNTAB'], 'SINKCORR': ['SNKCFILE'], 'BIASCORR': ['BIASFILE'], 'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'], 'FLSHCORR': ['FLSHFILE'], 'CRCORR': ['CRREJTAB'], 'SHADCORR': ['SHADFILE'], 'DARKCORR': ['DARKFILE', 'TDCTAB'], 'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'], 'PHOTCORR': ['IMPHTTAB'], 'LFLGCORR': ['MLINTAB'], 'GLINCORR': ['MLINTAB'], 'NLINCORR': ['NLINFILE'], 'ZSIGCORR': ['DARKFILE', 'NLINFILE'], 'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'], 'SGEOCORR': ['SDSTFILE'], 'X1DCORR': ['XTRACTAB', 'SDCTAB'], 'SC2DCORR': [ 'CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB', 'SRWTAB' ], 'BACKCORR': ['XTRACTAB'], 'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB'] } hdr = fits.getheader(input_image, ext=0) # Mandatory CRDS reference file. # Destriping tries to ingest some *FILE regardless of *CORR. ref_files = ref_from_image(input_image, ['CCDTAB', 'DARKFILE', 'PFLTFILE']) for step in corr_lookup: # Not all images have the CORR step and it is not always on. # Destriping also does reverse-calib. if ((step not in hdr) or (hdr[step].strip().upper() not in ('PERFORM', 'COMPLETE'))): continue ref_files += ref_from_image(input_image, corr_lookup[step]) return list(set(ref_files)) # Remove duplicates
def calref_from_image(input_image): """ Return a list of reference filenames, as defined in the primary header of the given input image, necessary for calibration. This is mostly needed for destriping tools. """ # NOTE: Add additional mapping as needed. # Map *CORR to associated CRDS reference file. corr_lookup = { 'DQICORR': ['BPIXTAB', 'SNKCFILE'], 'ATODCORR': ['ATODTAB'], 'BLEVCORR': ['OSCNTAB'], 'SINKCORR': ['SNKCFILE'], 'BIASCORR': ['BIASFILE'], 'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'], 'FLSHCORR': ['FLSHFILE'], 'CRCORR': ['CRREJTAB'], 'SHADCORR': ['SHADFILE'], 'DARKCORR': ['DARKFILE', 'TDCTAB'], 'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'], 'PHOTCORR': ['IMPHTTAB'], 'LFLGCORR': ['MLINTAB'], 'GLINCORR': ['MLINTAB'], 'NLINCORR': ['NLINFILE'], 'ZSIGCORR': ['DARKFILE', 'NLINFILE'], 'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'], 'SGEOCORR': ['SDSTFILE'], 'X1DCORR': ['XTRACTAB', 'SDCTAB'], 'SC2DCORR': ['CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB', 'SRWTAB'], 'BACKCORR': ['XTRACTAB'], 'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']} hdr = fits.getheader(input_image, ext=0) # Mandatory CRDS reference file. # Destriping tries to ingest some *FILE regardless of *CORR. ref_files = ref_from_image(input_image, ['CCDTAB', 'DARKFILE', 'PFLTFILE']) for step in corr_lookup: # Not all images have the CORR step and it is not always on. # Destriping also does reverse-calib. if ((step not in hdr) or (hdr[step].strip().upper() not in ('PERFORM', 'COMPLETE'))): continue ref_files += ref_from_image(input_image, corr_lookup[step]) return list(set(ref_files)) # Remove duplicates
def calref_from_image(input_image): """ Return a list of reference filenames, as defined in the primary header of the given input image, necessary for calibration; i.e., only those associated with ``*CORR`` set to ``PERFORM`` will be considered. """ # NOTE: Add additional mapping as needed. # Map mandatory CRDS reference file for instrument/detector combo. # This is for file not tied to any particular *CORR or used throughout. det_lookup = { ('COS', 'FUV'): ['PROFTAB', 'SPWCSTAB'], ('COS', 'NUV'): []} # NOTE: Add additional mapping as needed. # Map *CORR to associated CRDS reference file. corr_lookup = { 'BADTCORR': ['BADTTAB'], 'TEMPCORR': ['BRFTAB'], 'GEOCORR': ['GEOFILE'], 'DGEOCORR': ['DGEOFILE'], 'YWLKCORR': ['YWLKFILE'], 'XWLKCORR': ['XWLKFILE'], 'DEADCORR': ['DEADTAB'], 'PHACORR': ['PHATAB', 'PHAFILE'], 'FLATCORR': ['FLATFILE'], 'WAVECORR': ['LAMPTAB', 'DISPTAB', 'TWOZXTAB', 'XTRACTAB'], 'BRSTCORR': ['BRSTTAB'], 'TRCECORR': ['TRACETAB'], 'ALGNCORR': ['TWOZXTAB'], 'DQICORR': ['SPOTTAB', 'TRACETAB', 'BPIXTAB', 'GSAGTAB'], 'X1DCORR': ['WCPTAB', 'TWOZXTAB', 'XTRACTAB'], 'BACKCORR': ['TWOZXTAB', 'XTRACTAB'], 'FLUXCORR': ['FLUXTAB', 'TDSTAB', 'PHOTTAB'], 'WALKCORR': ['WALKTAB']} hdr = fits.getheader(input_image, ext=0) ref_files = ref_from_image( input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])]) for step in corr_lookup: # Not all images have the CORR step and it is not always on. if (step not in hdr) or (hdr[step].strip().upper() != 'PERFORM'): continue ref_files += ref_from_image(input_image, corr_lookup[step]) return list(set(ref_files)) # Remove duplicates
def get_input_file(self, *args, refsep='$'): """ Download or copy input file (e.g., RAW) into the working directory. The associated CRDS reference files in ``refstr`` are also downloaded, if necessary. """ filename = self.get_data(*args) ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE']) print("Looking for REF_FILES: {}".format(ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file refname = self.get_data('customRef', ref_file) else: # Download from FTP, if applicable refname = os.path.join(ref_file) if self.use_ftp_crds: download_crds(refname, self.timeout) return filename
def get_input_file(self, *args, refsep='$', **kwargs): # If user has specified action for docopy, apply it with # default behavior being whatever was defined in the base class. docopy = kwargs.get('docopy', self.docopy) # Download or copy input file (e.g., RAW) into the working directory. # The associated CRDS reference files in ``refstr`` are also # downloaded, if necessary. curdir = os.getcwd() filenames = self.get_data(*args, docopy=docopy) for filename in filenames: ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup) print("Looking for {} REF_FILES: {}".format(filename, ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file self.get_data('customRef', ref_file, docopy=docopy) else: # Start by checking to see whether IRAF variable *ref/*tab # has been added to os.environ refdir, refname = ref_file.split(refsep) refdir_parent = os.path.split(refdir)[0] # Define refdir to point to current directory if: # i. refdir is not defined in environment already # ii. refdir in os.environ points to another test directory # This logic should leave refdir unchanged if it already # points to a globally defined directory. if refdir not in os.environ or refdir_parent in curdir: os.environ[refdir] = curdir + os.sep # Download from FTP, if applicable if self.use_ftp_crds: download_crds(ref_file, timeout=self.timeout) return filenames
def get_input_file(self, *args, refsep='$', **kwargs): # If user has specified action for docopy, apply it with # default behavior being whatever was defined in the base class. docopy = kwargs.get('docopy', self.docopy) # Download or copy input file (e.g., RAW) into the working directory. # The associated CRDS reference files in ``refstr`` are also # downloaded, if necessary. filename = self.get_data(*args, docopy=docopy) ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup) print("Looking for REF_FILES: {}".format(ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file self.get_data('customRef', ref_file, docopy=docopy) else: # Download from FTP, if applicable if self.use_ftp_crds: download_crds(ref_file, timeout=self.timeout) return filename
def calref_from_image(input_image): """ Return a list of reference filenames, as defined in the primary header of the given input image, necessary for calibration. """ # NOTE: Add additional mapping as needed. # Map mandatory CRDS reference file for instrument/detector combo. # This is for file not tied to any particular *CORR or used throughout. det_lookup = { ('ACS', 'WFC'): ['CCDTAB'], ('ACS', 'HRC'): ['CCDTAB'], ('ACS', 'SBC'): ['CCDTAB'], ('WFC3', 'UVIS'): ['CCDTAB'], ('WFC3', 'IR'): ['CCDTAB'], ('STIS', 'CCD'): ['CCDTAB'], ('STIS', 'FUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB'], ('STIS', 'NUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB']} # NOTE: Add additional mapping as needed. # Map *CORR to associated CRDS reference file. corr_lookup = { 'DQICORR': ['BPIXTAB', 'SNKCFILE'], 'ATODCORR': ['ATODTAB'], 'BLEVCORR': ['OSCNTAB'], 'SINKCORR': ['SNKCFILE'], 'BIASCORR': ['BIASFILE'], 'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'], 'FLSHCORR': ['FLSHFILE'], 'CRCORR': ['CRREJTAB'], 'SHADCORR': ['SHADFILE'], 'DARKCORR': ['DARKFILE', 'TDCTAB'], 'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'], 'PHOTCORR': ['IMPHTTAB'], 'LFLGCORR': ['MLINTAB'], 'GLINCORR': ['MLINTAB'], 'NLINCORR': ['NLINFILE'], 'ZSIGCORR': ['DARKFILE', 'NLINFILE'], 'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'], 'SGEOCORR': ['SDSTFILE'], 'X1DCORR': ['XTRACTAB', 'SDCTAB'], 'SC2DCORR': ['CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB', 'SRWTAB'], 'BACKCORR': ['XTRACTAB'], 'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB']} hdr = fits.getheader(input_image, ext=0) ref_files = ref_from_image( input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])]) for step in corr_lookup: # Not all images have the CORR step and it is not always on. # Download ALL reference files associated with a calibration # step present in the header if step not in hdr: continue single_step_files = ref_from_image(input_image, corr_lookup[step]) if single_step_files: ref_files += single_step_files return ref_files
def compare_wcs_alignment(dataset, force=False): """Return results from aligning dataset using all available WCS solutions. This code will ALWAYS make sure the ASTROMETRY_STEP_CONTROL variable is set to "ON" when running and will reset to the original state when completed. This insures that the code ALWAYS queries the astrometry database to apply all avaialable a priori WCS solutions. Parameters ----------- dataset : str Rootname of either a single (un-associated) exposure or an ASN force : bool Specify whether or not to overwrite dataset files found locally with fresh copies retrieved from MAST. Returns ------- results : dict A dictionary whose keys are the WCS's found and fit to GAIA. Each WCS has entries for: * imageName - filenames of input exposures included in the fit * offset_x - offset in X (pixels) * offset_y - offset in X (pixels) * rotation - rotation in degrees * scale - scale from fit * rms_x - RMS in pixels * rms_y - RMS in pixels * fit_rms - RMS in arcseconds * total_rms - RMS of entire fit in arcseconds * status - flag indicating success/failure of fit * fit_qual - flag indicating quality of fit (1-5) * matched_sources - number of sources used in fit ASSUMPTIONS ----------- - All images in dataset have the same set of a priori solutions - All images in dataset have the same setting for the IDCTAB file """ # Setup # Remember what state the environment was in before this code runs control = os.environ.get('ASTROMETRY_STEP_CONTROL') # Insure that database will be queried for new WCS solutions os.environ['ASTROMETRY_STEP_CONTROL'] = 'ON' try: # Step 1: # Determine alignment for pipeline-defined WCS align_table = align.perform_align([dataset], catalog_list=['GAIADR2', 'GAIADR1'], num_sources=250, clobber=force, debug=True, product_type='pipeline') results = align_table.filtered_table if not results: msg = "No valid exposures found for {}.".format(dataset) msg += "\n Please check that input was either a valid ASN" msg += "\n or a single un-associated exposure." raise ValueError(msg) imglist = results['imageName'].astype(str).tolist() # Step 2: # Create results output organized by WCSNAME default_wcsname = fits.getval(imglist[0], 'wcsname', ext=1) log.info("Default WCSNAME: {}".format(default_wcsname)) alignment = {default_wcsname: extract_results(results)} # Download the calibration reference files to ensure availability ref_files = ref_from_image( imglist[0], ['IDCTAB', 'DGEOFILE', 'NPOLFILE', 'D2IMFILE']) for file in ref_files: download_crds(file, verbose=True) # Step 3: # Update inputs with latest distortion model and pull in solutions from dB imglist = updatewcs.updatewcs(imglist) img0 = imglist[0] # Step 4: # Loop over each WCS solution and perform alignment to GAIA wcsnames = headerlet.get_headerlet_kw_names(img0, kw='WCSNAME') if not wcsnames: msg = "No a priori solutions found for {}".format(img0) log.error(msg) raise ValueError(msg) for wcs in wcsnames: log.info("Starting with {}".format(wcs)) if 'OPUS' in wcs or wcs == default_wcsname: continue # skip default pipeline solutions, since we have already aligned it # apply WCS from headerlet for img in imglist: wnames = headerlet.get_headerlet_kw_names(img, kw='WCSNAME') hnames = headerlet.get_headerlet_kw_names(img) # print("[testutils]WCSNAMES[{}]: {}".format(img, wnames)) if wcs in wnames: hdrname = hnames[wnames.index(wcs)] log.info("[testutils] Applying WCS {} to {}".format( hdrname, img)) headerlet.restore_from_headerlet(img, hdrname=hdrname, archive=False, force=True) print("[testutils] Aligning: {} for WCSNAME: {}".format( dataset, wcs)) align_table = align.perform_align( [dataset], catalog_list=['GAIADR2', 'GAIADR1'], num_sources=250, clobber=False, debug=True, product_type='pipeline') results = align_table.filtered_table alignment[wcs] = extract_results(results) except Exception as err: print(traceback.format_exc()) raise err finally: # Regardless of what happens, always reset the environment variable # if it was modified in the first place. # Restore user environment to original state if control is None: # Need to be explicit here since T/F are actually valid del os.environ['ASTROMETRY_STEP_CONTROL'] else: os.environ['ASTROMETRY_STEP_CONTROL'] = control return alignment
def calref_from_image(input_image): """ Return a list of reference filenames, as defined in the primary header of the given input image, necessary for calibration. """ # NOTE: Add additional mapping as needed. # Map mandatory CRDS reference file for instrument/detector combo. # This is for file not tied to any particular *CORR or used throughout. det_lookup = { ('ACS', 'WFC'): ['CCDTAB'], ('ACS', 'HRC'): ['CCDTAB'], ('ACS', 'SBC'): ['CCDTAB'], ('WFC3', 'UVIS'): ['CCDTAB'], ('WFC3', 'IR'): ['CCDTAB'], ('STIS', 'CCD'): ['CCDTAB'], ('STIS', 'FUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB'], ('STIS', 'NUV-MAMA'): ['CCDTAB', 'DISPTAB', 'INANGTAB', 'APDESTAB', 'SPTRCTAB'] } # NOTE: Add additional mapping as needed. # Map *CORR to associated CRDS reference file. corr_lookup = { 'DQICORR': ['BPIXTAB', 'SNKCFILE'], 'ATODCORR': ['ATODTAB'], 'BLEVCORR': ['OSCNTAB'], 'SINKCORR': ['SNKCFILE'], 'BIASCORR': ['BIASFILE'], 'PCTECORR': ['PCTETAB', 'DRKCFILE', 'BIACFILE'], 'FLSHCORR': ['FLSHFILE'], 'CRCORR': ['CRREJTAB'], 'SHADCORR': ['SHADFILE'], 'DARKCORR': ['DARKFILE', 'TDCTAB'], 'FLATCORR': ['PFLTFILE', 'DFLTFILE', 'LFLTFILE'], 'PHOTCORR': ['IMPHTTAB'], 'LFLGCORR': ['MLINTAB'], 'GLINCORR': ['MLINTAB'], 'NLINCORR': ['NLINFILE'], 'ZSIGCORR': ['DARKFILE', 'NLINFILE'], 'WAVECORR': ['LAMPTAB', 'WCPTAB', 'SDCTAB'], 'SGEOCORR': ['SDSTFILE'], 'X1DCORR': ['XTRACTAB', 'SDCTAB'], 'SC2DCORR': [ 'CDSTAB', 'ECHSCTAB', 'EXSTAB', 'RIPTAB', 'HALOTAB', 'TELTAB', 'SRWTAB' ], 'BACKCORR': ['XTRACTAB'], 'FLUXCORR': ['APERTAB', 'PHOTTAB', 'PCTAB', 'TDSTAB'] } hdr = fits.getheader(input_image, ext=0) ref_files = ref_from_image(input_image, det_lookup[(hdr['INSTRUME'], hdr['DETECTOR'])]) for step in corr_lookup: # Not all images have the CORR step and it is not always on. # Download ALL reference files associated with a calibration # step present in the header if step not in hdr: continue single_step_files = ref_from_image(input_image, corr_lookup[step]) if single_step_files: ref_files += single_step_files return ref_files