def test_raw_from_asn(_jail): # Make a dummy input file (to avoid package data headache) tab = Table() tab['MEMNAME'] = ['J6LQ01NAQ', 'J6LQ01NDQ', 'J6LQ01011'] tab['MEMTYPE'] = ['EXP-CRJ', 'EXP-CRJ', 'PROD-CRJ'] tab['MEMPRSNT'] = [True, True, True] datafile = 'dummy_asn.fits' tab.write(datafile, format='fits', overwrite=True) raw_files = raw_from_asn(datafile) assert raw_files == ['j6lq01naq_raw.fits', 'j6lq01ndq_raw.fits'] # Make sure do not download existing file. # This will fail if download is attemped. download_crds(datafile)
def get_input_file(self, *args, refsep='$'): """ Download or copy input file (e.g., RAW) into the working directory. The associated CRDS reference files in ``refstr`` are also downloaded, if necessary. """ filename = self.get_data(*args) ref_files = ref_from_image(filename, ['IDCTAB', 'OFFTAB', 'NPOLFILE', 'D2IMFILE', 'DGEOFILE']) print("Looking for REF_FILES: {}".format(ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file refname = self.get_data('customRef', ref_file) else: # Download from FTP, if applicable refname = os.path.join(ref_file) if self.use_ftp_crds: download_crds(refname, self.timeout) return filename
def get_input_file(self, *args, refsep='$', **kwargs): # If user has specified action for docopy, apply it with # default behavior being whatever was defined in the base class. docopy = kwargs.get('docopy', self.docopy) # Download or copy input file (e.g., RAW) into the working directory. # The associated CRDS reference files in ``refstr`` are also # downloaded, if necessary. curdir = os.getcwd() filenames = self.get_data(*args, docopy=docopy) for filename in filenames: ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup) print("Looking for {} REF_FILES: {}".format(filename, ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file self.get_data('customRef', ref_file, docopy=docopy) else: # Start by checking to see whether IRAF variable *ref/*tab # has been added to os.environ refdir, refname = ref_file.split(refsep) refdir_parent = os.path.split(refdir)[0] # Define refdir to point to current directory if: # i. refdir is not defined in environment already # ii. refdir in os.environ points to another test directory # This logic should leave refdir unchanged if it already # points to a globally defined directory. if refdir not in os.environ or refdir_parent in curdir: os.environ[refdir] = curdir + os.sep # Download from FTP, if applicable if self.use_ftp_crds: download_crds(ref_file, timeout=self.timeout) return filenames
def get_input_file(self, *args, refsep='$', **kwargs): # If user has specified action for docopy, apply it with # default behavior being whatever was defined in the base class. docopy = kwargs.get('docopy', self.docopy) # Download or copy input file (e.g., RAW) into the working directory. # The associated CRDS reference files in ``refstr`` are also # downloaded, if necessary. filename = self.get_data(*args, docopy=docopy) ref_files = ref_from_image(filename, reffile_lookup=self.reffile_lookup) print("Looking for REF_FILES: {}".format(ref_files)) for ref_file in ref_files: if ref_file.strip() == '': continue if refsep not in ref_file: # Local file self.get_data('customRef', ref_file, docopy=docopy) else: # Download from FTP, if applicable if self.use_ftp_crds: download_crds(ref_file, timeout=self.timeout) return filename
def get_input_file(self, filename): """ Copy input file (ASN, RAW, etc) into the working directory. If ASN is given, RAW files in the ASN table is also copied. The associated CRDS reference files are also copied or downloaded, if necessary. Data directory layout for HSTCAL:: instrument/ detector/ input/ truth/ .. note:: If given filename has ".orig" suffix, the suffix is also automatically removed. Parameters ---------- filename : str Filename of the ASN/RAW/etc to copy over, along with its associated files. """ # Copy over main input file. dest = get_bigdata(self.env, self.instrument, self.detector, 'input', filename) # For historical reason, need to remove ".orig" suffix if it exists. if filename.endswith('.orig'): newfilename = filename.rstrip('.orig') os.rename(filename, newfilename) filename = newfilename if filename.endswith('_asn.fits'): all_raws = raw_from_asn(filename) for raw in all_raws: # Download RAWs in ASN. get_bigdata(self.env, self.instrument, self.detector, 'input', raw) else: all_raws = [filename] first_pass = ('JENKINS_URL' in os.environ and 'ssbjenkins' in os.environ['JENKINS_URL']) for raw in all_raws: ref_files = calref_from_image(raw) for ref_file in ref_files: # Special reference files that live with inputs. if ('$' not in ref_file and os.path.basename(ref_file) == ref_file): get_bigdata(self.env, self.instrument, self.detector, 'input', ref_file) continue # Jenkins cannot see Central Storage on push event, # and somehow setting, say, jref to "." does not work anymore. # So, we need this hack. if '$' in ref_file and first_pass: first_pass = False if not os.path.isdir('/grp/hst/cdbs'): ref_path = os.path.dirname(dest) + os.sep var = ref_file.split('$')[0] os.environ[var] = ref_path # hacky hack hack # Download reference files, if needed only. download_crds(ref_file, timeout=self.timeout)
def compare_wcs_alignment(dataset, force=False): """Return results from aligning dataset using all available WCS solutions. This code will ALWAYS make sure the ASTROMETRY_STEP_CONTROL variable is set to "ON" when running and will reset to the original state when completed. This insures that the code ALWAYS queries the astrometry database to apply all avaialable a priori WCS solutions. Parameters ----------- dataset : str Rootname of either a single (un-associated) exposure or an ASN force : bool Specify whether or not to overwrite dataset files found locally with fresh copies retrieved from MAST. Returns ------- results : dict A dictionary whose keys are the WCS's found and fit to GAIA. Each WCS has entries for: * imageName - filenames of input exposures included in the fit * offset_x - offset in X (pixels) * offset_y - offset in X (pixels) * rotation - rotation in degrees * scale - scale from fit * rms_x - RMS in pixels * rms_y - RMS in pixels * fit_rms - RMS in arcseconds * total_rms - RMS of entire fit in arcseconds * status - flag indicating success/failure of fit * fit_qual - flag indicating quality of fit (1-5) * matched_sources - number of sources used in fit ASSUMPTIONS ----------- - All images in dataset have the same set of a priori solutions - All images in dataset have the same setting for the IDCTAB file """ # Setup # Remember what state the environment was in before this code runs control = os.environ.get('ASTROMETRY_STEP_CONTROL') # Insure that database will be queried for new WCS solutions os.environ['ASTROMETRY_STEP_CONTROL'] = 'ON' try: # Step 1: # Determine alignment for pipeline-defined WCS align_table = align.perform_align([dataset], catalog_list=['GAIADR2', 'GAIADR1'], num_sources=250, clobber=force, debug=True, product_type='pipeline') results = align_table.filtered_table if not results: msg = "No valid exposures found for {}.".format(dataset) msg += "\n Please check that input was either a valid ASN" msg += "\n or a single un-associated exposure." raise ValueError(msg) imglist = results['imageName'].astype(str).tolist() # Step 2: # Create results output organized by WCSNAME default_wcsname = fits.getval(imglist[0], 'wcsname', ext=1) log.info("Default WCSNAME: {}".format(default_wcsname)) alignment = {default_wcsname: extract_results(results)} # Download the calibration reference files to ensure availability ref_files = ref_from_image( imglist[0], ['IDCTAB', 'DGEOFILE', 'NPOLFILE', 'D2IMFILE']) for file in ref_files: download_crds(file, verbose=True) # Step 3: # Update inputs with latest distortion model and pull in solutions from dB imglist = updatewcs.updatewcs(imglist) img0 = imglist[0] # Step 4: # Loop over each WCS solution and perform alignment to GAIA wcsnames = headerlet.get_headerlet_kw_names(img0, kw='WCSNAME') if not wcsnames: msg = "No a priori solutions found for {}".format(img0) log.error(msg) raise ValueError(msg) for wcs in wcsnames: log.info("Starting with {}".format(wcs)) if 'OPUS' in wcs or wcs == default_wcsname: continue # skip default pipeline solutions, since we have already aligned it # apply WCS from headerlet for img in imglist: wnames = headerlet.get_headerlet_kw_names(img, kw='WCSNAME') hnames = headerlet.get_headerlet_kw_names(img) # print("[testutils]WCSNAMES[{}]: {}".format(img, wnames)) if wcs in wnames: hdrname = hnames[wnames.index(wcs)] log.info("[testutils] Applying WCS {} to {}".format( hdrname, img)) headerlet.restore_from_headerlet(img, hdrname=hdrname, archive=False, force=True) print("[testutils] Aligning: {} for WCSNAME: {}".format( dataset, wcs)) align_table = align.perform_align( [dataset], catalog_list=['GAIADR2', 'GAIADR1'], num_sources=250, clobber=False, debug=True, product_type='pipeline') results = align_table.filtered_table alignment[wcs] = extract_results(results) except Exception as err: print(traceback.format_exc()) raise err finally: # Regardless of what happens, always reset the environment variable # if it was modified in the first place. # Restore user environment to original state if control is None: # Need to be explicit here since T/F are actually valid del os.environ['ASTROMETRY_STEP_CONTROL'] else: os.environ['ASTROMETRY_STEP_CONTROL'] = control return alignment
def get_input_file(self, filename): """ Copy input file (ASN, RAW, etc) into the working directory. If ASN is given, RAW files in the ASN table is also copied. The associated CRDS reference files are also copied or downloaded, if necessary. Data directory layout for CALCOS:: detector/ input/ truth/ Parameters ---------- filename : str Filename of the ASN/RAW/etc to copy over, along with its associated files. """ # Copy over main input file. dest = get_bigdata('scsb-calcos', self.env, self.detector, 'input', filename) # TODO: This logic was based on ONE FUV test. Please revise as needed. if filename.endswith('_asn.fits'): all_raws = raw_from_asn(filename, '_rawtag_a.fits') all_raws += raw_from_asn(filename, '_rawtag_b.fits') for raw in all_raws: # Download RAWs in ASN. get_bigdata('scsb-calcos', self.env, self.detector, 'input', raw) for raw in raw_from_asn(filename, '_spt.fits'): # Input SPTs get_bigdata('scsb-calcos', self.env, self.detector, 'input', raw) else: all_raws = [filename] first_pass = ('JENKINS_URL' in os.environ and 'ssbjenkins' in os.environ['JENKINS_URL']) for raw in all_raws: ref_files = calref_from_image(raw) for ref_file in ref_files: # Special reference files that live with inputs. if ('$' not in ref_file and os.path.basename(ref_file) == ref_file): get_bigdata('scsb-calcos', self.env, self.detector, 'input', ref_file) continue # Jenkins cannot see Central Storage on push event, # and somehow setting, say, jref to "." does not work anymore. # So, we need this hack. if '$' in ref_file and first_pass: first_pass = False if not os.path.isdir('/grp/hst/cdbs'): ref_path = os.path.dirname(dest) + os.sep var = ref_file.split('$')[0] os.environ[var] = ref_path # hacky hack hack # Download reference files, if needed only. download_crds(ref_file, timeout=self.timeout)
def get_input_files(self, filenames): """ Copy input files (ASN, RAW, etc) into the working directory. If ASN is given, RAW files in the ASN table are also copied. The associated CRDS reference files are also copied or downloaded, if necessary. Data directory layout for CALCOS:: detector/ input/ truth/ Parameters ---------- filename : list List of filenames of the ASN/RAW/etc to copy over, along with their associated files. """ all_raws = [] for file in filenames: if file.endswith('_rawtag_a.fits') or file.endswith('_rawtag_b.fits'): all_raws.append(file) # List of filenames can include _rawtag, _asn and _spt files dest = get_bigdata('scsb-calcos', self.env, self.detector, 'input', file) # If file is an association table, download raw files specified in the table if file.endswith('_asn.fits'): asn_raws = raw_from_asn(file, '_rawtag_a.fits') asn_raws += raw_from_asn(file, '_rawtag_b.fits') for raw in asn_raws: # Download RAWs in ASN. get_bigdata('scsb-calcos', self.env, self.detector, 'input', raw) all_raws += asn_raws first_pass = ('JENKINS_URL' in os.environ and 'ssbjenkins' in os.environ['JENKINS_URL']) for raw in all_raws: ref_files = calref_from_image(raw) for ref_file in ref_files: # Special reference files that live with inputs. if ('$' not in ref_file and os.path.basename(ref_file) == ref_file): get_bigdata('scsb-calcos', self.env, self.detector, 'input', ref_file) continue # Jenkins cannot see Central Storage on push event, # and somehow setting, say, jref to "." does not work anymore. # So, we need this hack. if '$' in ref_file and first_pass: first_pass = False if not os.path.isdir('/grp/hst/cdbs'): ref_path = os.path.dirname(dest) + os.sep var = ref_file.split('$')[0] os.environ[var] = ref_path # hacky hack hack # Download reference files, if needed only. download_crds(ref_file, timeout=self.timeout)